Java DictionaryManager类代码示例

OStack程序员社区-中国程序员成长平台 › 门户 › 编程› Java›Java编程经验

原作者: [db:作者] 来自: [db:来源] 收藏邀请

本文整理汇总了Java中org.apache.kylin.dict.DictionaryManager类的典型用法代码示例。如果您正苦于以下问题：Java DictionaryManager类的具体用法？Java DictionaryManager怎么用？Java DictionaryManager使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。

DictionaryManager类属于org.apache.kylin.dict包，在下文中一共展示了DictionaryManager类的17个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Java代码示例。

示例1: makeDictForNewSegment

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
/**
 * For the new segment, we need to create new dimension dictionaries by merging underlying
 * dictionaries. (https://issues.apache.org/jira/browse/KYLIN-2457, https://issues.apache.org/jira/browse/KYLIN-2800)
 * @param cube
 * @param newSeg
 * @throws IOException
 */
private void makeDictForNewSegment(KylinConfig conf, CubeInstance cube, CubeSegment newSeg, List<CubeSegment> mergingSegments) throws IOException {
    DictionaryManager dictMgr = DictionaryManager.getInstance(conf);
    CubeDesc cubeDesc = cube.getDescriptor();

    for (TblColRef col : cubeDesc.getAllColumnsNeedDictionaryBuilt()) {
        logger.info("Merging fact table dictionary on : " + col);
        List<DictionaryInfo> dictInfos = new ArrayList<DictionaryInfo>();
        for (CubeSegment segment : mergingSegments) {
            logger.info("Including fact table dictionary of segment : " + segment);
            if (segment.getDictResPath(col) != null) {
                DictionaryInfo dictInfo = dictMgr.getDictionaryInfo(segment.getDictResPath(col));
                if (dictInfo != null && !dictInfos.contains(dictInfo)) {
                    dictInfos.add(dictInfo);
                } else {
                    logger.warn("Failed to load DictionaryInfo from " + segment.getDictResPath(col));
                }
            }
        }
        mergeDictionaries(dictMgr, newSeg, dictInfos, col);
    }
}

开发者ID:apache，项目名称:kylin，代码行数:29，代码来源:MergeDictionaryStep.java

示例2: writeDictionary

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public static Map<TblColRef, Dictionary<String>> writeDictionary(CubeSegment cubeSegment, Map<TblColRef, Dictionary<String>> dictionaryMap, long startOffset, long endOffset) {
    Map<TblColRef, Dictionary<String>> realDictMap = Maps.newHashMap();

    for (Map.Entry<TblColRef, Dictionary<String>> entry : dictionaryMap.entrySet()) {
        final TblColRef tblColRef = entry.getKey();
        final Dictionary<String> dictionary = entry.getValue();
        IReadableTable.TableSignature signature = new IReadableTable.TableSignature();
        signature.setLastModifiedTime(System.currentTimeMillis());
        signature.setPath(String.format("streaming_%s_%s", startOffset, endOffset));
        signature.setSize(endOffset - startOffset);
        DictionaryInfo dictInfo = new DictionaryInfo(tblColRef.getColumnDesc(), tblColRef.getDatatype(), signature);
        logger.info("writing dictionary for TblColRef:" + tblColRef.toString());
        DictionaryManager dictionaryManager = DictionaryManager.getInstance(cubeSegment.getCubeDesc().getConfig());
        try {
            DictionaryInfo realDict = dictionaryManager.trySaveNewDict(dictionary, dictInfo);
            cubeSegment.putDictResPath(tblColRef, realDict.getResourcePath());
            realDictMap.put(tblColRef, (Dictionary<String>) realDict.getDictionaryObject());
        } catch (IOException e) {
            throw new RuntimeException("error save dictionary for column:" + tblColRef, e);
        }
    }

    return realDictMap;
}

开发者ID:apache，项目名称:kylin，代码行数:26，代码来源:CubingUtils.java

示例3: getDictionary

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
/**
 * return null if no dictionary for given column
 */
@SuppressWarnings("unchecked")
public Dictionary<String> getDictionary(CubeSegment cubeSeg, TblColRef col) {
    DictionaryInfo info = null;
    try {
        DictionaryManager dictMgr = getDictionaryManager();
        String dictResPath = cubeSeg.getDictResPath(col);
        if (dictResPath == null)
            return null;

        info = dictMgr.getDictionaryInfo(dictResPath);
        if (info == null)
            throw new IllegalStateException("No dictionary found by " + dictResPath
                    + ", invalid cube state; cube segment" + cubeSeg + ", col " + col);
    } catch (IOException e) {
        throw new IllegalStateException("Failed to get dictionary for cube segment" + cubeSeg + ", col" + col,
                e);
    }
    return (Dictionary<String>) info.getDictionaryObject();
}

开发者ID:apache，项目名称:kylin，代码行数:23，代码来源:CubeManager.java

示例4: getDictionary

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
/**
 * return null if no dictionary for given column
 */
public Dictionary<?> getDictionary(CubeSegment cubeSeg, TblColRef col) {
    DictionaryInfo info = null;
    try {
        DictionaryManager dictMgr = getDictionaryManager();
        // logger.info("Using metadata url " + metadataUrl +
        // " for DictionaryManager");
        String dictResPath = cubeSeg.getDictResPath(col);
        if (dictResPath == null)
            return null;

        info = dictMgr.getDictionaryInfo(dictResPath);
        if (info == null)
            throw new IllegalStateException("No dictionary found by " + dictResPath + ", invalid cube state; cube segment" + cubeSeg + ", col " + col);
    } catch (IOException e) {
        throw new IllegalStateException("Failed to get dictionary for cube segment" + cubeSeg + ", col" + col, e);
    }

    return info.getDictionaryObject();
}

开发者ID:KylinOLAP，项目名称:Kylin，代码行数:23，代码来源:CubeManager.java

示例5: getDictionary

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
/**
 * return null if no dictionary for given column
 */
public Dictionary<?> getDictionary(IISegment iiSeg, TblColRef col) {
    DictionaryInfo info = null;
    try {
        DictionaryManager dictMgr = getDictionaryManager();
        // logger.info("Using metadata url " + metadataUrl +
        // " for DictionaryManager");
        String dictResPath = iiSeg.getDictResPath(col);
        if (dictResPath == null)
            return null;

        info = dictMgr.getDictionaryInfo(dictResPath);
        if (info == null)
            throw new IllegalStateException("No dictionary found by " + dictResPath + ", invalid II state; II segment" + iiSeg + ", col " + col);
    } catch (IOException e) {
        throw new IllegalStateException("Failed to get dictionary for II segment" + iiSeg + ", col" + col, e);
    }

    return info.getDictionaryObject();
}

开发者ID:KylinOLAP，项目名称:Kylin，代码行数:23，代码来源:IIManager.java

示例6: basic

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
@Test
public void basic() throws Exception {
    dictMgr = DictionaryManager.getInstance(getTestConfig());
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_desc");
    TblColRef col = cubeDesc.findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_FORMAT_NAME");

    MockDistinctColumnValuesProvider mockupData = new MockDistinctColumnValuesProvider("A", "B", "C");

    DictionaryInfo info1 = dictMgr.buildDictionary(col, mockupData.getDistinctValuesFor(col));
    System.out.println(JsonUtil.writeValueAsIndentString(info1));

    DictionaryInfo info2 = dictMgr.buildDictionary(col, mockupData.getDistinctValuesFor(col));
    System.out.println(JsonUtil.writeValueAsIndentString(info2));

    // test check duplicate
    assertTrue(info1.getUuid() == info2.getUuid());
    assertTrue(info1 == dictMgr.getDictionaryInfo(info1.getResourcePath()));
    assertTrue(info2 == dictMgr.getDictionaryInfo(info2.getResourcePath()));
    assertTrue(info1.getDictionaryObject() == info2.getDictionaryObject());

    // verify dictionary entries
    @SuppressWarnings("unchecked")
    Dictionary<String> dict = (Dictionary<String>) info1.getDictionaryObject();
    int id = 0;
    for (String v : mockupData.set) {
        assertEquals(id, dict.getIdFromValue(v, 0));
        assertEquals(v, dict.getValueFromId(id));
        id++;
    }

    // test empty dictionary
    MockDistinctColumnValuesProvider mockupEmpty = new MockDistinctColumnValuesProvider();
    DictionaryInfo info3 = dictMgr.buildDictionary(col, mockupEmpty.getDistinctValuesFor(col));
    System.out.println(JsonUtil.writeValueAsIndentString(info3));
    assertEquals(0, info3.getCardinality());
    assertEquals(0, info3.getDictionaryObject().getSize());
    System.out.println(info3.getDictionaryObject().getMaxId());
    System.out.println(info3.getDictionaryObject().getMinId());
    System.out.println(info3.getDictionaryObject().getSizeOfId());
}

开发者ID:apache，项目名称:kylin，代码行数:41，代码来源:ITDictionaryManagerTest.java

示例7: mergeDictionaries

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
private DictionaryInfo mergeDictionaries(DictionaryManager dictMgr, CubeSegment cubeSeg, List<DictionaryInfo> dicts, TblColRef col) throws IOException {
    DictionaryInfo dictInfo = dictMgr.mergeDictionary(dicts);
    if (dictInfo != null)
        cubeSeg.putDictResPath(col, dictInfo.getResourcePath());

    return dictInfo;
}

开发者ID:apache，项目名称:kylin，代码行数:8，代码来源:MergeDictionaryStep.java

示例8: checkNeedMerging

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
private Boolean checkNeedMerging(TblColRef col) throws IOException {
    Boolean ret = dictsNeedMerging.get(col);
    if (ret != null)
        return ret;
    else {
        ret = cubeDesc.getRowkey().isUseDictionary(col);
        if (ret) {
            String dictTable = (String) DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
            ret = cubeDesc.getFactTable().equalsIgnoreCase(dictTable);
        }
        dictsNeedMerging.put(col, ret);
        return ret;
    }
}

开发者ID:KylinOLAP，项目名称:Kylin，代码行数:15，代码来源:MergeCuboidMapper.java

示例9: setup

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
@Override
protected void setup(Context context) throws IOException {
    super.publishConfiguration(context.getConfiguration());

    Configuration conf = context.getConfiguration();

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
    cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
    cube = CubeManager.getInstance(config).getCube(cubeName);
    cubeDesc = cube.getDescriptor();
    intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
    List<TblColRef> columns = baseCuboid.getColumns();

    ArrayList<Integer> factDictCols = new ArrayList<Integer>();
    RowKeyDesc rowkey = cubeDesc.getRowkey();
    DictionaryManager dictMgr = DictionaryManager.getInstance(config);
    for (int i = 0; i < columns.size(); i++) {
        TblColRef col = columns.get(i);
        if (rowkey.isUseDictionary(col) == false)
            continue;

        String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
        if (cubeDesc.getModel().isFactTable(scanTable)) {
            factDictCols.add(i);
        }
    }
    this.factDictCols = new int[factDictCols.size()];
    for (int i = 0; i < factDictCols.size(); i++)
        this.factDictCols[i] = factDictCols.get(i);

    schema = HCatInputFormat.getTableSchema(context.getConfiguration());
}

开发者ID:KylinOLAP，项目名称:Kylin，代码行数:36，代码来源:FactDistinctColumnsMapper.java

示例10: setUp

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
@Before
public void setUp() throws Exception {
    this.createTestMetadata();

    MetadataManager.clearCache();
    DictionaryManager.clearCache();
    CubeDescManager.clearCache();
    CubeManager.clearCache();
    IIDescManager.clearCache();
    IIManager.clearCache();
    ProjectManager.clearCache();
}

开发者ID:KylinOLAP，项目名称:Kylin，代码行数:13，代码来源:ServiceTestBase.java

示例11: buildDictionary

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, String factColumnsPath) throws IOException {
    CubeDesc cubeDesc = cubeSeg.getCubeDesc();
    if (!cubeDesc.getRowkey().isUseDictionary(col))
        return null;

    DictionaryManager dictMgr = getDictionaryManager();
    DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, factColumnsPath);
    cubeSeg.putDictResPath(col, dictInfo.getResourcePath());

    saveResource(cubeSeg.getCubeInstance());

    return dictInfo;
}

开发者ID:KylinOLAP，项目名称:Kylin，代码行数:14，代码来源:CubeManager.java

示例12: TableRecordInfo

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
public TableRecordInfo(IISegment iiSegment) {

        seg = iiSegment;
        desc = seg.getIIInstance().getDescriptor();
        allColumns = desc.listAllColumns();
        nColumns = allColumns.size();
        dictionaries = new Dictionary<?>[nColumns];
        measureSerializers = new FixedLenMeasureCodec<?>[nColumns];

        DictionaryManager dictMgr = DictionaryManager.getInstance(desc.getConfig());
        int index = 0;
        for (TblColRef tblColRef : desc.listAllColumns()) {
            ColumnDesc col = tblColRef.getColumn();
            if (desc.isMetricsCol(index)) {
                measureSerializers[index] = FixedLenMeasureCodec.get(col.getType());
            } else {
                String dictPath = seg.getDictResPath(tblColRef);
                try {
                    dictionaries[index] = dictMgr.getDictionary(dictPath);
                } catch (IOException e) {
                    throw new RuntimeException("dictionary " + dictPath + " does not exist ", e);
                }
            }
            index++;
        }

        digest = createDigest();
    }

开发者ID:KylinOLAP，项目名称:Kylin，代码行数:29，代码来源:TableRecordInfo.java

示例13: buildInvertedIndexDictionary

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
public void buildInvertedIndexDictionary(IISegment iiSeg, String factColumnsPath) throws IOException {
    logger.info("Start building ii dictionary");
    DictionaryManager dictMgr = getDictionaryManager();
    IIDesc iiDesc = iiSeg.getIIInstance().getDescriptor();
    for (TblColRef column : iiDesc.listAllColumns()) {
        logger.info("Dealing with column {}", column);
        if (iiDesc.isMetricsCol(column)) {
            continue;
        }

        DictionaryInfo dict = dictMgr.buildDictionary(iiDesc.getModel(), "true", column, factColumnsPath);
        iiSeg.putDictResPath(column, dict.getResourcePath());
    }
    saveResource(iiSeg.getIIInstance());
}

开发者ID:KylinOLAP，项目名称:Kylin，代码行数:16，代码来源:IIManager.java

示例14: getDictionaryManager

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
private DictionaryManager getDictionaryManager() {
    return DictionaryManager.getInstance(config);
}

开发者ID:apache，项目名称:kylin，代码行数:4，代码来源:CubeManager.java

示例15: mergeDictionaries

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
private DictionaryInfo mergeDictionaries(DictionaryManager dictMgr, CubeSegment cubeSeg, List<DictionaryInfo> dicts, TblColRef col) throws IOException {
    DictionaryInfo dictInfo = dictMgr.mergeDictionary(dicts);
    cubeSeg.putDictResPath(col, dictInfo.getResourcePath());

    return dictInfo;
}

开发者ID:KylinOLAP，项目名称:Kylin，代码行数:7，代码来源:MergeDictionaryStep.java

示例16: setUp

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
@Before
public void setUp() throws Exception {

    createTestMetadata();

    logger.info("The metadataUrl is : " + getTestConfig());

    MetadataManager.clearCache();
    CubeManager.clearCache();
    ProjectManager.clearCache();
    DictionaryManager.clearCache();

    // hack for distributed cache
    // CubeManager.removeInstance(KylinConfig.createInstanceFromUri("../job/meta"));//to
    // make sure the following mapper could get latest CubeManger
    FileUtils.deleteDirectory(new File("../job/meta"));

    MergeCuboidMapper mapper = new MergeCuboidMapper();
    mapDriver = MapDriver.newMapDriver(mapper);

    cubeManager = CubeManager.getInstance(getTestConfig());
    cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_ready_2_segments");
    dictionaryManager = DictionaryManager.getInstance(getTestConfig());
    lfn = cube.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_FORMAT_NAME");
    lsi = cube.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "CAL_DT");
    ssc = cube.getDescriptor().findColumnRef("DEFAULT.TEST_CATEGORY_GROUPINGS", "META_CATEG_NAME");

    DictionaryInfo sharedDict = makeSharedDict();

    boolean isFirstSegment = true;
    for (CubeSegment segment : cube.getSegments()) {

        TableSignature signature = new TableSignature();
        signature.setSize(100);
        signature.setLastModifiedTime(System.currentTimeMillis());
        signature.setPath("fake_dict_for" + lfn.getName() + segment.getName());

        DictionaryInfo newDictInfo = new DictionaryInfo(lfn.getTable(), lfn.getColumn().getName(), lfn.getColumn().getZeroBasedIndex(), "string", signature, "");

        List<byte[]> values = new ArrayList<byte[]>();
        values.add(new byte[] { 97, 97, 97 });
        if (isFirstSegment)
            values.add(new byte[] { 99, 99, 99 });
        else
            values.add(new byte[] { 98, 98, 98 });
        Dictionary<?> dict = DictionaryGenerator.buildDictionaryFromValueList(newDictInfo, values);
        dictionaryManager.trySaveNewDict(dict, newDictInfo);
        ((TrieDictionary) dict).dump(System.out);

        segment.putDictResPath(lfn, newDictInfo.getResourcePath());
        segment.putDictResPath(lsi, sharedDict.getResourcePath());
        segment.putDictResPath(ssc, sharedDict.getResourcePath());

        // cubeManager.saveResource(segment.getCubeInstance());
        // cubeManager.afterCubeUpdated(segment.getCubeInstance());
        cubeManager.updateCube(cube);

        isFirstSegment = false;
    }

}

开发者ID:KylinOLAP，项目名称:Kylin，代码行数:62，代码来源:MergeCuboidMapperTest.java

示例17: setup

import org.apache.kylin.dict.DictionaryManager; //导入依赖的package包/类
@Before
public void setup() throws Exception {
    createTestMetadata();
    dictMgr = DictionaryManager.getInstance(getTestConfig());
}

开发者ID:KylinOLAP，项目名称:Kylin，代码行数:6，代码来源:DictionaryManagerTest.java

注：本文中的org.apache.kylin.dict.DictionaryManager类示例整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。