本文整理汇总了Java中org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName类的典型用法代码示例。如果您正苦于以下问题:Java PrimitiveTypeName类的具体用法?Java PrimitiveTypeName怎么用?Java PrimitiveTypeName使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
PrimitiveTypeName类属于org.apache.parquet.schema.PrimitiveType包,在下文中一共展示了PrimitiveTypeName类的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Java代码示例。
示例1: ColumnReader
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
protected ColumnReader(DeprecatedParquetVectorizedReader parentReader, int allocateSize, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, V v, SchemaElement schemaElement) throws ExecutionSetupException {
this.parentReader = parentReader;
this.columnDescriptor = descriptor;
this.columnChunkMetaData = columnChunkMetaData;
this.isFixedLength = fixedLength;
this.schemaElement = schemaElement;
this.valueVec = v;
this.pageReader = (parentReader.getSingleStream() != null)?
new DeprecatedSingleStreamPageReader(this, parentReader.getSingleStream(), parentReader.getHadoopPath(), columnChunkMetaData) :
new PageReader(this, parentReader.getFileSystem(), parentReader.getHadoopPath(), columnChunkMetaData);
if (columnDescriptor.getType() != PrimitiveType.PrimitiveTypeName.BINARY) {
if (columnDescriptor.getType() == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) {
dataTypeLengthInBits = columnDescriptor.getTypeLength() * 8;
} else if (columnDescriptor.getType() == PrimitiveTypeName.INT96
&& (valueVec instanceof TimeStampMilliVector || valueVec instanceof NullableTimeStampMilliVector)) {
// if int 96 column is being read as a Timestamp, this truncates the time format used by Impala
// dataTypeLengthInBits is only ever used when computing offsets into the destination vector, so it
// needs to be set to the bit width of the resulting Arrow type, usually this matches the input length
dataTypeLengthInBits = 64;
} else {
dataTypeLengthInBits = DeprecatedParquetVectorizedReader.getTypeLengthInBits(columnDescriptor.getType());
}
}
}
开发者ID:dremio,项目名称:dremio-oss,代码行数:27,代码来源:ColumnReader.java
示例2: serialize
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Override
public void serialize(ColumnMetadata_v2 value, JsonGenerator jgen, SerializerProvider provider)
throws IOException, JsonProcessingException {
jgen.writeStartObject();
jgen.writeArrayFieldStart("name");
for (String n : value.name) {
jgen.writeString(n);
}
jgen.writeEndArray();
if (value.mxValue != null) {
Object val;
if (value.primitiveType == PrimitiveTypeName.BINARY && value.mxValue != null) {
val = new String(((Binary) value.mxValue).getBytes());
} else {
val = value.mxValue;
}
jgen.writeObjectField("mxValue", val);
}
if (value.nulls != null) {
jgen.writeObjectField("nulls", value.nulls);
}
jgen.writeEndObject();
}
开发者ID:axbaretto,项目名称:drill,代码行数:24,代码来源:Metadata.java
示例3: testStillUseStatsWithSignedSortOrderIfSingleValue
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private void testStillUseStatsWithSignedSortOrderIfSingleValue(StatsHelper helper) {
ParquetMetadataConverter converter = new ParquetMetadataConverter();
BinaryStatistics stats = new BinaryStatistics();
stats.incrementNumNulls();
stats.updateStats(Binary.fromString("A"));
stats.incrementNumNulls();
stats.updateStats(Binary.fromString("A"));
stats.incrementNumNulls();
PrimitiveType binaryType = Types.required(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named("b");
Statistics convertedStats = converter.fromParquetStatistics(
Version.FULL_VERSION,
ParquetMetadataConverter.toParquetStatistics(stats),
binaryType);
Assert.assertFalse("Stats should not be empty: " + convertedStats, convertedStats.isEmpty());
Assert.assertArrayEquals("min == max: " + convertedStats, convertedStats.getMaxBytes(), convertedStats.getMinBytes());
}
开发者ID:apache,项目名称:parquet-mr,代码行数:19,代码来源:TestParquetMetadataConverter.java
示例4: getStatsBasedOnType
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
/**
* Returns the typed statistics object based on the passed type parameter
* @param type PrimitiveTypeName type of the column
* @return instance of a typed statistics class
* @deprecated Use {@link #createStats(Type)} instead
*/
@Deprecated
public static Statistics getStatsBasedOnType(PrimitiveTypeName type) {
switch (type) {
case INT32:
return new IntStatistics();
case INT64:
return new LongStatistics();
case FLOAT:
return new FloatStatistics();
case DOUBLE:
return new DoubleStatistics();
case BOOLEAN:
return new BooleanStatistics();
case BINARY:
return new BinaryStatistics();
case INT96:
return new BinaryStatistics();
case FIXED_LEN_BYTE_ARRAY:
return new BinaryStatistics();
default:
throw new UnknownColumnTypeException(type);
}
}
开发者ID:apache,项目名称:parquet-mr,代码行数:30,代码来源:Statistics.java
示例5: testConvertToThriftStatistics
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testConvertToThriftStatistics() throws Exception {
long[] longArray = new long[] {39L, 99L, 12L, 1000L, 65L, 542L, 2533461316L, -253346131996L, Long.MAX_VALUE, Long.MIN_VALUE};
LongStatistics parquetMRstats = new LongStatistics();
for (long l: longArray) {
parquetMRstats.updateStats(l);
}
final String createdBy =
"parquet-mr version 1.8.0 (build d4d5a07ec9bd262ca1e93c309f1d7d4a74ebda4c)";
Statistics thriftStats =
org.apache.parquet.format.converter.ParquetMetadataConverter.toParquetStatistics(parquetMRstats);
LongStatistics convertedBackStats =
(LongStatistics) org.apache.parquet.format.converter.ParquetMetadataConverter.fromParquetStatistics(
createdBy, thriftStats, PrimitiveTypeName.INT64);
assertEquals(parquetMRstats.getMax(), convertedBackStats.getMax());
assertEquals(parquetMRstats.getMin(), convertedBackStats.getMin());
assertEquals(parquetMRstats.getNumNulls(), convertedBackStats.getNumNulls());
}
开发者ID:apache,项目名称:parquet-mr,代码行数:21,代码来源:TestParquetFileWriter.java
示例6: testV2OnlyStats
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testV2OnlyStats() {
testV2OnlyStats(Types.optional(PrimitiveTypeName.INT32).as(OriginalType.UINT_8).named(""),
0x7F,
0x80);
testV2OnlyStats(Types.optional(PrimitiveTypeName.INT32).as(OriginalType.UINT_16).named(""),
0x7FFF,
0x8000);
testV2OnlyStats(Types.optional(PrimitiveTypeName.INT32).as(OriginalType.UINT_32).named(""),
0x7FFFFFFF,
0x80000000);
testV2OnlyStats(Types.optional(PrimitiveTypeName.INT64).as(OriginalType.UINT_64).named(""),
0x7FFFFFFFFFFFFFFFL,
0x8000000000000000L);
testV2OnlyStats(Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.DECIMAL).precision(6).named(""),
new BigInteger("-765875"),
new BigInteger("876856"));
testV2OnlyStats(
Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(14).as(OriginalType.DECIMAL).precision(7)
.named(""),
new BigInteger("-6769643"),
new BigInteger("9864675"));
}
开发者ID:apache,项目名称:parquet-mr,代码行数:24,代码来源:TestParquetMetadataConverter.java
示例7: validate
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private void validate(PrimitiveTypeName p) {
Type currentType = types.peek().asGroupType().getType(fields.peek());
int c = fieldValueCount.pop() + 1;
fieldValueCount.push(c);
LOG.debug("validate {} for {}",p ,currentType.getName());
switch (currentType.getRepetition()) {
case OPTIONAL:
case REQUIRED:
if (c > 1) {
throw new InvalidRecordException("repeated value when the type is not repeated in " + currentType);
}
break;
case REPEATED:
break;
default:
throw new InvalidRecordException("unknown repetition " + currentType.getRepetition() + " in " + currentType);
}
if (!currentType.isPrimitive() || currentType.asPrimitiveType().getPrimitiveTypeName() != p) {
throw new InvalidRecordException("expected type " + p + " but got "+ currentType);
}
}
开发者ID:apache,项目名称:parquet-mr,代码行数:22,代码来源:ValidatingRecordConsumer.java
示例8: testEachPrimitiveType
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testEachPrimitiveType() {
MessageTypeBuilder builder = buildMessage();
StringBuilder schema = new StringBuilder();
schema.append("message EachType {\n");
for (PrimitiveTypeName type : PrimitiveTypeName.values()) {
// add a schema entry, e.g., " required int32 int32_;\n"
if (type == FIXED_LEN_BYTE_ARRAY) {
schema.append(" required fixed_len_byte_array(3) fixed_;");
builder.required(FIXED_LEN_BYTE_ARRAY).length(3).named("fixed_");
} else {
schema.append(" required ").append(type)
.append(" ").append(type).append("_;\n");
builder.required(type).named(type.toString() + "_");
}
}
schema.append("}\n");
MessageType expected = builder.named("EachType");
MessageType parsed = parseMessageType(schema.toString());
assertEquals(expected, parsed);
MessageType reparsed = parseMessageType(parsed.toString());
assertEquals(expected, reparsed);
}
开发者ID:apache,项目名称:parquet-mr,代码行数:26,代码来源:TestParquetParser.java
示例9: test
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void test() throws IOException {
MemPageStore memPageStore = new MemPageStore(10);
ColumnDescriptor col = new ColumnDescriptor(path , PrimitiveTypeName.INT64, 2, 2);
LongStatistics stats = new LongStatistics();
PageWriter pageWriter = memPageStore.getPageWriter(col);
pageWriter.writePage(BytesInput.from(new byte[735]), 209, stats, BIT_PACKED, BIT_PACKED, PLAIN);
pageWriter.writePage(BytesInput.from(new byte[743]), 209, stats, BIT_PACKED, BIT_PACKED, PLAIN);
pageWriter.writePage(BytesInput.from(new byte[743]), 209, stats, BIT_PACKED, BIT_PACKED, PLAIN);
pageWriter.writePage(BytesInput.from(new byte[735]), 209, stats, BIT_PACKED, BIT_PACKED, PLAIN);
PageReader pageReader = memPageStore.getPageReader(col);
long totalValueCount = pageReader.getTotalValueCount();
System.out.println(totalValueCount);
int total = 0;
do {
DataPage readPage = pageReader.readPage();
total += readPage.getValueCount();
System.out.println(readPage);
// TODO: assert
} while (total < totalValueCount);
}
开发者ID:apache,项目名称:parquet-mr,代码行数:22,代码来源:TestMemPageStore.java
示例10: getPrimitive
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
public PrimitiveTypeName getPrimitive(Type type) {
switch (type) {
case BYTE_ARRAY: // TODO: rename BINARY and remove this switch
return PrimitiveTypeName.BINARY;
case INT64:
return PrimitiveTypeName.INT64;
case INT32:
return PrimitiveTypeName.INT32;
case BOOLEAN:
return PrimitiveTypeName.BOOLEAN;
case FLOAT:
return PrimitiveTypeName.FLOAT;
case DOUBLE:
return PrimitiveTypeName.DOUBLE;
case INT96:
return PrimitiveTypeName.INT96;
case FIXED_LEN_BYTE_ARRAY:
return PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
default:
throw new RuntimeException("Unknown type " + type);
}
}
开发者ID:apache,项目名称:parquet-mr,代码行数:23,代码来源:ParquetMetadataConverter.java
示例11: testIgnoreStatsWithSignedSortOrder
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testIgnoreStatsWithSignedSortOrder() {
ParquetMetadataConverter converter = new ParquetMetadataConverter();
BinaryStatistics stats = new BinaryStatistics();
stats.incrementNumNulls();
stats.updateStats(Binary.fromString("A"));
stats.incrementNumNulls();
stats.updateStats(Binary.fromString("z"));
stats.incrementNumNulls();
PrimitiveType binaryType = Types.required(PrimitiveTypeName.BINARY)
.as(OriginalType.UTF8).named("b");
Statistics convertedStats = converter.fromParquetStatistics(
Version.FULL_VERSION,
StatsHelper.V1.toParquetStatistics(stats),
binaryType);
Assert.assertTrue("Stats should be empty: " + convertedStats, convertedStats.isEmpty());
}
开发者ID:apache,项目名称:parquet-mr,代码行数:20,代码来源:TestParquetMetadataConverter.java
示例12: testFloatingPointStringIndependentFromLocale
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testFloatingPointStringIndependentFromLocale() {
Statistics<?> floatStats = Statistics.createStats(Types.optional(PrimitiveTypeName.FLOAT).named("test-float"));
floatStats.updateStats(123.456f);
Statistics<?> doubleStats = Statistics.createStats(Types.optional(PrimitiveTypeName.DOUBLE).named("test-double"));
doubleStats.updateStats(12345.6789);
Locale defaultLocale = Locale.getDefault();
try {
// Set the locale to French where the decimal separator would be ',' instead of '.'
Locale.setDefault(Locale.FRENCH);
assertEquals("min: 123.456, max: 123.456, num_nulls: 0", floatStats.toString());
assertEquals("min: 12345.6789, max: 12345.6789, num_nulls: 0", doubleStats.toString());
} finally {
Locale.setDefault(defaultLocale);
}
}
开发者ID:apache,项目名称:parquet-mr,代码行数:18,代码来源:TestStatistics.java
示例13: testPrimitiveTypeConstruction
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testPrimitiveTypeConstruction() {
PrimitiveTypeName[] types = new PrimitiveTypeName[] {
BOOLEAN, INT32, INT64, INT96, FLOAT, DOUBLE, BINARY
};
for (PrimitiveTypeName type : types) {
String name = type.toString() + "_";
for (Type.Repetition repetition : Type.Repetition.values()) {
PrimitiveType expected = new PrimitiveType(repetition, type, name);
PrimitiveType built = Types.primitive(type, repetition).named(name);
Assert.assertEquals(expected, built);
switch (repetition) {
case REQUIRED:
built = Types.required(type).named(name);
break;
case OPTIONAL:
built = Types.optional(type).named(name);
break;
case REPEATED:
built = Types.repeated(type).named(name);
break;
}
Assert.assertEquals(expected, built);
}
}
}
开发者ID:apache,项目名称:parquet-mr,代码行数:27,代码来源:TestTypeBuilders.java
示例14: testDECIMALAnnotationRejectsUnsupportedTypes
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testDECIMALAnnotationRejectsUnsupportedTypes() {
PrimitiveTypeName[] unsupported = new PrimitiveTypeName[]{
BOOLEAN, INT96, DOUBLE, FLOAT
};
for (final PrimitiveTypeName type : unsupported) {
assertThrows("Should reject non-binary type: " + type,
IllegalStateException.class, new Callable<Type>() {
@Override
public Type call() throws Exception {
return Types.required(type)
.as(DECIMAL).precision(9).scale(2)
.named("d");
}
});
}
}
开发者ID:apache,项目名称:parquet-mr,代码行数:18,代码来源:TestTypeBuilders.java
示例15: get
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Deprecated
public static ColumnChunkMetaData get(
ColumnPath path,
PrimitiveTypeName type,
CompressionCodecName codec,
Set<Encoding> encodings,
Statistics statistics,
long firstDataPage,
long dictionaryPageOffset,
long valueCount,
long totalSize,
long totalUncompressedSize) {
return get(
path, type, codec, null, encodings, statistics, firstDataPage, dictionaryPageOffset,
valueCount, totalSize, totalUncompressedSize);
}
开发者ID:apache,项目名称:parquet-mr,代码行数:17,代码来源:ColumnChunkMetaData.java
示例16: checkForPartitionColumn
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
/**
* When reading the very first footer, any column is a potential partition column. So for the first footer, we check
* every column to see if it is single valued, and if so, add it to the list of potential partition columns. For the
* remaining footers, we will not find any new partition columns, but we may discover that what was previously a
* potential partition column now no longer qualifies, so it needs to be removed from the list.
* @return whether column is a potential partition column
*/
private boolean checkForPartitionColumn(ParquetFileMetadata fileMetadata, ColumnMetadata columnMetadata, boolean first, long rowCount) {
SchemaPath schemaPath = SchemaPath.getCompoundPath(columnMetadata.getName());
if (schemaPath.getAsUnescapedPath().equals(UPDATE_COLUMN)) {
return true;
}
final PrimitiveTypeName primitiveType = fileMetadata.getPrimitiveType(columnMetadata.getName());
final OriginalType originalType = fileMetadata.getOriginalType(columnMetadata.getName());
if (first) {
if (hasSingleValue(columnMetadata, rowCount)) {
columnTypeMap.put(schemaPath, getType(primitiveType, originalType));
return true;
} else {
return false;
}
} else {
if (!columnTypeMap.keySet().contains(schemaPath)) {
return false;
} else {
if (!hasSingleValue(columnMetadata, rowCount)) {
columnTypeMap.remove(schemaPath);
return false;
}
if (!getType(primitiveType, originalType).equals(columnTypeMap.get(schemaPath))) {
columnTypeMap.remove(schemaPath);
return false;
}
}
}
return true;
}
开发者ID:dremio,项目名称:dremio-oss,代码行数:39,代码来源:ParquetGroupScanUtils.java
示例17: getPrimitiveType
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private PrimitiveType getPrimitiveType(Field field) {
MajorType majorType = getMajorTypeForField(field);
MinorType minorType = majorType.getMinorType();
String name = field.getName();
PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType);
if (primitiveTypeName == null) {
return null;
}
OriginalType originalType = ParquetTypeHelper.getOriginalTypeForMinorType(minorType);
int length = ParquetTypeHelper.getLengthForMinorType(minorType);
DecimalMetadata decimalMetadata = ParquetTypeHelper.getDecimalMetadataForField(majorType);
return new PrimitiveType(OPTIONAL, primitiveTypeName, length, name, originalType, decimalMetadata, null);
}
开发者ID:dremio,项目名称:dremio-oss,代码行数:14,代码来源:ParquetRecordWriter.java
示例18: ColumnMetadata_v1
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
public ColumnMetadata_v1(SchemaPath name, PrimitiveTypeName primitiveType, OriginalType originalType,
Object max, Object min, Long nulls) {
this.name = name;
this.primitiveType = primitiveType;
this.originalType = originalType;
this.max = max;
this.min = min;
this.nulls = nulls;
}
开发者ID:axbaretto,项目名称:drill,代码行数:10,代码来源:Metadata.java
示例19: getMin
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@JsonProperty(value = "min")
public Object getMin() {
if (primitiveType == PrimitiveTypeName.BINARY && min != null) {
return new String(((Binary) min).getBytes());
}
return min;
}
开发者ID:axbaretto,项目名称:drill,代码行数:8,代码来源:Metadata.java
示例20: getMax
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@JsonProperty(value = "max")
public Object getMax() {
if (primitiveType == PrimitiveTypeName.BINARY && max != null) {
return new String(((Binary) max).getBytes());
}
return max;
}
开发者ID:axbaretto,项目名称:drill,代码行数:8,代码来源:Metadata.java
注:本文中的org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName类示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论