本文整理汇总了Java中org.apache.pig.data.DataType类的典型用法代码示例。如果您正苦于以下问题:Java DataType类的具体用法?Java DataType怎么用?Java DataType使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
DataType类属于org.apache.pig.data包,在下文中一共展示了DataType类的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Java代码示例。
示例1: getSchema
import org.apache.pig.data.DataType; //导入依赖的package包/类
@Override
public ResourceSchema getSchema(final String s, final Job job) throws IOException {
ResourceSchema.ResourceFieldSchema remoteAddrFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("remoteAddr", DataType.BYTEARRAY));
ResourceSchema.ResourceFieldSchema accessTimeFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("accessTime", DataType.BYTEARRAY));
ResourceSchema.ResourceFieldSchema methodFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("method", DataType.BYTEARRAY));
ResourceSchema.ResourceFieldSchema urlFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("url", DataType.BYTEARRAY));
ResourceSchema.ResourceFieldSchema protocolFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("protocol", DataType.BYTEARRAY));
ResourceSchema.ResourceFieldSchema agentFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("agent", DataType.BYTEARRAY));
ResourceSchema.ResourceFieldSchema referFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("refer", DataType.BYTEARRAY));
ResourceSchema.ResourceFieldSchema statusFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("status", DataType.INTEGER));
ResourceSchema.ResourceFieldSchema lengthFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("length", DataType.INTEGER));
ResourceSchema resourceSchema = new ResourceSchema();
resourceSchema.setFields(new ResourceSchema.ResourceFieldSchema[]{remoteAddrFieldSchema, accessTimeFieldSchema, methodFieldSchema, urlFieldSchema, protocolFieldSchema, agentFieldSchema, referFieldSchema, statusFieldSchema, lengthFieldSchema});
return resourceSchema;
}
开发者ID:mumuhadoop,项目名称:mumu-pig,代码行数:19,代码来源:NginxLogLoader.java
示例2: testIntegerTypeInTuple
import org.apache.pig.data.DataType; //导入依赖的package包/类
@Test
public void testIntegerTypeInTuple() throws IOException {
Query query = new Query();
query.value = "";
Schema fakeSchema = getSchema(makeFieldSchema("a", DataType.INTEGER),
makeFieldSchema("b", DataType.LONG));
Tuple fakeTuple = makeTuple(Integer.valueOf(42), Long.valueOf(84));
sty = getSty(withMockResult(withMockSchema(getServer(), fakeSchema), fakeTuple));
runWithoutOutput(() -> sty.execute(query));
Assert.assertFalse(query.failed());
List<TypedObject> columnOne = query.getResult().getColumn("a").getValues();
List<TypedObject> columnTwo = query.getResult().getColumn("b").getValues();
Assert.assertNotNull(columnOne);
Assert.assertEquals(columnOne.size(), 1);
Assert.assertEquals(columnOne.get(0).data, Long.valueOf(42));
Assert.assertNotNull(columnTwo);
Assert.assertEquals(columnTwo.size(), 1);
Assert.assertEquals(columnTwo.get(0).data, Long.valueOf(84));
}
开发者ID:yahoo,项目名称:validatar,代码行数:21,代码来源:StyTest.java
示例3: testFloatTypeInTuple
import org.apache.pig.data.DataType; //导入依赖的package包/类
@Test
public void testFloatTypeInTuple() throws IOException {
Query query = new Query();
query.value = "";
Schema fakeSchema = getSchema(makeFieldSchema("a", DataType.FLOAT),
makeFieldSchema("b", DataType.DOUBLE));
Tuple fakeTuple = makeTuple(Float.valueOf(2.1f), Double.valueOf(4.2));
sty = getSty(withMockResult(withMockSchema(getServer(), fakeSchema), fakeTuple));
runWithoutOutput(() -> sty.execute(query));
Assert.assertFalse(query.failed());
List<TypedObject> columnOne = query.getResult().getColumn("a").getValues();
List<TypedObject> columnTwo = query.getResult().getColumn("b").getValues();
Assert.assertNotNull(columnOne);
Assert.assertEquals(columnOne.size(), 1);
Assert.assertTrue(Math.abs((Double) columnOne.get(0).data - Double.valueOf(2.1)) < EPSILON);
Assert.assertNotNull(columnTwo);
Assert.assertEquals(columnTwo.size(), 1);
Assert.assertTrue(Math.abs((Double) columnTwo.get(0).data - Double.valueOf(4.2)) < EPSILON);
}
开发者ID:yahoo,项目名称:validatar,代码行数:21,代码来源:StyTest.java
示例4: testBigNumericTypeInTuple
import org.apache.pig.data.DataType; //导入依赖的package包/类
@Test
public void testBigNumericTypeInTuple() throws IOException {
Query query = new Query();
query.value = "";
Schema fakeSchema = getSchema(makeFieldSchema("a", DataType.BIGINTEGER),
makeFieldSchema("b", DataType.BIGDECIMAL));
Tuple fakeTuple = makeTuple(new BigInteger("42"), new BigDecimal("42.1"));
sty = getSty(withMockResult(withMockSchema(getServer(), fakeSchema), fakeTuple));
runWithoutOutput(() -> sty.execute(query));
Assert.assertFalse(query.failed());
List<TypedObject> columnOne = query.getResult().getColumn("a").getValues();
List<TypedObject> columnTwo = query.getResult().getColumn("b").getValues();
Assert.assertNotNull(columnOne);
Assert.assertEquals(columnOne.size(), 1);
Assert.assertEquals(columnOne.get(0).data, new BigDecimal("42"));
Assert.assertNotNull(columnTwo);
Assert.assertEquals(columnTwo.size(), 1);
Assert.assertEquals(columnTwo.get(0).data, new BigDecimal("42.1"));
}
开发者ID:yahoo,项目名称:validatar,代码行数:21,代码来源:StyTest.java
示例5: testStringTypeInTuple
import org.apache.pig.data.DataType; //导入依赖的package包/类
@Test
public void testStringTypeInTuple() throws IOException {
Query query = new Query();
query.value = "";
Schema fakeSchema = getSchema(makeFieldSchema("a", DataType.BYTE),
makeFieldSchema("b", DataType.BYTEARRAY),
makeFieldSchema("c", DataType.CHARARRAY));
Tuple fakeTuple = makeTuple(Byte.valueOf("1"), new DataByteArray("foo".getBytes()), "bar");
sty = getSty(withMockResult(withMockSchema(getServer(), fakeSchema), fakeTuple));
runWithoutOutput(() -> sty.execute(query));
Assert.assertFalse(query.failed());
List<TypedObject> columnOne = query.getResult().getColumn("a").getValues();
List<TypedObject> columnTwo = query.getResult().getColumn("b").getValues();
List<TypedObject> columnThree = query.getResult().getColumn("c").getValues();
Assert.assertNotNull(columnOne);
Assert.assertEquals(columnOne.size(), 1);
Assert.assertEquals(columnOne.get(0).data, "1");
Assert.assertNotNull(columnTwo);
Assert.assertEquals(columnTwo.size(), 1);
Assert.assertEquals(columnTwo.get(0).data, "foo");
Assert.assertNotNull(columnThree);
Assert.assertEquals(columnThree.size(), 1);
Assert.assertEquals(columnThree.get(0).data, "bar");
}
开发者ID:yahoo,项目名称:validatar,代码行数:26,代码来源:StyTest.java
示例6: buildElNinoInputSchema
import org.apache.pig.data.DataType; //导入依赖的package包/类
private Schema buildElNinoInputSchema() throws FrontendException {
// Build Field Schema
List<FieldSchema> fieldSchemas = new ArrayList<FieldSchema>();
fieldSchemas.add(new Schema.FieldSchema("buoy_day_ID", DataType.CHARARRAY));
fieldSchemas.add(new Schema.FieldSchema("buoy" , DataType.CHARARRAY));
fieldSchemas.add(new Schema.FieldSchema("day" , DataType.CHARARRAY));
fieldSchemas.add(new Schema.FieldSchema("latitude" , DataType.DOUBLE ));
fieldSchemas.add(new Schema.FieldSchema("longitude" , DataType.DOUBLE ));
fieldSchemas.add(new Schema.FieldSchema("zon_winds" , DataType.DOUBLE ));
fieldSchemas.add(new Schema.FieldSchema("mer_winds" , DataType.DOUBLE ));
fieldSchemas.add(new Schema.FieldSchema("humidity" , DataType.DOUBLE ));
fieldSchemas.add(new Schema.FieldSchema("airtemp" , DataType.DOUBLE ));
fieldSchemas.add(new Schema.FieldSchema("s_s_temp" , DataType.DOUBLE ));
return new Schema(fieldSchemas);
}
开发者ID:Netflix,项目名称:Surus,代码行数:19,代码来源:ScorePMML_ElNinoTest.java
示例7: buildAuditInputSchema
import org.apache.pig.data.DataType; //导入依赖的package包/类
private Schema buildAuditInputSchema() throws FrontendException {
// Build Field Schema
List<FieldSchema> fieldSchemas = new ArrayList<FieldSchema>();
fieldSchemas.add(new Schema.FieldSchema("id" , DataType.LONG));
fieldSchemas.add(new Schema.FieldSchema("age" , DataType.INTEGER));
fieldSchemas.add(new Schema.FieldSchema("employment" , DataType.CHARARRAY));
fieldSchemas.add(new Schema.FieldSchema("education" , DataType.CHARARRAY));
fieldSchemas.add(new Schema.FieldSchema("marital" , DataType.CHARARRAY));
fieldSchemas.add(new Schema.FieldSchema("occupation" , DataType.CHARARRAY));
fieldSchemas.add(new Schema.FieldSchema("income" , DataType.DOUBLE));
fieldSchemas.add(new Schema.FieldSchema("gender" , DataType.CHARARRAY));
fieldSchemas.add(new Schema.FieldSchema("deductions" , DataType.DOUBLE));
fieldSchemas.add(new Schema.FieldSchema("hours" , DataType.INTEGER));
fieldSchemas.add(new Schema.FieldSchema("ignore_accounts", DataType.CHARARRAY));
fieldSchemas.add(new Schema.FieldSchema("risk_adjustment", DataType.INTEGER));
fieldSchemas.add(new Schema.FieldSchema("target_adjusted", DataType.INTEGER));
return new Schema(fieldSchemas);
}
开发者ID:Netflix,项目名称:Surus,代码行数:22,代码来源:ScorePMML_AuditTest.java
示例8: testOutputSchema
import org.apache.pig.data.DataType; //导入依赖的package包/类
@Test
public void testOutputSchema() throws Exception {
// given
EmptyBagToNull udf = new EmptyBagToNull();
Schema irrelevantSchema = null;
// execute
Schema resultSchema = udf.outputSchema(irrelevantSchema);
// assert
assertNotNull(resultSchema);
assertEquals(1, resultSchema.getFields().size());
assertEquals(DataType.BAG, resultSchema.getField(0).type);
assertEquals(1, resultSchema.getField(0).schema.getFields().size());
assertEquals(DataType.TUPLE, resultSchema.getField(0).schema.getField(0).type);
assertEquals(1, resultSchema.getField(0).schema.getField(0).schema.getFields().size());
assertEquals(DataType.CHARARRAY, resultSchema.getField(0).schema.getField(0).schema.getField(0).type);
}
开发者ID:openaire,项目名称:iis,代码行数:19,代码来源:EmptyBagToNullTest.java
示例9: testOutputSchema
import org.apache.pig.data.DataType; //导入依赖的package包/类
@Test
public void testOutputSchema() throws Exception {
// given
StringBagsDifference udf = new StringBagsDifference();
Schema irrelevantSchema = null;
// execute
Schema resultSchema = udf.outputSchema(irrelevantSchema);
// assert
assertNotNull(resultSchema);
assertEquals(1, resultSchema.getFields().size());
assertEquals(DataType.BAG, resultSchema.getField(0).type);
assertEquals(1, resultSchema.getField(0).schema.getFields().size());
assertEquals(DataType.TUPLE, resultSchema.getField(0).schema.getField(0).type);
assertEquals(1, resultSchema.getField(0).schema.getField(0).schema.getFields().size());
assertEquals(DataType.CHARARRAY, resultSchema.getField(0).schema.getField(0).schema.getField(0).type);
}
开发者ID:openaire,项目名称:iis,代码行数:19,代码来源:StringBagsDifferenceTest.java
示例10: testExecSingleIdField
import org.apache.pig.data.DataType; //导入依赖的package包/类
@Test
public void testExecSingleIdField() throws Exception {
// given
TupleFactory tupleFactory = TupleFactory.getInstance();
Schema schema = new Schema();
schema.add(new FieldSchema(null, DataType.CHARARRAY));
schema.add(new FieldSchema(null, DataType.CHARARRAY));
schema.add(new FieldSchema("id", DataType.CHARARRAY));
idReplacer.setInputSchema(schema);
String idFieldName = "id";
String newId = "updatedId";
String oldId = "oldId";
// execute
Tuple result = idReplacer.exec(tupleFactory.newTuple(Lists.newArrayList(
idFieldName, newId, oldId)));
assertNotNull(result);
assertEquals(1, result.getAll().size());
assertEquals(newId, result.get(0));
}
开发者ID:openaire,项目名称:iis,代码行数:21,代码来源:IdReplacerUDFTest.java
示例11: testOutputSchema
import org.apache.pig.data.DataType; //导入依赖的package包/类
@Test
public void testOutputSchema() throws Exception {
// given
Schema schema = new Schema();
schema.add(new FieldSchema(null, DataType.CHARARRAY));
schema.add(new FieldSchema(null, DataType.CHARARRAY));
FieldSchema idFieldSchema = new FieldSchema("id", DataType.CHARARRAY);
schema.add(idFieldSchema);
FieldSchema dedupFieldSchema = new FieldSchema("dedup", DataType.BOOLEAN);
schema.add(dedupFieldSchema);
// execute
Schema resultSchema = idReplacer.outputSchema(schema);
// assert
assertNotNull(resultSchema);
assertEquals(1, resultSchema.getFields().size());
assertEquals(idFieldSchema, resultSchema.getField(0).schema.getField(0));
assertEquals(dedupFieldSchema, resultSchema.getField(0).schema.getField(1));
}
开发者ID:openaire,项目名称:iis,代码行数:21,代码来源:IdReplacerUDFTest.java
示例12: testOutputSchema
import org.apache.pig.data.DataType; //导入依赖的package包/类
@Test
public void testOutputSchema() throws Exception {
// given
StringBagsMerger udf = new StringBagsMerger();
Schema irrelevantSchema = null;
// execute
Schema resultSchema = udf.outputSchema(irrelevantSchema);
// assert
assertNotNull(resultSchema);
assertEquals(1, resultSchema.getFields().size());
assertEquals(DataType.BAG, resultSchema.getField(0).type);
assertEquals(1, resultSchema.getField(0).schema.getFields().size());
assertEquals(DataType.TUPLE, resultSchema.getField(0).schema.getField(0).type);
assertEquals(1, resultSchema.getField(0).schema.getField(0).schema.getFields().size());
assertEquals(DataType.CHARARRAY, resultSchema.getField(0).schema.getField(0).schema.getField(0).type);
}
开发者ID:openaire,项目名称:iis,代码行数:19,代码来源:StringBagsMergerTest.java
示例13: testOutputSchema
import org.apache.pig.data.DataType; //导入依赖的package包/类
@Test
public void testOutputSchema() throws Exception {
// given
Schema irrelevantSchema = null;
// execute
Schema resultSchema = udf.outputSchema(irrelevantSchema);
// assert
assertNotNull(resultSchema);
assertEquals(1, resultSchema.getFields().size());
assertEquals(DataType.BAG, resultSchema.getField(0).type);
assertEquals(DataType.TUPLE, resultSchema.getField(0).schema.getField(0).type);
assertEquals(DataType.INTEGER, resultSchema.getField(0).schema.getField(0).schema.getField(0).type);
assertEquals(DataType.CHARARRAY, resultSchema.getField(0).schema.getField(0).schema.getField(1).type);
}
开发者ID:openaire,项目名称:iis,代码行数:17,代码来源:StringListToListWithIndexesTest.java
示例14: testOutputSchema
import org.apache.pig.data.DataType; //导入依赖的package包/类
@Test
public void testOutputSchema() throws Exception {
// given
NullToEmptyBag udf = new NullToEmptyBag();
Schema irrelevantSchema = null;
// execute
Schema resultSchema = udf.outputSchema(irrelevantSchema);
// assert
assertNotNull(resultSchema);
assertEquals(1, resultSchema.getFields().size());
assertEquals(DataType.BAG, resultSchema.getField(0).type);
assertEquals(1, resultSchema.getField(0).schema.getFields().size());
assertEquals(DataType.TUPLE, resultSchema.getField(0).schema.getField(0).type);
assertEquals(1, resultSchema.getField(0).schema.getField(0).schema.getFields().size());
assertEquals(DataType.CHARARRAY, resultSchema.getField(0).schema.getField(0).schema.getField(0).type);
}
开发者ID:openaire,项目名称:iis,代码行数:19,代码来源:NullToEmptyBagTest.java
示例15: updateUnion
import org.apache.pig.data.DataType; //导入依赖的package包/类
static void updateUnion(final DataBag bag, final Union union) throws ExecException {
// Bag is not empty. process each innerTuple in the bag
for (final Tuple innerTuple : bag) {
final Object f0 = innerTuple.get(0); // consider only field 0
if (f0 == null) {
continue;
}
final byte type = innerTuple.getType(0);
if (type == DataType.BYTEARRAY) {
final DataByteArray dba = (DataByteArray) f0;
union.update(HllSketch.wrap(Memory.wrap(dba.get())));
} else {
throw new IllegalArgumentException("Field type was not DataType.BYTEARRAY: " + type);
}
}
}
开发者ID:DataSketches,项目名称:sketches-pig,代码行数:17,代码来源:UnionSketch.java
示例16: outputSchema
import org.apache.pig.data.DataType; //导入依赖的package包/类
/**
* The output is a Sketch Result Tuple Schema.
*/
@Override
public Schema outputSchema(final Schema input) {
if (input != null) {
try {
final Schema tupleSchema = new Schema();
tupleSchema.add(new Schema.FieldSchema("Estimate", DataType.DOUBLE));
tupleSchema.add(new Schema.FieldSchema("UpperBound", DataType.DOUBLE));
tupleSchema.add(new Schema.FieldSchema("LowerBound", DataType.DOUBLE));
return new Schema(new Schema.FieldSchema(getSchemaName(this
.getClass().getName().toLowerCase(), input), tupleSchema, DataType.TUPLE));
}
catch (final FrontendException e) {
// fall through
}
}
return null;
}
开发者ID:DataSketches,项目名称:sketches-pig,代码行数:21,代码来源:ErrorBounds.java
示例17: updateUnion
import org.apache.pig.data.DataType; //导入依赖的package包/类
/*************************************************************************************************
* Updates a union from a bag of sketches
*
* @param bag A bag of sketchTuples.
* @param union The union to update
*/
private static void updateUnion(final DataBag bag, final com.yahoo.sketches.theta.Union union) {
// Bag is not empty. process each innerTuple in the bag
for (Tuple innerTuple : bag) {
// validate the inner Tuples
final Object f0 = extractFieldAtIndex(innerTuple, 0);
if (f0 == null) {
continue;
}
final Byte type = extractTypeAtIndex(innerTuple, 0);
if (type == null) {
continue;
}
// add only the first field of the innerTuple to the union
if (type == DataType.BYTEARRAY) {
final DataByteArray dba = (DataByteArray) f0;
if (dba.size() > 0) {
union.update(Memory.wrap(dba.get()));
}
} else {
throw new IllegalArgumentException("Field type was not DataType.BYTEARRAY: " + type);
}
}
}
开发者ID:DataSketches,项目名称:sketches-pig,代码行数:30,代码来源:Union.java
示例18: updateIntersection
import org.apache.pig.data.DataType; //导入依赖的package包/类
/*************************************************************************************************
* Updates an intersection from a bag of sketches
*
* @param bag A bag of sketchTuples.
* @param intersection The intersection to update
* @param seed to check against incoming sketches
*/
private static void updateIntersection(final DataBag bag, final Intersection intersection,
final long seed) {
//Bag is not empty. process each innerTuple in the bag
for (Tuple innerTuple : bag) {
//validate the inner Tuples
final Object f0 = extractFieldAtIndex(innerTuple, 0);
if (f0 == null) {
continue;
}
final Byte type = extractTypeAtIndex(innerTuple, 0);
// add only the first field of the innerTuple to the intersection
if (type == DataType.BYTEARRAY) {
final DataByteArray dba = (DataByteArray) f0;
final Memory srcMem = Memory.wrap(dba.get());
final Sketch sketch = Sketch.wrap(srcMem, seed);
intersection.update(sketch);
}
else {
throw new IllegalArgumentException(
"Field type was not DataType.BYTEARRAY: " + type);
}
}
}
开发者ID:DataSketches,项目名称:sketches-pig,代码行数:31,代码来源:Intersect.java
示例19: outputSchema
import org.apache.pig.data.DataType; //导入依赖的package包/类
/**
* The output consists of two longs, or 128 bits, plus the result of the modulo division if
* specified.
*/
@Override
public Schema outputSchema(final Schema input) {
if (input != null) {
try {
final Schema tupleSchema = new Schema();
tupleSchema.add(new Schema.FieldSchema("Hash0", DataType.LONG));
tupleSchema.add(new Schema.FieldSchema("Hash1", DataType.LONG));
tupleSchema.add(new Schema.FieldSchema("ModuloResult", DataType.INTEGER));
return new Schema(new Schema.FieldSchema(getSchemaName(this
.getClass().getName().toLowerCase(), input), tupleSchema, DataType.TUPLE));
}
catch (final FrontendException e) {
//fall through
}
}
return null;
}
开发者ID:DataSketches,项目名称:sketches-pig,代码行数:22,代码来源:MurmurHash3.java
示例20: outputSchema
import org.apache.pig.data.DataType; //导入依赖的package包/类
@Override
public Schema outputSchema(final Schema input) {
try {
if (input == null || input.size() == 0
|| input.getField(0).type != DataType.BYTEARRAY) {
throw new IllegalArgumentException("Input to GetVarOptSamples must be a DataByteArray: "
+ (input == null ? "null" : input.toString()));
}
final Schema weightedSampleSchema = new Schema();
weightedSampleSchema.add(new Schema.FieldSchema(WEIGHT_ALIAS, DataType.DOUBLE));
weightedSampleSchema.add(new Schema.FieldSchema(RECORD_ALIAS, DataType.TUPLE));
return new Schema(new Schema.FieldSchema(getSchemaName(this
.getClass().getName().toLowerCase(), input), weightedSampleSchema, DataType.BAG));
} catch (final FrontendException e) {
throw new RuntimeException(e);
}
}
开发者ID:DataSketches,项目名称:sketches-pig,代码行数:20,代码来源:GetVarOptSamples.java
注:本文中的org.apache.pig.data.DataType类示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论