本文整理汇总了Java中org.apache.pig.tools.pigstats.InputStats类的典型用法代码示例。如果您正苦于以下问题:Java InputStats类的具体用法?Java InputStats怎么用?Java InputStats使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
InputStats类属于org.apache.pig.tools.pigstats包,在下文中一共展示了InputStats类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Java代码示例。
示例1: addOneInputStats
import org.apache.pig.tools.pigstats.InputStats; //导入依赖的package包/类
private void addOneInputStats(String fileName, int index) {
long records = -1;
Long n = multiInputCounters.get(
MRPigStatsUtil.getMultiInputsCounterName(fileName, index));
if (n != null) {
records = n;
} else {
// the file could be empty
if (!disableCounter) records = 0;
else {
LOG.warn("unable to get input counter for " + fileName);
}
}
InputStats is = new InputStats(fileName, -1, records, (state == JobState.SUCCESS));
is.setConf(conf);
inputs.add(is);
}
开发者ID:sigmoidanalytics,项目名称:spork,代码行数:18,代码来源:MRJobStats.java
示例2: testLongCounterName
import org.apache.pig.tools.pigstats.InputStats; //导入依赖的package包/类
@Test
public void testLongCounterName() throws Exception {
// Pig now restricts the string size of its counter name to less than 64 characters.
PrintWriter w = new PrintWriter(new FileWriter("myinputfile"));
w.println("1\t2\t3");
w.println("5\t3\t4");
w.println("3\t4\t5");
w.println("5\t6\t7");
w.println("3\t7\t8");
w.close();
String longfilename = "longlonglonglonglonglonglonglonglonglonglonglongfilefilefilename";
Util.copyFromLocalToCluster(cluster, "myinputfile", longfilename);
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = load '" + longfilename + "' as (a0:int, a1:int, a2:int);");
w1.println("C = join A by a0, B by a0;");
w1.println("store C into '" + OUTPUT_FILE + "';");
w1.close();
try {
String[] args = { PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener());
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<InputStats> inputs = stats.getInputStats();
assertEquals(2, inputs.size());
for (InputStats instats : inputs) {
assertEquals(5, instats.getNumberRecords());
}
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
}
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:38,代码来源:TestPigRunner.java
示例3: testDuplicateCounterName
import org.apache.pig.tools.pigstats.InputStats; //导入依赖的package包/类
@Test
public void testDuplicateCounterName() throws Exception {
// Pig now restricts the string size of its counter name to less than 64 characters.
PrintWriter w = new PrintWriter(new FileWriter("myinputfile"));
w.println("1\t2\t3");
w.println("5\t3\t4");
w.close();
String samefilename = "tmp/input";
Util.copyFromLocalToCluster(cluster, "myinputfile", samefilename);
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = load '" + samefilename + "' as (a0:int, a1:int, a2:int);");
w1.println("C = join A by a0, B by a0;");
w1.println("store C into '" + OUTPUT_FILE + "';");
w1.close();
try {
String[] args = { PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener());
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<InputStats> inputs = stats.getInputStats();
assertEquals(2, inputs.size());
for (InputStats instats : inputs) {
if (instats.getLocation().endsWith("tmp/input")) {
assertEquals(2, instats.getNumberRecords());
} else {
assertEquals(5, instats.getNumberRecords());
}
}
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
}
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:39,代码来源:TestPigRunner.java
示例4: testEmptyFileCounter
import org.apache.pig.tools.pigstats.InputStats; //导入依赖的package包/类
@Test //PIG-1893
public void testEmptyFileCounter() throws Exception {
PrintWriter w = new PrintWriter(new FileWriter("myinputfile"));
w.close();
Util.copyFromLocalToCluster(cluster, "myinputfile", "1.txt");
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = load '1.txt' as (a0:int, a1:int, a2:int);");
w1.println("C = join A by a0, B by a0;");
w1.println("store C into '" + OUTPUT_FILE + "';");
w1.close();
try {
String[] args = { PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener());
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<InputStats> inputs = stats.getInputStats();
assertEquals(2, inputs.size());
for (InputStats instats : inputs) {
if (instats.getLocation().endsWith("1.txt")) {
assertEquals(0, instats.getNumberRecords());
} else {
assertEquals(5, instats.getNumberRecords());
}
}
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
}
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:37,代码来源:TestPigRunner.java
示例5: testDisablePigCounters
import org.apache.pig.tools.pigstats.InputStats; //导入依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters() throws Exception {
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("C = join A by a0, B by a0;");
w1.println("store C into '" + OUTPUT_FILE + "';");
w1.close();
try {
String[] args = { "-Dpig.disable.counter=true", PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener());
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<InputStats> inputs = stats.getInputStats();
assertEquals(2, inputs.size());
for (InputStats instats : inputs) {
// the multi-input counters are disabled
assertEquals(-1, instats.getNumberRecords());
}
List<OutputStats> outputs = stats.getOutputStats();
assertEquals(1, outputs.size());
OutputStats outstats = outputs.get(0);
assertEquals(9, outstats.getNumberRecords());
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
}
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:33,代码来源:TestPigRunner.java
示例6: testDisablePigCounters2
import org.apache.pig.tools.pigstats.InputStats; //导入依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters2() throws Exception {
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = filter A by a0 > 3;");
w1.println("store A into 'output';");
w1.println("store B into 'tmp/output';");
w1.close();
try {
String[] args = { "-Dpig.disable.counter=true", PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener());
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<OutputStats> outputs = stats.getOutputStats();
assertEquals(2, outputs.size());
for (OutputStats outstats : outputs) {
// the multi-output counters are disabled
assertEquals(-1, outstats.getNumberRecords());
}
List<InputStats> inputs = stats.getInputStats();
assertEquals(1, inputs.size());
InputStats instats = inputs.get(0);
assertEquals(5, instats.getNumberRecords());
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
Util.deleteFile(cluster, "tmp/output");
}
}
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:35,代码来源:TestPigRunner.java
示例7: addInputStatistics
import org.apache.pig.tools.pigstats.InputStats; //导入依赖的package包/类
public void addInputStatistics() {
if (loads == null) {
return;
}
for (FileSpec fs : loads) {
long records = -1;
long hdfsBytesRead = -1;
String filename = fs.getFileName();
if (counters != null) {
Map<String, Long> taskCounter = counters.get(TASK_COUNTER_GROUP);
if (taskCounter != null
&& taskCounter.get(TaskCounter.INPUT_RECORDS_PROCESSED.name()) != null) {
records = taskCounter.get(TaskCounter.INPUT_RECORDS_PROCESSED.name());
if (this.isMapOpts) {
mapInputRecords += records;
} else {
reduceInputRecords += records;
}
}
if (counters.get(FS_COUNTER_GROUP) != null &&
counters.get(FS_COUNTER_GROUP).get(PigStatsUtil.HDFS_BYTES_READ) != null) {
hdfsBytesRead = counters.get(FS_COUNTER_GROUP).get(PigStatsUtil.HDFS_BYTES_READ);
}
}
InputStats is = new InputStats(filename, hdfsBytesRead,
records, (state == JobState.SUCCESS));
is.setConf(conf);
inputs.add(is);
}
}
开发者ID:sigmoidanalytics,项目名称:spork,代码行数:32,代码来源:TezVertexStats.java
示例8: testLongCounterName
import org.apache.pig.tools.pigstats.InputStats; //导入依赖的package包/类
@Test
public void testLongCounterName() throws Exception {
// Pig now restricts the string size of its counter name to less than 64 characters.
PrintWriter w = new PrintWriter(new FileWriter("myinputfile"));
w.println("1\t2\t3");
w.println("5\t3\t4");
w.println("3\t4\t5");
w.println("5\t6\t7");
w.println("3\t7\t8");
w.close();
String longfilename = "longlonglonglonglonglonglonglonglonglonglonglongfilefilefilename";
Util.copyFromLocalToCluster(cluster, "myinputfile", longfilename);
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = load '" + longfilename + "' as (a0:int, a1:int, a2:int);");
w1.println("C = join A by a0, B by a0;");
w1.println("store C into '" + OUTPUT_FILE + "';");
w1.close();
try {
String[] args = { "-x", execType, PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<InputStats> inputs = stats.getInputStats();
assertEquals(2, inputs.size());
for (InputStats instats : inputs) {
assertEquals(5, instats.getNumberRecords());
}
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
}
}
开发者ID:sigmoidanalytics,项目名称:spork,代码行数:38,代码来源:TestPigRunner.java
示例9: testDuplicateCounterName
import org.apache.pig.tools.pigstats.InputStats; //导入依赖的package包/类
@Test
public void testDuplicateCounterName() throws Exception {
// Pig now restricts the string size of its counter name to less than 64 characters.
PrintWriter w = new PrintWriter(new FileWriter("myinputfile"));
w.println("1\t2\t3");
w.println("5\t3\t4");
w.close();
String samefilename = "tmp/input";
Util.copyFromLocalToCluster(cluster, "myinputfile", samefilename);
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = load '" + samefilename + "' as (a0:int, a1:int, a2:int);");
w1.println("C = join A by a0, B by a0;");
w1.println("store C into '" + OUTPUT_FILE + "';");
w1.close();
try {
String[] args = { "-x", execType, PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<InputStats> inputs = stats.getInputStats();
assertEquals(2, inputs.size());
for (InputStats instats : inputs) {
if (instats.getLocation().endsWith("tmp/input")) {
assertEquals(2, instats.getNumberRecords());
} else {
assertEquals(5, instats.getNumberRecords());
}
}
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
}
}
开发者ID:sigmoidanalytics,项目名称:spork,代码行数:39,代码来源:TestPigRunner.java
示例10: testEmptyFileCounter
import org.apache.pig.tools.pigstats.InputStats; //导入依赖的package包/类
@Test //PIG-1893
public void testEmptyFileCounter() throws Exception {
PrintWriter w = new PrintWriter(new FileWriter("myinputfile"));
w.close();
Util.copyFromLocalToCluster(cluster, "myinputfile", "1.txt");
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = load '1.txt' as (a0:int, a1:int, a2:int);");
w1.println("C = join A by a0, B by a0;");
w1.println("store C into '" + OUTPUT_FILE + "';");
w1.close();
try {
String[] args = { "-x", execType, PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<InputStats> inputs = stats.getInputStats();
assertEquals(2, inputs.size());
for (InputStats instats : inputs) {
if (instats.getLocation().endsWith("1.txt")) {
assertEquals(0, instats.getNumberRecords());
} else {
assertEquals(5, instats.getNumberRecords());
}
}
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
}
}
开发者ID:sigmoidanalytics,项目名称:spork,代码行数:37,代码来源:TestPigRunner.java
示例11: testDisablePigCounters
import org.apache.pig.tools.pigstats.InputStats; //导入依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters() throws Exception {
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("C = join A by a0, B by a0;");
w1.println("store C into '" + OUTPUT_FILE + "';");
w1.close();
try {
String[] args = {"-Dpig.disable.counter=true", "-x", execType, PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<InputStats> inputs = stats.getInputStats();
assertEquals(2, inputs.size());
if (execType.equals("tez")) {
assertEquals(5, inputs.get(0).getNumberRecords());
assertEquals(5, inputs.get(1).getNumberRecords());
} else {
for (InputStats instats : inputs) {
// the multi-input counters are disabled
assertEquals(-1, instats.getNumberRecords());
}
}
List<OutputStats> outputs = stats.getOutputStats();
assertEquals(1, outputs.size());
OutputStats outstats = outputs.get(0);
assertEquals(9, outstats.getNumberRecords());
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
}
}
开发者ID:sigmoidanalytics,项目名称:spork,代码行数:38,代码来源:TestPigRunner.java
示例12: testDisablePigCounters2
import org.apache.pig.tools.pigstats.InputStats; //导入依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters2() throws Exception {
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = filter A by a0 > 3;");
w1.println("store A into 'output';");
w1.println("store B into 'tmp/output';");
w1.close();
try {
String[] args = { "-Dpig.disable.counter=true", "-x", execType, PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<OutputStats> outputs = stats.getOutputStats();
assertEquals(2, outputs.size());
if (execType.equals("tez")) {
assertEquals(outputs.get(0).getNumberRecords(), 5);
assertEquals(outputs.get(1).getNumberRecords(), 2);
} else {
for (OutputStats outstats : outputs) {
// the multi-output counters are disabled
assertEquals(-1, outstats.getNumberRecords());
}
}
List<InputStats> inputs = stats.getInputStats();
assertEquals(1, inputs.size());
InputStats instats = inputs.get(0);
assertEquals(5, instats.getNumberRecords());
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
Util.deleteFile(cluster, "tmp/output");
}
}
开发者ID:sigmoidanalytics,项目名称:spork,代码行数:40,代码来源:TestPigRunner.java
示例13: display
import org.apache.pig.tools.pigstats.InputStats; //导入依赖的package包/类
private void display() {
SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT);
StringBuilder sb = new StringBuilder();
sb.append("\n");
sb.append(String.format("%1$20s: %2$-100s%n", "HadoopVersion", getHadoopVersion()));
sb.append(String.format("%1$20s: %2$-100s%n", "PigVersion", getPigVersion()));
sb.append(String.format("%1$20s: %2$-100s%n", "TezVersion", TezExecType.getTezVersion()));
sb.append(String.format("%1$20s: %2$-100s%n", "UserId", userId));
sb.append(String.format("%1$20s: %2$-100s%n", "FileName", getFileName()));
sb.append(String.format("%1$20s: %2$-100s%n", "StartedAt", sdf.format(new Date(startTime))));
sb.append(String.format("%1$20s: %2$-100s%n", "FinishedAt", sdf.format(new Date(endTime))));
sb.append(String.format("%1$20s: %2$-100s%n", "Features", getFeatures()));
sb.append("\n");
if (returnCode == ReturnCode.SUCCESS) {
sb.append("Success!\n");
} else if (returnCode == ReturnCode.PARTIAL_FAILURE) {
sb.append("Some tasks have failed! Stop running all dependent tasks\n");
} else {
sb.append("Failed!\n");
}
sb.append("\n");
// Print diagnostic info in case of failure
if (returnCode == ReturnCode.FAILURE
|| returnCode == ReturnCode.PARTIAL_FAILURE) {
if (errorMessage != null) {
String[] lines = errorMessage.split("\n");
for (int i = 0; i < lines.length; i++) {
String s = lines[i].trim();
if (i == 0 || !StringUtils.isEmpty(s)) {
sb.append(String.format("%1$20s: %2$-100s%n", i == 0 ? "ErrorMessage" : "", s));
}
}
sb.append("\n");
}
}
for (TezDAGStats dagStats : tezDAGStatsMap.values()) {
sb.append(dagStats.getDisplayString());
sb.append("\n");
}
sb.append("Input(s):\n");
for (InputStats is : getInputStats()) {
sb.append(is.getDisplayString().trim()).append("\n");
}
sb.append("\n");
sb.append("Output(s):\n");
for (OutputStats os : getOutputStats()) {
sb.append(os.getDisplayString().trim()).append("\n");
}
LOG.info("Script Statistics:\n" + sb.toString());
}
开发者ID:sigmoidanalytics,项目名称:spork,代码行数:54,代码来源:TezPigScriptStats.java
示例14: PigIoStats
import org.apache.pig.tools.pigstats.InputStats; //导入依赖的package包/类
public PigIoStats(InputStats stats) {
this.bytes = stats.getBytes();
this.records = stats.getNumberRecords();
this.name = stats.getName();
this.location = stats.getLocation();
}
开发者ID:azkaban,项目名称:azkaban-plugins,代码行数:7,代码来源:PigIoStats.java
示例15: PigJobStats
import org.apache.pig.tools.pigstats.InputStats; //导入依赖的package包/类
public PigJobStats(JobStats stats) {
numberMaps = stats.getNumberMaps();
minMapTime = stats.getMinMapTime();
maxMapTime = stats.getMaxMapTime();
avgMapTime = stats.getAvgMapTime();
numberReduces = stats.getNumberReduces();
minReduceTime = stats.getMinReduceTime();
maxReduceTime = stats.getMaxReduceTime();
avgReduceTime = stats.getAvgREduceTime();
bytesWritten = stats.getBytesWritten();
hdfsBytesWritten = stats.getHdfsBytesWritten();
mapInputRecords = stats.getMapInputRecords();
mapOutputRecords = stats.getMapOutputRecords();
reduceInputRecords = stats.getReduceInputRecords();
reduceOutputRecords = stats.getReduceOutputRecords();
proactiveSpillCountObjects = stats.getProactiveSpillCountObjects();
proactiveSpillCountRecs = stats.getProactiveSpillCountRecs();
recordsWritten = stats.getRecordWrittern();
smmSpillCount = stats.getSMMSpillCount();
errorMessage = stats.getErrorMessage();
List<InputStats> inputs = stats.getInputs();
inputStats = new ArrayList<PigIoStats>();
for (InputStats input : inputs) {
inputStats.add(new PigIoStats(input.getName(), input.getLocation(), input
.getBytes(), input.getNumberRecords()));
}
List<OutputStats> outputs = stats.getOutputs();
outputStats = new ArrayList<PigIoStats>();
for (OutputStats output : outputs) {
outputStats.add(new PigIoStats(output.getName(), output.getLocation(),
output.getBytes(), output.getNumberRecords()));
}
}
开发者ID:azkaban,项目名称:azkaban-plugins,代码行数:42,代码来源:PigJobStats.java
注:本文中的org.apache.pig.tools.pigstats.InputStats类示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论