下面列出了怎么用org.apache.hadoop.mapred.TaskAttemptContext的API类实例代码及写法,或者点击链接到github查看源代码。
@Test
public void testCloseWithTaskCommit() throws Exception {
OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
DummyOutputCommitter outputCommitter = mock(DummyOutputCommitter.class);
when(outputCommitter.needsTaskCommit(nullable(TaskAttemptContext.class))).thenReturn(true);
DummyRecordWriter recordWriter = mock(DummyRecordWriter.class);
JobConf jobConf = mock(JobConf.class);
HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
outputFormat.recordWriter = recordWriter;
outputFormat.outputCommitter = outputCommitter;
outputFormat.close();
verify(recordWriter, times(1)).close(nullable(Reporter.class));
verify(outputCommitter, times(1)).commitTask(nullable(TaskAttemptContext.class));
}
@Test
public void testCloseWithoutTaskCommit() throws Exception {
OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
DummyOutputCommitter outputCommitter = mock(DummyOutputCommitter.class);
when(outputCommitter.needsTaskCommit(any(TaskAttemptContext.class))).thenReturn(false);
DummyRecordWriter recordWriter = mock(DummyRecordWriter.class);
JobConf jobConf = mock(JobConf.class);
HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
outputFormat.recordWriter = recordWriter;
outputFormat.outputCommitter = outputCommitter;
outputFormat.close();
verify(recordWriter, times(1)).close(any(Reporter.class));
verify(outputCommitter, times(0)).commitTask(any(TaskAttemptContext.class));
}
@Test
public void testCloseWithTaskCommit() throws Exception {
OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
DummyOutputCommitter outputCommitter = mock(DummyOutputCommitter.class);
when(outputCommitter.needsTaskCommit(nullable(TaskAttemptContext.class))).thenReturn(true);
DummyRecordWriter recordWriter = mock(DummyRecordWriter.class);
JobConf jobConf = mock(JobConf.class);
HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
outputFormat.recordWriter = recordWriter;
outputFormat.outputCommitter = outputCommitter;
outputFormat.close();
verify(recordWriter, times(1)).close(nullable(Reporter.class));
verify(outputCommitter, times(1)).commitTask(nullable(TaskAttemptContext.class));
}
@Test
public void testCloseWithoutTaskCommit() throws Exception {
OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
DummyOutputCommitter outputCommitter = mock(DummyOutputCommitter.class);
when(outputCommitter.needsTaskCommit(any(TaskAttemptContext.class))).thenReturn(false);
DummyRecordWriter recordWriter = mock(DummyRecordWriter.class);
JobConf jobConf = mock(JobConf.class);
HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
outputFormat.recordWriter = recordWriter;
outputFormat.outputCommitter = outputCommitter;
outputFormat.close();
verify(recordWriter, times(1)).close(any(Reporter.class));
verify(outputCommitter, times(0)).commitTask(any(TaskAttemptContext.class));
}
@Test
public void testCloseWithTaskCommit() throws Exception {
OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
DummyOutputCommitter outputCommitter = mock(DummyOutputCommitter.class);
when(outputCommitter.needsTaskCommit(nullable(TaskAttemptContext.class))).thenReturn(true);
DummyRecordWriter recordWriter = mock(DummyRecordWriter.class);
JobConf jobConf = mock(JobConf.class);
HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
outputFormat.recordWriter = recordWriter;
outputFormat.outputCommitter = outputCommitter;
outputFormat.close();
verify(recordWriter, times(1)).close(nullable(Reporter.class));
verify(outputCommitter, times(1)).commitTask(nullable(TaskAttemptContext.class));
}
@Test
public void testCloseWithoutTaskCommit() throws Exception {
OutputFormat<String, Long> dummyOutputFormat = mock(DummyOutputFormat.class);
DummyOutputCommitter outputCommitter = mock(DummyOutputCommitter.class);
when(outputCommitter.needsTaskCommit(any(TaskAttemptContext.class))).thenReturn(false);
DummyRecordWriter recordWriter = mock(DummyRecordWriter.class);
JobConf jobConf = mock(JobConf.class);
HadoopOutputFormat<String, Long> outputFormat = new HadoopOutputFormat<>(dummyOutputFormat, jobConf);
outputFormat.recordWriter = recordWriter;
outputFormat.outputCommitter = outputCommitter;
outputFormat.close();
verify(recordWriter, times(1)).close(any(Reporter.class));
verify(outputCommitter, times(0)).commitTask(any(TaskAttemptContext.class));
}
/**
* Commit task.
*
* @throws IOException In failed.
*/
public void commit() throws IOException {
if (writer != null) {
OutputCommitter outputCommitter = jobConf.getOutputCommitter();
TaskAttemptContext taskCtx = new TaskAttemptContextImpl(jobConf, attempt);
if (outputCommitter.needsTaskCommit(taskCtx))
outputCommitter.commitTask(taskCtx);
}
}
@Override
public boolean needsTaskCommit(TaskAttemptContext taskAttemptContext) throws IOException {
return false;
}
@Override
public boolean needsTaskCommit(TaskAttemptContext taskAttemptContext) throws IOException {
return false;
}
@Override
public void setupTask(TaskAttemptContext taskContext) throws IOException {
writeFile(taskContext.getJobConf(), TASK_SETUP_FILE_NAME);
}
@Override
public boolean needsTaskCommit(TaskAttemptContext taskContext)
throws IOException {
return true;
}
@Override
public void commitTask(TaskAttemptContext taskContext) throws IOException {
writeFile(taskContext.getJobConf(), TASK_COMMIT_FILE_NAME);
}
@Override
public void abortTask(TaskAttemptContext taskContext) throws IOException {
writeFile(taskContext.getJobConf(), TASK_ABORT_FILE_NAME);
}
@Override
public void setupTask(TaskAttemptContext taskContext) throws IOException {
writeFile(taskContext.getJobConf(), TASK_SETUP_FILE_NAME);
}
@Override
public boolean needsTaskCommit(TaskAttemptContext taskContext)
throws IOException {
return true;
}
@Override
public void commitTask(TaskAttemptContext taskContext) throws IOException {
writeFile(taskContext.getJobConf(), TASK_COMMIT_FILE_NAME);
}
@Override
public void abortTask(TaskAttemptContext taskContext) throws IOException {
writeFile(taskContext.getJobConf(), TASK_ABORT_FILE_NAME);
}
@Override
public boolean needsTaskCommit(TaskAttemptContext taskAttemptContext) throws IOException {
return false;
}
@Override
public void setupTask(TaskAttemptContext taskContext) throws IOException {
}
@Override
public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException {
return false;
}
@Override
public void commitTask(TaskAttemptContext taskContext) throws IOException {
}
@Override
public void setupTask(TaskAttemptContext taskContext) throws IOException {
}
@Override
public boolean needsTaskCommit(TaskAttemptContext taskContext) throws IOException {
return false;
}
@Override
public void commitTask(TaskAttemptContext taskContext) throws IOException {
}
public org.apache.hadoop.mapreduce.TaskAttemptContext getTaskAttemptContext() {
return taskAttemptContext;
}
public org.apache.hadoop.mapreduce.TaskAttemptContext getTaskAttemptContext() {
return taskAttemptContext;
}
@Override
public void setupTask(TaskAttemptContext taskAttemptContext) throws IOException {
}
@Override
public void commitTask(TaskAttemptContext taskAttemptContext) throws IOException {
}
@Override
public void abortTask(TaskAttemptContext taskAttemptContext) throws IOException {
}
@Override
public void setupTask(TaskAttemptContext taskAttemptContext) throws IOException {
}