类org.apache.hadoop.mapred.OutputCommitter源码实例Demo

下面列出了怎么用org.apache.hadoop.mapred.OutputCommitter的API类实例代码及写法,或者点击链接到github查看源代码。

源代码1 项目: ignite   文件: HadoopV1CleanupTask.java
/** {@inheritDoc} */
@Override public void run(HadoopTaskContext taskCtx) throws IgniteCheckedException {
    HadoopV2TaskContext ctx = (HadoopV2TaskContext)taskCtx;

    JobContext jobCtx = ctx.jobContext();

    try {
        OutputCommitter committer = jobCtx.getJobConf().getOutputCommitter();

        if (abort)
            committer.abortJob(jobCtx, JobStatus.State.FAILED);
        else
            committer.commitJob(jobCtx);
    }
    catch (IOException e) {
        throw new IgniteCheckedException(e);
    }
}
 
源代码2 项目: ignite   文件: HadoopV1SetupTask.java
/** {@inheritDoc} */
@Override public void run(HadoopTaskContext taskCtx) throws IgniteCheckedException {
    HadoopV2TaskContext ctx = (HadoopV2TaskContext)taskCtx;

    try {
        ctx.jobConf().getOutputFormat().checkOutputSpecs(null, ctx.jobConf());

        OutputCommitter committer = ctx.jobConf().getOutputCommitter();

        if (committer != null)
            committer.setupJob(ctx.jobContext());
    }
    catch (IOException e) {
        throw new IgniteCheckedException(e);
    }
}
 
源代码3 项目: Flink-CEPplus   文件: HadoopOutputFormatBase.java
@Override
public void finalizeGlobal(int parallelism) throws IOException {

	try {
		JobContext jobContext = new JobContextImpl(this.jobConf, new JobID());
		OutputCommitter outputCommitter = this.jobConf.getOutputCommitter();

		// finalize HDFS output format
		outputCommitter.commitJob(jobContext);
	} catch (Exception e) {
		throw new RuntimeException(e);
	}
}
 
源代码4 项目: flink   文件: HadoopOutputFormatBase.java
@Override
public void finalizeGlobal(int parallelism) throws IOException {

	try {
		JobContext jobContext = new JobContextImpl(this.jobConf, new JobID());
		OutputCommitter outputCommitter = this.jobConf.getOutputCommitter();

		// finalize HDFS output format
		outputCommitter.commitJob(jobContext);
	} catch (Exception e) {
		throw new RuntimeException(e);
	}
}
 
源代码5 项目: flink   文件: HiveTableOutputFormat.java
private void commitJob(String location) throws IOException {
	jobConf.set(OUTDIR, location);
	JobContext jobContext = new JobContextImpl(this.jobConf, new JobID());
	OutputCommitter outputCommitter = this.jobConf.getOutputCommitter();
	// finalize HDFS output format
	outputCommitter.commitJob(jobContext);
}
 
源代码6 项目: flink   文件: HiveTableOutputFormat.java
HivePartitionWriter(JobConf jobConf, OutputFormat outputFormat, FileSinkOperator.RecordWriter recordWriter,
					OutputCommitter outputCommitter) {
	this.jobConf = jobConf;
	this.outputFormat = outputFormat;
	this.recordWriter = recordWriter;
	this.outputCommitter = outputCommitter;
}
 
源代码7 项目: ignite   文件: HadoopV1OutputCollector.java
/**
 * Commit task.
 *
 * @throws IOException In failed.
 */
public void commit() throws IOException {
    if (writer != null) {
        OutputCommitter outputCommitter = jobConf.getOutputCommitter();

        TaskAttemptContext taskCtx = new TaskAttemptContextImpl(jobConf, attempt);

        if (outputCommitter.needsTaskCommit(taskCtx))
            outputCommitter.commitTask(taskCtx);
    }
}
 
源代码8 项目: flink   文件: HadoopOutputFormatBase.java
@Override
public void finalizeGlobal(int parallelism) throws IOException {

	try {
		JobContext jobContext = new JobContextImpl(this.jobConf, new JobID());
		OutputCommitter outputCommitter = this.jobConf.getOutputCommitter();

		// finalize HDFS output format
		outputCommitter.commitJob(jobContext);
	} catch (Exception e) {
		throw new RuntimeException(e);
	}
}
 
源代码9 项目: Flink-CEPplus   文件: HadoopOutputFormat.java
public HadoopOutputFormat(org.apache.hadoop.mapred.OutputFormat<K, V> mapredOutputFormat, Class<OutputCommitter> outputCommitterClass, JobConf job) {
	this(mapredOutputFormat, job);
	super.getJobConf().setOutputCommitter(outputCommitterClass);
}
 
源代码10 项目: flink   文件: HadoopOutputFormat.java
public HadoopOutputFormat(org.apache.hadoop.mapred.OutputFormat<K, V> mapredOutputFormat, Class<OutputCommitter> outputCommitterClass, JobConf job) {
	this(mapredOutputFormat, job);
	super.getJobConf().setOutputCommitter(outputCommitterClass);
}
 
源代码11 项目: flink   文件: HadoopOutputFormat.java
public HadoopOutputFormat(org.apache.hadoop.mapred.OutputFormat<K, V> mapredOutputFormat, Class<OutputCommitter> outputCommitterClass, JobConf job) {
	this(mapredOutputFormat, job);
	super.getJobConf().setOutputCommitter(outputCommitterClass);
}
 
 类所在包
 类方法
 同包方法