下面列出了org.apache.hadoop.mapred.FileAlreadyExistsException#org.apache.hadoop.mapred.InvalidJobConfException 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
public void checkOutputSpecs(JobContext job
) throws FileAlreadyExistsException, IOException{
// Ensure that the output directory is set and not already there
Path outDir = getOutputPath(job);
if (outDir == null) {
throw new InvalidJobConfException("Output directory not set.");
}
// get delegation token for outDir's file system
TokenCache.obtainTokensForNamenodes(job.getCredentials(),
new Path[] { outDir }, job.getConfiguration());
if (outDir.getFileSystem(job.getConfiguration()).exists(outDir)) {
throw new FileAlreadyExistsException("Output directory " + outDir +
" already exists");
}
}
public void checkOutputSpecs(JobContext job
) throws FileAlreadyExistsException, IOException{
// Ensure that the output directory is set and not already there
Path outDir = getOutputPath(job);
if (outDir == null) {
throw new InvalidJobConfException("Output directory not set.");
}
// get delegation token for outDir's file system
TokenCache.obtainTokensForNamenodes(job.getCredentials(),
new Path[] { outDir }, job.getConfiguration());
if (outDir.getFileSystem(job.getConfiguration()).exists(outDir)) {
throw new FileAlreadyExistsException("Output directory " + outDir +
" already exists");
}
}
/**
* Overridden to avoid throwing an exception if the specified directory
* for export already exists.
*/
@Override
public void checkOutputSpecs(JobContext job) throws FileAlreadyExistsException, IOException {
Path outDir = getOutputPath(job);
if(outDir == null) {
throw new InvalidJobConfException("Output directory not set.");
} else {
TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[]{outDir}, job.getConfiguration());
/*
if(outDir.getFileSystem(job.getConfiguration()).exists(outDir)) {
System.out.println("Output dir already exists, no problem");
throw new FileAlreadyExistsException("Output directory " + outDir + " already exists");
}
*/
}
}
/**
* Overridden to avoid throwing an exception if the specified directory
* for export already exists.
*/
@Override
public void checkOutputSpecs(JobContext job) throws FileAlreadyExistsException, IOException {
Path outDir = getOutputPath(job);
if(outDir == null) {
throw new InvalidJobConfException("Output directory not set.");
} else {
TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[]{outDir}, job.getConfiguration());
/*
if(outDir.getFileSystem(job.getConfiguration()).exists(outDir)) {
System.out.println("Output dir already exists, no problem");
throw new FileAlreadyExistsException("Output directory " + outDir + " already exists");
}
*/
}
}
@Override
public void checkOutputSpecs(FileSystem ignored, JobConf job) throws FileAlreadyExistsException, InvalidJobConfException, IOException {
// Ensure that the output directory is set and not already there
Path outDir = getOutputPath(job);
if (outDir == null && job.getNumReduceTasks() != 0) {
throw new InvalidJobConfException("Output directory not set in JobConf.");
}
if (outDir != null) {
FileSystem fs = outDir.getFileSystem(job);
// normalize the output directory
outDir = fs.makeQualified(outDir);
setOutputPath(job, outDir);
// get delegation token for the outDir's file system
TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[]{outDir}, job);
String jobUuid = job.get("zephyr.job.uuid");
if (jobUuid == null)
throw new InvalidJobConfException("This output format REQUIRES the value zephyr.job.uuid to be specified in the job configuration!");
// // check its existence
// if (fs.exists(outDir)) {
// throw new FileAlreadyExistsException("Output directory " + outDir
// + " already exists");
// }
}
}
@Override
public void checkOutputSpecs(JobContext job) throws IOException {
super.checkOutputSpecs(job);
if (getCompressOutput(job) &&
getOutputCompressionType(job) == CompressionType.RECORD ) {
throw new InvalidJobConfException("SequenceFileAsBinaryOutputFormat "
+ "doesn't support Record Compression" );
}
}
@Override
public void checkOutputSpecs(JobContext job
) throws InvalidJobConfException, IOException {
// Ensure that the output directory is set
Path outDir = getOutputPath(job);
if (outDir == null) {
throw new InvalidJobConfException("Output directory not set in JobConf.");
}
final Configuration jobConf = job.getConfiguration();
// get delegation token for outDir's file system
TokenCache.obtainTokensForNamenodes(job.getCredentials(),
new Path[] { outDir }, jobConf);
final FileSystem fs = outDir.getFileSystem(jobConf);
if (fs.exists(outDir)) {
// existing output dir is considered empty iff its only content is the
// partition file.
//
final FileStatus[] outDirKids = fs.listStatus(outDir);
boolean empty = false;
if (outDirKids != null && outDirKids.length == 1) {
final FileStatus st = outDirKids[0];
final String fname = st.getPath().getName();
empty =
!st.isDirectory() && TeraInputFormat.PARTITION_FILENAME.equals(fname);
}
if (TeraSort.getUseSimplePartitioner(job) || !empty) {
throw new FileAlreadyExistsException("Output directory " + outDir
+ " already exists");
}
}
}
@Override
public void checkOutputSpecs(JobContext job
) throws InvalidJobConfException, IOException {
// Ensure that the output directory is set
Path outDir = getOutputPath(job);
if (outDir == null) {
throw new InvalidJobConfException("Output directory not set in JobConf.");
}
final Configuration jobConf = job.getConfiguration();
// get delegation token for outDir's file system
TokenCache.obtainTokensForNamenodes(job.getCredentials(),
new Path[] { outDir }, jobConf);
final FileSystem fs = outDir.getFileSystem(jobConf);
try {
// existing output dir is considered empty iff its only content is the
// partition file.
//
final FileStatus[] outDirKids = fs.listStatus(outDir);
boolean empty = false;
if (outDirKids != null && outDirKids.length == 1) {
final FileStatus st = outDirKids[0];
final String fname = st.getPath().getName();
empty =
!st.isDirectory() && TeraInputFormat.PARTITION_FILENAME.equals(fname);
}
if (TeraSort.getUseSimplePartitioner(job) || !empty) {
throw new FileAlreadyExistsException("Output directory " + outDir
+ " already exists");
}
} catch (FileNotFoundException ignored) {
}
}
public MapRedGfxdRecordWriter(Configuration conf) throws IOException {
this.tableName = conf.get(OUTPUT_TABLE);
try {
this.batchExecutor = util.new RowCommandBatchExecutor(getDriver(conf),
conf.get(OUTPUT_URL), conf.getInt(OUTPUT_BATCH_SIZE,
OUTPUT_BATCH_SIZE_DEFAULT));
} catch (ClassNotFoundException e) {
logger.error("Gemfirexd client classes are missing from the classpath", e);
throw new InvalidJobConfException(e);
}
}
/**
* Validates correctness and completeness of job's output configuration. Job
* configuration must contain url, table and schema name
*
* @param conf
* job conf
* @throws InvalidJobConfException
*/
protected static void validateConfiguration(Configuration conf)
throws InvalidJobConfException {
// User must configure the output region name.
String url = conf.get(OUTPUT_URL);
if (url == null || url.trim().isEmpty()) {
throw new InvalidJobConfException("Output URL not configured.");
}
String table = conf.get(OUTPUT_TABLE);
if (table == null || table.trim().isEmpty()) {
throw new InvalidJobConfException("Output table name not provided.");
}
}
public GfxdRecordWriter(Configuration conf) throws IOException {
this.tableName = conf.get(OUTPUT_TABLE);
try {
this.batchExecutor = util.new RowCommandBatchExecutor(getDriver(conf),
conf.get(OUTPUT_URL), conf.getInt(OUTPUT_BATCH_SIZE,
OUTPUT_BATCH_SIZE_DEFAULT));
} catch (ClassNotFoundException e) {
logger.error("Gemfirexd client classes are missing from the classpath", e);
throw new InvalidJobConfException(e);
}
}
/**
* Validates correctness and completeness of job's output configuration
*
* @param conf
* @throws InvalidJobConfException
*/
protected void validateConfiguration(Configuration conf)
throws InvalidJobConfException {
// User must configure the output region name.
String region = conf.get(REGION);
if (region == null || region.trim().isEmpty()) {
throw new InvalidJobConfException("Output Region name not provided.");
}
// TODO validate if a client connected to gemfire cluster can be created
}
@Override
public void checkOutputSpecs(JobContext job) throws IOException {
super.checkOutputSpecs(job);
if (getCompressOutput(job) &&
getOutputCompressionType(job) == CompressionType.RECORD ) {
throw new InvalidJobConfException("SequenceFileAsBinaryOutputFormat "
+ "doesn't support Record Compression" );
}
}
@Override
public void checkOutputSpecs(JobContext job
) throws InvalidJobConfException, IOException {
// Ensure that the output directory is set
Path outDir = getOutputPath(job);
if (outDir == null) {
throw new InvalidJobConfException("Output directory not set in JobConf.");
}
final Configuration jobConf = job.getConfiguration();
// get delegation token for outDir's file system
TokenCache.obtainTokensForNamenodes(job.getCredentials(),
new Path[] { outDir }, jobConf);
final FileSystem fs = outDir.getFileSystem(jobConf);
if (fs.exists(outDir)) {
// existing output dir is considered empty iff its only content is the
// partition file.
//
final FileStatus[] outDirKids = fs.listStatus(outDir);
boolean empty = false;
if (outDirKids != null && outDirKids.length == 1) {
final FileStatus st = outDirKids[0];
final String fname = st.getPath().getName();
empty =
!st.isDirectory() && TeraInputFormat.PARTITION_FILENAME.equals(fname);
}
if (TeraSort.getUseSimplePartitioner(job) || !empty) {
throw new FileAlreadyExistsException("Output directory " + outDir
+ " already exists");
}
}
}
public MapRedGfxdRecordWriter(Configuration conf) throws IOException {
this.tableName = conf.get(OUTPUT_TABLE);
try {
this.batchExecutor = util.new RowCommandBatchExecutor(getDriver(conf),
conf.get(OUTPUT_URL), conf.getInt(OUTPUT_BATCH_SIZE,
OUTPUT_BATCH_SIZE_DEFAULT));
} catch (ClassNotFoundException e) {
logger.error("Gemfirexd client classes are missing from the classpath", e);
throw new InvalidJobConfException(e);
}
}
/**
* Validates correctness and completeness of job's output configuration. Job
* configuration must contain url, table and schema name
*
* @param conf
* job conf
* @throws InvalidJobConfException
*/
protected static void validateConfiguration(Configuration conf)
throws InvalidJobConfException {
// User must configure the output region name.
String url = conf.get(OUTPUT_URL);
if (url == null || url.trim().isEmpty()) {
throw new InvalidJobConfException("Output URL not configured.");
}
String table = conf.get(OUTPUT_TABLE);
if (table == null || table.trim().isEmpty()) {
throw new InvalidJobConfException("Output table name not provided.");
}
}
public GfxdRecordWriter(Configuration conf) throws IOException {
this.tableName = conf.get(OUTPUT_TABLE);
try {
this.batchExecutor = util.new RowCommandBatchExecutor(getDriver(conf),
conf.get(OUTPUT_URL), conf.getInt(OUTPUT_BATCH_SIZE,
OUTPUT_BATCH_SIZE_DEFAULT));
} catch (ClassNotFoundException e) {
logger.error("Gemfirexd client classes are missing from the classpath", e);
throw new InvalidJobConfException(e);
}
}
/**
* Validates correctness and completeness of job's output configuration
*
* @param conf
* @throws InvalidJobConfException
*/
protected void validateConfiguration(Configuration conf)
throws InvalidJobConfException {
// User must configure the output region name.
String region = conf.get(REGION);
if (region == null || region.trim().isEmpty()) {
throw new InvalidJobConfException("Output Region name not provided.");
}
// TODO validate if a client connected to gemfire cluster can be created
}
public void checkOutputSpecs(FileSystem fs, JobConf job) throws IOException {
Path out = FileOutputFormat.getOutputPath(job);
if ((out == null) && (job.getNumReduceTasks() != 0)) {
throw new InvalidJobConfException(
"Output directory not set in JobConf.");
}
if (fs == null) {
fs = out.getFileSystem(job);
}
if (fs.exists(new Path(out, CrawlDatum.FETCH_DIR_NAME)))
throw new IOException("Segment already fetched!");
}
@Override
public void checkOutputSpecs(FileSystem ignored, JobConf job)
throws FileAlreadyExistsException, InvalidJobConfException, IOException {
String tableName = job.get(OUTPUT_TABLE);
if (tableName == null) {
throw new IOException("Must specify table name");
}
}
public void checkOutputSpecs(JobContext job
) throws FileAlreadyExistsException, IOException{
// Ensure that the output directory is set and not already there
Path outDir = getOutputPath(job);
if (outDir == null) {
throw new InvalidJobConfException("Output directory not set.");
}
if (outDir.getFileSystem(job.getConfiguration()).exists(outDir)) {
throw new FileAlreadyExistsException("Output directory " + outDir +
" already exists");
}
}
public void checkOutputSpecs(FileSystem fs, JobConf job) throws IOException {
Path out = FileOutputFormat.getOutputPath(job);
if ((out == null) && (job.getNumReduceTasks() != 0)) {
throw new InvalidJobConfException(
"Output directory not set in JobConf.");
}
if (fs == null) {
fs = out.getFileSystem(job);
}
if (fs.exists(new Path(out, CrawlDatum.FETCH_DIR_NAME)))
throw new IOException("Segment already fetched!");
}
@Override
public void checkOutputSpecs(JobContext job
) throws InvalidJobConfException, IOException {
// Ensure that the output directory is set
Path outDir = getOutputPath(job);
if (outDir == null) {
throw new InvalidJobConfException("Output directory not set in JobConf.");
}
// get delegation token for outDir's file system
TokenCache.obtainTokensForNamenodes(job.getCredentials(),
new Path[] { outDir }, job.getConfiguration());
}
public void checkOutputSpecs(JobContext job
) throws FileAlreadyExistsException, IOException{
// Ensure that the output directory is set and not already there
Path outDir = getOutputPath(job);
if (outDir == null) {
throw new InvalidJobConfException("Output directory not set.");
}
if (outDir.getFileSystem(job.getConfiguration()).exists(outDir)) {
throw new FileAlreadyExistsException("Output directory " + outDir +
" already exists");
}
}
public int submitAndMonitorJob() throws IOException {
if (jar_ != null && isLocalHadoop()) {
// getAbs became required when shell and subvm have different working dirs...
File wd = new File(".").getAbsoluteFile();
RunJar.unJar(new File(jar_), wd);
}
// if jobConf_ changes must recreate a JobClient
jc_ = new JobClient(jobConf_);
running_ = null;
try {
running_ = jc_.submitJob(jobConf_);
jobId_ = running_.getID();
if (background_) {
LOG.info("Job is running in background.");
} else if (!jc_.monitorAndPrintJob(jobConf_, running_)) {
LOG.error("Job not successful!");
return 1;
}
LOG.info("Output directory: " + output_);
} catch(FileNotFoundException fe) {
LOG.error("Error launching job , bad input path : " + fe.getMessage());
return 2;
} catch(InvalidJobConfException je) {
LOG.error("Error launching job , Invalid job conf : " + je.getMessage());
return 3;
} catch(FileAlreadyExistsException fae) {
LOG.error("Error launching job , Output path already exists : "
+ fae.getMessage());
return 4;
} catch(IOException ioe) {
LOG.error("Error Launching job : " + ioe.getMessage());
return 5;
} catch (InterruptedException ie) {
LOG.error("Error monitoring job : " + ie.getMessage());
return 6;
} finally {
jc_.close();
}
return 0;
}
public int submitAndMonitorJob() throws IOException {
if (jar_ != null && isLocalHadoop()) {
// getAbs became required when shell and subvm have different working dirs...
File wd = new File(".").getAbsoluteFile();
RunJar.unJar(new File(jar_), wd);
}
// if jobConf_ changes must recreate a JobClient
jc_ = new JobClient(jobConf_);
running_ = null;
try {
running_ = jc_.submitJob(jobConf_);
jobId_ = running_.getID();
if (background_) {
LOG.info("Job is running in background.");
} else if (!jc_.monitorAndPrintJob(jobConf_, running_)) {
LOG.error("Job not successful!");
return 1;
}
LOG.info("Output directory: " + output_);
} catch(FileNotFoundException fe) {
LOG.error("Error launching job , bad input path : " + fe.getMessage());
return 2;
} catch(InvalidJobConfException je) {
LOG.error("Error launching job , Invalid job conf : " + je.getMessage());
return 3;
} catch(FileAlreadyExistsException fae) {
LOG.error("Error launching job , Output path already exists : "
+ fae.getMessage());
return 4;
} catch(IOException ioe) {
LOG.error("Error Launching job : " + ioe.getMessage());
return 5;
} catch (InterruptedException ie) {
LOG.error("Error monitoring job : " + ie.getMessage());
return 6;
} finally {
jc_.close();
}
return 0;
}
public int submitAndMonitorJob() throws IOException {
if (jar_ != null && isLocalHadoop()) {
// getAbs became required when shell and subvm have different working dirs...
File wd = new File(".").getAbsoluteFile();
StreamUtil.unJar(new File(jar_), wd);
}
// if jobConf_ changes must recreate a JobClient
jc_ = new JobClient(jobConf_);
boolean error = true;
running_ = null;
String lastReport = null;
try {
running_ = jc_.submitJob(jobConf_);
jobId_ = running_.getJobID();
LOG.info("getLocalDirs(): " + Arrays.asList(jobConf_.getLocalDirs()));
LOG.info("Running job: " + jobId_);
jobInfo();
while (!running_.isComplete()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
running_ = jc_.getJob(jobId_);
String report = null;
report = " map " + Math.round(running_.mapProgress() * 100) + "% reduce "
+ Math.round(running_.reduceProgress() * 100) + "%";
if (!report.equals(lastReport)) {
LOG.info(report);
lastReport = report;
}
}
if (!running_.isSuccessful()) {
jobInfo();
LOG.error("Job not Successful!");
return 1;
}
LOG.info("Job complete: " + jobId_);
LOG.info("Output: " + output_);
error = false;
} catch(FileNotFoundException fe) {
LOG.error("Error launching job , bad input path : " + fe.getMessage());
return 2;
} catch(InvalidJobConfException je) {
LOG.error("Error launching job , Invalid job conf : " + je.getMessage());
return 3;
} catch(FileAlreadyExistsException fae) {
LOG.error("Error launching job , Output path already exists : "
+ fae.getMessage());
return 4;
} catch(IOException ioe) {
LOG.error("Error Launching job : " + ioe.getMessage());
return 5;
} finally {
if (error && (running_ != null)) {
LOG.info("killJob...");
running_.killJob();
}
jc_.close();
}
return 0;
}
public int submitAndMonitorJob() throws IOException {
if (jar_ != null && isLocalHadoop()) {
// getAbs became required when shell and subvm have different working dirs...
File wd = new File(".").getAbsoluteFile();
StreamUtil.unJar(new File(jar_), wd);
}
// if jobConf_ changes must recreate a JobClient
jc_ = new JobClient(jobConf_);
boolean error = true;
running_ = null;
String lastReport = null;
try {
running_ = jc_.submitJob(jobConf_);
jobId_ = running_.getID();
LOG.info("getLocalDirs(): " + Arrays.asList(jobConf_.getLocalDirs()));
LOG.info("Running job: " + jobId_);
jobInfo();
while (!running_.isComplete()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
running_ = jc_.getJob(jobId_);
String report = null;
report = " map " + Math.round(running_.mapProgress() * 100) + "% reduce "
+ Math.round(running_.reduceProgress() * 100) + "%";
if (!report.equals(lastReport)) {
LOG.info(report);
lastReport = report;
}
}
if (!running_.isSuccessful()) {
jobInfo();
LOG.error("Job not Successful!");
return 1;
}
LOG.info("Job complete: " + jobId_);
LOG.info("Output: " + output_);
error = false;
} catch(FileNotFoundException fe) {
LOG.error("Error launching job , bad input path : " + fe.getMessage());
return 2;
} catch(InvalidJobConfException je) {
LOG.error("Error launching job , Invalid job conf : " + je.getMessage());
return 3;
} catch(FileAlreadyExistsException fae) {
LOG.error("Error launching job , Output path already exists : "
+ fae.getMessage());
return 4;
} catch(IOException ioe) {
LOG.error("Error Launching job : " + ioe.getMessage());
return 5;
} finally {
if (error && (running_ != null)) {
LOG.info("killJob...");
running_.killJob();
}
jc_.close();
}
return 0;
}
/**
* Over-ride the default FileOutputFormat's checkOutputSpecs() to
* allow for the target directory to already exist.
*/
public void checkOutputSpecs( FileSystem ignored, JobConf job )
throws FileAlreadyExistsException, InvalidJobConfException, IOException
{
}
public int submitAndMonitorJob() throws IOException {
if (jar_ != null && isLocalHadoop()) {
// getAbs became required when shell and subvm have different working dirs...
File wd = new File(".").getAbsoluteFile();
StreamUtil.unJar(new File(jar_), wd);
}
// if jobConf_ changes must recreate a JobClient
jc_ = new JobClient(jobConf_);
boolean error = true;
running_ = null;
String lastReport = null;
try {
running_ = jc_.submitJob(jobConf_);
jobId_ = running_.getID();
LOG.info("getLocalDirs(): " + Arrays.asList(jobConf_.getLocalDirs()));
LOG.info("Running job: " + jobId_);
jobInfo();
while (!running_.isComplete()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
running_ = jc_.getJob(jobId_);
String report = null;
report = " map " + Math.round(running_.mapProgress() * 100) + "% reduce "
+ Math.round(running_.reduceProgress() * 100) + "%";
if (!report.equals(lastReport)) {
LOG.info(report);
lastReport = report;
}
}
if (!running_.isSuccessful()) {
jobInfo();
LOG.error("Job not Successful!");
return 1;
}
LOG.info("Job complete: " + jobId_);
LOG.info("Output: " + output_);
error = false;
} catch(FileNotFoundException fe) {
LOG.error("Error launching job , bad input path : " + fe.getMessage());
return 2;
} catch(InvalidJobConfException je) {
LOG.error("Error launching job , Invalid job conf : " + je.getMessage());
return 3;
} catch(FileAlreadyExistsException fae) {
LOG.error("Error launching job , Output path already exists : "
+ fae.getMessage());
return 4;
} catch(IOException ioe) {
LOG.error("Error Launching job : " + ioe.getMessage());
return 5;
} finally {
if (error && (running_ != null)) {
LOG.info("killJob...");
running_.killJob();
}
jc_.close();
}
return 0;
}