类org.apache.hadoop.mapreduce.lib.db.DBInputFormat源码实例Demo

下面列出了怎么用org.apache.hadoop.mapreduce.lib.db.DBInputFormat的API类实例代码及写法,或者点击链接到github查看源代码。

源代码1 项目: beam   文件: HadoopFormatIOIT.java
private static void setupHadoopConfiguration(PostgresIOTestPipelineOptions options) {
  Configuration conf = new Configuration();
  DBConfiguration.configureDB(
      conf,
      "org.postgresql.Driver",
      DatabaseTestHelper.getPostgresDBUrl(options),
      options.getPostgresUsername(),
      options.getPostgresPassword());

  conf.set(DBConfiguration.INPUT_TABLE_NAME_PROPERTY, tableName);
  conf.setStrings(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY, "id", "name");
  conf.set(DBConfiguration.INPUT_ORDER_BY_PROPERTY, "id ASC");
  conf.setClass(DBConfiguration.INPUT_CLASS_PROPERTY, TestRowDBWritable.class, DBWritable.class);

  conf.setClass("key.class", LongWritable.class, Object.class);
  conf.setClass("value.class", TestRowDBWritable.class, Object.class);
  conf.setClass("mapreduce.job.inputformat.class", DBInputFormat.class, InputFormat.class);

  conf.set(DBConfiguration.OUTPUT_TABLE_NAME_PROPERTY, tableName);
  conf.set(DBConfiguration.OUTPUT_FIELD_COUNT_PROPERTY, "2");
  conf.setStrings(DBConfiguration.OUTPUT_FIELD_NAMES_PROPERTY, "id", "name");

  conf.setClass(HadoopFormatIO.OUTPUT_KEY_CLASS, TestRowDBWritable.class, Object.class);
  conf.setClass(HadoopFormatIO.OUTPUT_VALUE_CLASS, NullWritable.class, Object.class);
  conf.setClass(
      HadoopFormatIO.OUTPUT_FORMAT_CLASS_ATTR, DBOutputFormat.class, OutputFormat.class);
  conf.set(HadoopFormatIO.JOB_ID, String.valueOf(1));

  hadoopConfiguration = new SerializableConfiguration(conf);
}
 
源代码2 项目: beam   文件: HadoopFormatIOReadTest.java
@Test
public void testValidateConfigurationWithDBInputFormat() {
  Configuration conf = new Configuration();
  conf.setClass("key.class", LongWritable.class, Object.class);
  conf.setClass("value.class", Text.class, Object.class);
  conf.setClass("mapreduce.job.inputformat.class", DBInputFormat.class, InputFormat.class);

  thrown.expect(IllegalArgumentException.class);
  HadoopFormatIO.<String, String>read()
      .withConfiguration(new SerializableConfiguration(conf).get())
      .withKeyTranslation(myKeyTranslate)
      .withValueTranslation(myValueTranslate);
}
 
源代码3 项目: hadoop   文件: DBCountPageView.java
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {
  
  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;
  
  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }
  
  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);
      
  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);
  
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}
 
源代码4 项目: big-c   文件: DBCountPageView.java
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {
  
  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;
  
  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }
  
  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);
      
  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);
  
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}
 
 类方法
 同包方法