org.apache.hadoop.mapreduce.CounterGroup#iterator ( )源码实例Demo

public String toString() {
    StringBuilder builder = new StringBuilder();
    
    while (source.hasNext()) {
        Entry<String,Counters> nextCntr = source.next();
        builder.append("\n").append(nextCntr.getKey()).append("\n----------------------\n");
        Counters counters = nextCntr.getValue();
        
        for (String groupName : counters.getGroupNames()) {
            
            CounterGroup group = counters.getGroup(groupName);
            Iterator<Counter> cntrItr = group.iterator();
            while (cntrItr.hasNext()) {
                Counter counter = cntrItr.next();
                builder.append(groupName).append("\t").append(counter.getDisplayName()).append("=").append(counter.getValue()).append("\n");
            }
            
        }
    }
    
    return builder.toString();
}
 

private void dumpSummary(Job job, List<String> sourceNames) throws IOException {
  System.out.println("Join Input Matrix.");
  CounterGroup group = job.getCounters().getGroup("DynamicCounter");
  for (String source : sourceNames) {
    System.out.print(String.format("%25s\t", source));
  }
  if (group != null) {
    Iterator<Counter> iterator = group.iterator();
    while (iterator.hasNext()) {
      Counter counter = iterator.next();
      String displayName = counter.getDisplayName();
      String[] split = displayName.replace("[", "").replace("[", "").split(",");
      for (String str : split) {
        if (str.trim().equals("1")) {
          System.out.print(String.format("%25s\t", "1"));
        } else {
          System.out.print(String.format("%25s\t", "-"));
        }
      }
    }
  }
}
 

private void printCounters(StringBuffer buff, Counters totalCounters,
    Counters mapCounters, Counters reduceCounters) {
  // Killed jobs might not have counters
  if (totalCounters == null) {
    return;
  }
  buff.append("\nCounters: \n\n");
  buff.append(String.format("|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s|", 
      "Group Name",
      "Counter name",
      "Map Value",
      "Reduce Value",
      "Total Value"));
  buff.append("\n------------------------------------------"+
      "---------------------------------------------");
  for (String groupName : totalCounters.getGroupNames()) {
       CounterGroup totalGroup = totalCounters.getGroup(groupName);
       CounterGroup mapGroup = mapCounters.getGroup(groupName);
       CounterGroup reduceGroup = reduceCounters.getGroup(groupName);
    
       Format decimal = new DecimalFormat();
       Iterator<org.apache.hadoop.mapreduce.Counter> ctrItr =
         totalGroup.iterator();
       while(ctrItr.hasNext()) {
         org.apache.hadoop.mapreduce.Counter counter = ctrItr.next();
         String name = counter.getName();
         String mapValue = 
           decimal.format(mapGroup.findCounter(name).getValue());
         String reduceValue = 
           decimal.format(reduceGroup.findCounter(name).getValue());
         String totalValue = 
           decimal.format(counter.getValue());

         buff.append(
             String.format("%n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s", 
                 totalGroup.getDisplayName(),
                 counter.getDisplayName(),
                 mapValue, reduceValue, totalValue));
    }
  }
}
 

private void printCounters(StringBuffer buff, Counters totalCounters,
    Counters mapCounters, Counters reduceCounters) {
  // Killed jobs might not have counters
  if (totalCounters == null) {
    return;
  }
  buff.append("\nCounters: \n\n");
  buff.append(String.format("|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s|", 
      "Group Name",
      "Counter name",
      "Map Value",
      "Reduce Value",
      "Total Value"));
  buff.append("\n------------------------------------------"+
      "---------------------------------------------");
  for (String groupName : totalCounters.getGroupNames()) {
       CounterGroup totalGroup = totalCounters.getGroup(groupName);
       CounterGroup mapGroup = mapCounters.getGroup(groupName);
       CounterGroup reduceGroup = reduceCounters.getGroup(groupName);
    
       Format decimal = new DecimalFormat();
       Iterator<org.apache.hadoop.mapreduce.Counter> ctrItr =
         totalGroup.iterator();
       while(ctrItr.hasNext()) {
         org.apache.hadoop.mapreduce.Counter counter = ctrItr.next();
         String name = counter.getName();
         String mapValue = 
           decimal.format(mapGroup.findCounter(name).getValue());
         String reduceValue = 
           decimal.format(reduceGroup.findCounter(name).getValue());
         String totalValue = 
           decimal.format(counter.getValue());

         buff.append(
             String.format("%n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s", 
                 totalGroup.getDisplayName(),
                 counter.getDisplayName(),
                 mapValue, reduceValue, totalValue));
    }
  }
}
 

public static boolean findShortestPath(Configuration conf, Path inputPath,
                                       Path outputPath, String startNode,
                                       String targetNode)
    throws Exception {
  conf.set(TARGET_NODE, targetNode);

  Job job = new Job(conf);
  job.setJarByClass(Main.class);
  job.setMapperClass(Map.class);
  job.setReducerClass(Reduce.class);

  job.setInputFormatClass(KeyValueTextInputFormat.class);

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(Text.class);

  FileInputFormat.setInputPaths(job, inputPath);
  FileOutputFormat.setOutputPath(job, outputPath);

  if (!job.waitForCompletion(true)) {
    throw new Exception("Job failed");
  }

  Counter counter = job.getCounters()
      .findCounter(Reduce.PathCounter.TARGET_NODE_DISTANCE_COMPUTED);

  if (counter != null && counter.getValue() > 0) {
    CounterGroup group = job.getCounters().getGroup(Reduce.PathCounter.PATH.toString());
    Iterator<Counter> iter = group.iterator();
    iter.hasNext();
    String path = iter.next().getName();
    System.out.println("==========================================");
    System.out.println("= Shortest path found, details as follows.");
    System.out.println("= ");
    System.out.println("= Start node:  " + startNode);
    System.out.println("= End node:    " + targetNode);
    System.out.println("= Hops:        " + counter.getValue());
    System.out.println("= Path:        " + path);
    System.out.println("==========================================");
    return true;
  }
  return false;
}