org.apache.hadoop.mapred.Reporter#getCounter ( )源码实例Demo

@Nonnull
private static Booster train(@Nonnull final DMatrix dtrain, @Nonnegative final int round,
        @Nonnull final Map<String, Object> params, @Nullable final Reporter reporter)
        throws NoSuchMethodException, IllegalAccessException, InvocationTargetException,
        InstantiationException, XGBoostError {
    final Counters.Counter iterCounter = (reporter == null) ? null
            : reporter.getCounter("hivemall.XGBoostTrainUDTF$Counter", "iteration");

    final Booster booster = XGBoostUtils.createBooster(dtrain, params);
    for (int iter = 0; iter < round; iter++) {
        reportProgress(reporter);
        setCounterValue(iterCounter, iter + 1);

        booster.update(dtrain, iter);
    }
    return booster;
}
 

private static void oldApiCounter(Reporter reporter, Enum<?> counter, long value) {
    try {
        org.apache.hadoop.mapred.Counters.Counter c = reporter.getCounter(counter);
        if (c != null) {
            c.increment(value);
        }
    } catch (Exception ex) {
        // counter unavailable
    }
}
 

@Nonnull
private static Booster train(@Nonnull final DMatrix dtrain, @Nonnull final DMatrix dtest,
        @Nonnegative final int round, @Nonnegative final int earlyStoppingRounds,
        @Nonnull final Map<String, Object> params, @Nullable final Reporter reporter)
        throws NoSuchMethodException, IllegalAccessException, InvocationTargetException,
        InstantiationException, XGBoostError {
    final Counters.Counter iterCounter = (reporter == null) ? null
            : reporter.getCounter("hivemall.XGBoostTrainUDTF$Counter", "iteration");

    final Booster booster = XGBoostUtils.createBooster(dtrain, params);

    final boolean maximizeEvaluationMetrics =
            OptionUtils.getBoolean(params, "maximize_evaluation_metrics");
    float bestScore = maximizeEvaluationMetrics ? -Float.MAX_VALUE : Float.MAX_VALUE;
    int bestIteration = 0;

    final float[] metricsOut = new float[1];
    for (int iter = 0; iter < round; iter++) {
        reportProgress(reporter);
        setCounterValue(iterCounter, iter + 1);

        booster.update(dtrain, iter);

        String evalInfo =
                booster.evalSet(new DMatrix[] {dtest}, new String[] {"test"}, iter, metricsOut);
        logger.info(evalInfo);

        final float score = metricsOut[0];
        if (maximizeEvaluationMetrics) {
            // Update best score if the current score is better (no update when equal)
            if (score > bestScore) {
                bestScore = score;
                bestIteration = iter;
            }
        } else {
            if (score < bestScore) {
                bestScore = score;
                bestIteration = iter;
            }
        }

        if (shouldEarlyStop(earlyStoppingRounds, iter, bestIteration)) {
            logger.info(
                String.format("early stopping after %d rounds away from the best iteration",
                    earlyStoppingRounds));
            break;
        }
    }

    return booster;
}