/** Creates and stores activity reports on GCS, returns a list of files stored. */
private ImmutableList<String> stageActivityReports(
    YearMonth yearMonth,
    String subdir,
    String headerRow,
    ImmutableCollection<Map<TableFieldSchema, Object>> rows)
    throws IOException {
  ImmutableList.Builder<String> manifestBuilder = new ImmutableList.Builder<>();
  // Create a report csv for each tld from query table, and upload to GCS
  for (Map<TableFieldSchema, Object> row : rows) {
    // Get the tld (first cell in each row)
    String tld = row.values().iterator().next().toString();
    if (isNullOrEmpty(tld)) {
      throw new RuntimeException("Found an empty row in the activity report table!");
    }
    ImmutableList<String> rowStrings = ImmutableList.of(constructRow(row.values()));
    // Create and upload the activity report with a single row
    manifestBuilder.add(
        saveReportToGcs(
            tld, yearMonth, subdir, createReport(headerRow, rowStrings), ReportType.ACTIVITY));
  }
  return manifestBuilder.build();
}
 

/** Stores a report on GCS, returning the name of the file stored. */
private String saveReportToGcs(
    String tld, YearMonth yearMonth, String subdir, String reportCsv, ReportType reportType)
    throws IOException {
  // Upload resulting CSV file to GCS
  byte[] reportBytes = reportCsv.getBytes(UTF_8);
  String reportFilename =
      String.format(
          "%s-%s-%s.csv",
          tld,
          Ascii.toLowerCase(reportType.toString()),
          DateTimeFormat.forPattern("yyyyMM").print(yearMonth));
  String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
  final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
  gcsUtils.createFromBytes(gcsFilename, reportBytes);
  logger.atInfo().log("Wrote %d bytes to file location %s", reportBytes.length, gcsFilename);
  return reportFilename;
}
 

@Inject
GenerateInvoicesAction(
    @Config("projectId") String projectId,
    @Config("apacheBeamBucketUrl") String beamBucketUrl,
    @Config("invoiceTemplateUrl") String invoiceTemplateUrl,
    @Config("defaultJobZone") String jobZone,
    @Parameter(PARAM_SHOULD_PUBLISH) boolean shouldPublish,
    YearMonth yearMonth,
    Dataflow dataflow,
    Response response,
    BillingEmailUtils emailUtils) {
  this.projectId = projectId;
  this.beamBucketUrl = beamBucketUrl;
  this.invoiceTemplateUrl = invoiceTemplateUrl;
  this.jobZone = jobZone;
  this.shouldPublish = shouldPublish;
  this.yearMonth = yearMonth;
  this.dataflow = dataflow;
  this.response = response;
  this.emailUtils = emailUtils;
}
 

@Inject
BillingEmailUtils(
    SendEmailService emailService,
    YearMonth yearMonth,
    @Config("gSuiteOutgoingEmailAddress") InternetAddress outgoingEmailAddress,
    @Config("alertRecipientEmailAddress") InternetAddress alertRecipientAddress,
    @Config("invoiceEmailRecipients") ImmutableList<InternetAddress> invoiceEmailRecipients,
    @Config("billingBucket") String billingBucket,
    @Config("invoiceFilePrefix") String invoiceFilePrefix,
    @InvoiceDirectoryPrefix String invoiceDirectoryPrefix,
    GcsUtils gcsUtils) {
  this.emailService = emailService;
  this.yearMonth = yearMonth;
  this.outgoingEmailAddress = outgoingEmailAddress;
  this.alertRecipientAddress = alertRecipientAddress;
  this.invoiceEmailRecipients = invoiceEmailRecipients;
  this.billingBucket = billingBucket;
  this.invoiceFilePrefix = invoiceFilePrefix;
  this.invoiceDirectoryPrefix = invoiceDirectoryPrefix;
  this.gcsUtils = gcsUtils;
}
 

@Before
public void setUp() throws IOException {
  Dataflow dataflow = mock(Dataflow.class);
  Projects projects = mock(Projects.class);
  Jobs jobs = mock(Jobs.class);
  get = mock(Get.class);
  when(dataflow.projects()).thenReturn(projects);
  when(projects.jobs()).thenReturn(jobs);
  when(jobs.get("test-project", "12345")).thenReturn(get);
  expectedJob = new Job();
  when(get.execute()).thenReturn(expectedJob);
  emailUtils = mock(BillingEmailUtils.class);
  response = new FakeResponse();
  uploadAction =
      new PublishInvoicesAction(
          "test-project", "12345", emailUtils, dataflow, response, new YearMonth(2017, 10));
}
 

@Test
public void testCaughtIOException() throws IOException {
  when(launch.execute()).thenThrow(new IOException("expected"));
  action =
      new GenerateInvoicesAction(
          "test-project",
          "gs://test-project-beam",
          "gs://test-project-beam/templates/invoicing",
          "us-east1-c",
          true,
          new YearMonth(2017, 10),
          dataflow,
          response,
          emailUtils);
  action.run();
  assertThat(response.getStatus()).isEqualTo(500);
  assertThat(response.getPayload()).isEqualTo("Template launch failed: expected");
  verify(emailUtils).sendAlertEmail("Template Launch failed due to expected");
}
 

@Before
public void setUp() throws Exception {
  emailService = mock(SendEmailService.class);
  gcsUtils = mock(GcsUtils.class);
  when(gcsUtils.openInputStream(new GcsFilename("test-bucket", "results/REG-INV-2017-10.csv")))
      .thenReturn(
          new ByteArrayInputStream("test,data\nhello,world".getBytes(StandardCharsets.UTF_8)));
  contentCaptor = ArgumentCaptor.forClass(EmailMessage.class);

  emailUtils =
      new BillingEmailUtils(
          emailService,
          new YearMonth(2017, 10),
          new InternetAddress("[email protected]"),
          new InternetAddress("[email protected]"),
          ImmutableList.of(
              new InternetAddress("[email protected]"), new InternetAddress("[email protected]")),
          "test-bucket",
          "REG-INV",
          "results/",
          gcsUtils);
}
 

@Test
public void testCreate() throws IOException, OmiseException {
    Request<Token> request = new Token.CreateRequestBuilder()
            .card(new Card.Create()
                    .name("JOHN DOE")
                    .number("4242424242424242")
                    .expiration(YearMonth.now().withPeriodAdded(Period.years(1), 1))
                    .securityCode("123")
                    .city("Bangkok")
                    .postalCode("10240"))
            .build();

    Token token = getTestRequester().sendRequest(request);

    assertRequested("POST", "/tokens", 200);
    assertVaultRequest();

    assertEquals(TOKEN_ID, token.getId());
    assertFalse(token.isLiveMode());
    assertEquals("card_test_4yq6tuucl9h4erukfl0", token.getCard().getId());
}
 

/** Returns the aggregate query which generates the activity report from the saved view. */
@Override
public String getReportQuery(YearMonth yearMonth) {
  return String.format(
      "#standardSQL\nSELECT * FROM `%s.%s.%s`",
      projectId, ICANN_REPORTING_DATA_SET, getTableName(ACTIVITY_REPORT_AGGREGATION, yearMonth));
}
 

/** Returns the aggregate query which generates the transactions report from the saved view. */
@Override
public String getReportQuery(YearMonth yearMonth) {
  return String.format(
      "#standardSQL\nSELECT * FROM `%s.%s.%s`",
      projectId,
      ICANN_REPORTING_DATA_SET,
      getTableName(TRANSACTIONS_REPORT_AGGREGATION, yearMonth));
}
 

private String makeUrl(String filename) {
  // Filename is in the format tld-reportType-yearMonth.csv
  String tld = getTld(filename);
  // Remove the tld- prefix and csv suffix
  String remainder = filename.substring(tld.length() + 1, filename.length() - 4);
  List<String> elements = Splitter.on('-').splitToList(remainder);
  ReportType reportType = ReportType.valueOf(Ascii.toUpperCase(elements.get(0)));
  // Re-add hyphen between year and month, because ICANN is inconsistent between filename and URL
  String yearMonth =
      YearMonth.parse(elements.get(1), DateTimeFormat.forPattern("yyyyMM")).toString("yyyy-MM");
  return String.format("%s/%s/%s", getUrlPrefix(reportType), tld, yearMonth);
}
 

@Override
public String createQuery(YearMonth yearMonth) {
  return SqlTemplate.create(
          ResourceUtils.readResourceUtf8(
              Resources.getResource(this.getClass(), "sql/" + "dns_counts.sql")))
      .build();
}
 

/**
 * Creates and stores reports of a given type on GCS.
 *
 * <p>This is factored out to facilitate choosing which reports to upload,
 */
ImmutableList<String> stageReports(YearMonth yearMonth, String subdir, ReportType reportType)
    throws Exception {
  QueryBuilder queryBuilder =
      (reportType == ReportType.ACTIVITY) ? activityQueryBuilder : transactionsQueryBuilder;

  if (reportType == ReportType.ACTIVITY) {
    // Prepare for the DNS count query, which may have special needs.
    activityQueryBuilder.prepareForQuery(yearMonth);
  }

  ImmutableMap<String, String> viewQueryMap = queryBuilder.getViewQueryMap(yearMonth);
  // Generate intermediary views
  for (Entry<String, String> entry : viewQueryMap.entrySet()) {
    createIntermediaryTableView(entry.getKey(), entry.getValue(), reportType);
  }

  // Get an in-memory table of the aggregate query's result
  ImmutableTable<Integer, TableFieldSchema, Object> reportTable =
      bigquery.queryToLocalTableSync(queryBuilder.getReportQuery(yearMonth));

  // Get report headers from the table schema and convert into CSV format
  String headerRow = constructRow(getHeaders(reportTable.columnKeySet()));

  return (reportType == ReportType.ACTIVITY)
      ? stageActivityReports(yearMonth, subdir, headerRow, reportTable.rowMap().values())
      : stageTransactionsReports(yearMonth, subdir, headerRow, reportTable.rowMap().values());
}
 

/** Extracts an optional YearMonth in yyyy-MM format from the request. */
@Provides
@Parameter(PARAM_YEAR_MONTH)
static Optional<YearMonth> provideYearMonthOptional(HttpServletRequest req) {
  Optional<String> optionalYearMonthStr = extractOptionalParameter(req, PARAM_YEAR_MONTH);
  try {
    return optionalYearMonthStr.map(s -> YearMonth.parse(s, ISODateTimeFormat.yearMonth()));
  } catch (IllegalArgumentException e) {
    throw new BadRequestException(
        String.format(
            "yearMonth must be in yyyy-MM format, got %s instead",
            optionalYearMonthStr.orElse("UNSPECIFIED YEARMONTH")));
  }
}
 

/**
 * Provides the yearMonth in yyyy-MM format, if not specified in the request, defaults to one
 * month prior to run time.
 */
@Provides
static YearMonth provideYearMonth(
    @Parameter(PARAM_YEAR_MONTH) Optional<YearMonth> yearMonthOptional,
    @Parameter(PARAM_DATE) LocalDate date) {
  return yearMonthOptional.orElseGet(() -> new YearMonth(date.minusMonths(1)));
}
 

@Inject
PublishInvoicesAction(
    @Config("projectId") String projectId,
    @Parameter(ReportingModule.PARAM_JOB_ID) String jobId,
    BillingEmailUtils emailUtils,
    Dataflow dataflow,
    Response response,
    YearMonth yearMonth) {
  this.projectId = projectId;
  this.jobId = jobId;
  this.emailUtils = emailUtils;
  this.dataflow = dataflow;
  this.response = response;
  this.yearMonth = yearMonth;
}
 

@Test
public void testInvalidSubdir_throwsException() {
  action.overrideSubdir = Optional.of("/whoops");
  BadRequestException thrown =
      assertThrows(
          BadRequestException.class,
          () ->
              action.getSubdir(new YearMonth(2017, 6)));
  assertThat(thrown)
      .hasMessageThat()
      .contains("subdir must not start or end with a \"/\", got /whoops instead.");
}
 

@Test
public void testGivenYearMonth_returnsThatMonth() {
  assertThat(
          ReportingModule.provideYearMonth(
              Optional.of(new YearMonth(2017, 5)), new LocalDate(2017, 7, 6)))
      .isEqualTo(new YearMonth(2017, 5));
}
 

@Test
public void testLaunchTemplateJob_withPublish() throws Exception {
  action =
      new GenerateInvoicesAction(
          "test-project",
          "gs://test-project-beam",
          "gs://test-project-beam/templates/invoicing",
          "us-east1-c",
          true,
          new YearMonth(2017, 10),
          dataflow,
          response,
          emailUtils);
  action.run();
  LaunchTemplateParameters expectedParams =
      new LaunchTemplateParameters()
          .setJobName("invoicing-2017-10")
          .setEnvironment(
              new RuntimeEnvironment()
                  .setZone("us-east1-c")
                  .setTempLocation("gs://test-project-beam/temporary"))
          .setParameters(ImmutableMap.of("yearMonth", "2017-10"));
  verify(templates).launch("test-project", expectedParams);
  verify(launch).setGcsPath("gs://test-project-beam/templates/invoicing");
  assertThat(response.getStatus()).isEqualTo(200);
  assertThat(response.getPayload()).isEqualTo("Launched dataflow template.");

  TaskMatcher matcher =
      new TaskMatcher()
          .url("/_dr/task/publishInvoices")
          .method("POST")
          .param("jobId", "12345")
          .param("yearMonth", "2017-10");
  assertTasksEnqueued("beam-reporting", matcher);
}
 

@Test
public void testLaunchTemplateJob_withoutPublish() throws Exception {
  action =
      new GenerateInvoicesAction(
          "test-project",
          "gs://test-project-beam",
          "gs://test-project-beam/templates/invoicing",
          "us-east1-c",
          false,
          new YearMonth(2017, 10),
          dataflow,
          response,
          emailUtils);
  action.run();
  LaunchTemplateParameters expectedParams =
      new LaunchTemplateParameters()
          .setJobName("invoicing-2017-10")
          .setEnvironment(
              new RuntimeEnvironment()
                  .setZone("us-east1-c")
                  .setTempLocation("gs://test-project-beam/temporary"))
          .setParameters(ImmutableMap.of("yearMonth", "2017-10"));
  verify(templates).launch("test-project", expectedParams);
  verify(launch).setGcsPath("gs://test-project-beam/templates/invoicing");
  assertThat(response.getStatus()).isEqualTo(200);
  assertThat(response.getPayload()).isEqualTo("Launched dataflow template.");
  assertNoTasksEnqueued();
}
 

@Test
public void testGetExpiration() {
    Card card = new Card();
    card.setExpirationMonth(11);
    card.setExpirationYear(2099);

    YearMonth expiration = card.getExpiration();
    assertEquals(2099, expiration.getYear());
    assertEquals(11, expiration.getMonthOfYear());
}
 

@Test
public void creates_instance_of_YearMonth() {
    YearMonth yearMonth = fixture.create(YearMonth.class);
    assertThat(yearMonth, notNullValue());
    assertThat(yearMonth.getYear(), is(2001));
    assertThat(yearMonth.getMonthOfYear(), is(1));
}
 

@Override
public YearMonth parse(String text, Locale locale) throws ParseException {
	return YearMonth.parse(text);
}
 

@Override
public String print(YearMonth object, Locale locale) {
	return object.toString();
}
 

@Override
public void registerFormatters(FormatterRegistry registry) {
	JodaTimeConverters.registerConverters(registry);

	DateTimeFormatter dateFormatter = getFormatter(Type.DATE);
	DateTimeFormatter timeFormatter = getFormatter(Type.TIME);
	DateTimeFormatter dateTimeFormatter = getFormatter(Type.DATE_TIME);

	addFormatterForFields(registry,
			new ReadablePartialPrinter(dateFormatter),
			new LocalDateParser(dateFormatter),
			LocalDate.class);

	addFormatterForFields(registry,
			new ReadablePartialPrinter(timeFormatter),
			new LocalTimeParser(timeFormatter),
			LocalTime.class);

	addFormatterForFields(registry,
			new ReadablePartialPrinter(dateTimeFormatter),
			new LocalDateTimeParser(dateTimeFormatter),
			LocalDateTime.class);

	addFormatterForFields(registry,
			new ReadableInstantPrinter(dateTimeFormatter),
			new DateTimeParser(dateTimeFormatter),
			ReadableInstant.class);

	// In order to retain backwards compatibility we only register Date/Calendar
	// types when a user defined formatter is specified (see SPR-10105)
	if (this.formatters.containsKey(Type.DATE_TIME)) {
		addFormatterForFields(registry,
				new ReadableInstantPrinter(dateTimeFormatter),
				new DateTimeParser(dateTimeFormatter),
				Date.class, Calendar.class);
	}

	registry.addFormatterForFieldType(Period.class, new PeriodFormatter());
	registry.addFormatterForFieldType(Duration.class, new DurationFormatter());
	registry.addFormatterForFieldType(YearMonth.class, new YearMonthFormatter());
	registry.addFormatterForFieldType(MonthDay.class, new MonthDayFormatter());

	registry.addFormatterForFieldAnnotation(new JodaDateTimeFormatAnnotationFormatterFactory());
}
 

public YearMonth getYearMonth() {
	return yearMonth;
}
 

public void setYearMonth(YearMonth yearMonth) {
	this.yearMonth = yearMonth;
}
 

@Override
public YearMonth parse(String text, Locale locale) throws ParseException {
	return YearMonth.parse(text);
}
 

@Override
public String print(YearMonth object, Locale locale) {
	return object.toString();
}
 

@Override
public void registerFormatters(FormatterRegistry registry) {
	JodaTimeConverters.registerConverters(registry);

	DateTimeFormatter dateFormatter = getFormatter(Type.DATE);
	DateTimeFormatter timeFormatter = getFormatter(Type.TIME);
	DateTimeFormatter dateTimeFormatter = getFormatter(Type.DATE_TIME);

	addFormatterForFields(registry,
			new ReadablePartialPrinter(dateFormatter),
			new LocalDateParser(dateFormatter),
			LocalDate.class);

	addFormatterForFields(registry,
			new ReadablePartialPrinter(timeFormatter),
			new LocalTimeParser(timeFormatter),
			LocalTime.class);

	addFormatterForFields(registry,
			new ReadablePartialPrinter(dateTimeFormatter),
			new LocalDateTimeParser(dateTimeFormatter),
			LocalDateTime.class);

	addFormatterForFields(registry,
			new ReadableInstantPrinter(dateTimeFormatter),
			new DateTimeParser(dateTimeFormatter),
			ReadableInstant.class);

	// In order to retain backwards compatibility we only register Date/Calendar
	// types when a user defined formatter is specified (see SPR-10105)
	if (this.formatters.containsKey(Type.DATE_TIME)) {
		addFormatterForFields(registry,
				new ReadableInstantPrinter(dateTimeFormatter),
				new DateTimeParser(dateTimeFormatter),
				Date.class, Calendar.class);
	}

	registry.addFormatterForFieldType(Period.class, new PeriodFormatter());
	registry.addFormatterForFieldType(Duration.class, new DurationFormatter());
	registry.addFormatterForFieldType(YearMonth.class, new YearMonthFormatter());
	registry.addFormatterForFieldType(MonthDay.class, new MonthDayFormatter());

	registry.addFormatterForFieldAnnotation(new JodaDateTimeFormatAnnotationFormatterFactory());
}