public static Increment incrementFromThrift(TIncrement in) throws IOException {
  Increment out = new Increment(in.getRow());
  for (TColumnIncrement column : in.getColumns()) {
    out.addColumn(column.getFamily(), column.getQualifier(), column.getAmount());
  }

  if (in.isSetAttributes()) {
    addAttributes(out,in.getAttributes());
  }

  if (in.isSetDurability()) {
    out.setDurability(durabilityFromThrift(in.getDurability()));
  }

  if(in.getCellVisibility() != null) {
    out.setCellVisibility(new CellVisibility(in.getCellVisibility().getExpression()));
  }

  if (in.isSetReturnResults()) {
    out.setReturnResults(in.isReturnResults());
  }

  return out;
}
 

public static Append appendFromThrift(TAppend append) throws IOException {
  Append out = new Append(append.getRow());
  for (TColumnValue column : append.getColumns()) {
    out.addColumn(column.getFamily(), column.getQualifier(), column.getValue());
  }

  if (append.isSetAttributes()) {
    addAttributes(out, append.getAttributes());
  }

  if (append.isSetDurability()) {
    out.setDurability(durabilityFromThrift(append.getDurability()));
  }

  if(append.getCellVisibility() != null) {
    out.setCellVisibility(new CellVisibility(append.getCellVisibility().getExpression()));
  }

  if (append.isSetReturnResults()) {
    out.setReturnResults(append.isReturnResults());
  }

  return out;
}
 

@Override
protected void map(NullWritable key, NullWritable value, Context context) throws IOException,
    InterruptedException {
  String suffix = "/" + shortTaskId;
  int BLOCK_SIZE = (int) (recordsToWrite / 100);
  for (long i = 0; i < recordsToWrite;) {
    for (long idx = 0; idx < BLOCK_SIZE && i < recordsToWrite; idx++, i++) {
      int expIdx = rand.nextInt(BLOCK_SIZE) % VISIBILITY_EXPS_COUNT;
      String exp = VISIBILITY_EXPS[expIdx];
      byte[] row = Bytes.add(Bytes.toBytes(i), Bytes.toBytes(suffix), Bytes.toBytes(exp));
      Put p = new Put(row);
      p.addColumn(TEST_FAMILY, TEST_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY);
      p.setCellVisibility(new CellVisibility(exp));
      getCounter(expIdx).increment(1);
      mutator.mutate(p);

      if (i % 100 == 0) {
        context.setStatus("Written " + i + "/" + recordsToWrite + " records");
        context.progress();
      }
    }
    // End of block, flush all of them before we start writing anything
    // pointing to these!
    mutator.flush();
  }
}
 

private static int insertData(TableName tableName, String column, double prob)
    throws IOException {
  byte[] k = new byte[3];
  byte[][] famAndQf = CellUtil.parseColumn(Bytes.toBytes(column));

  List<Put> puts = new ArrayList<>(9);
  for (int i = 0; i < 9; i++) {
    Put put = new Put(Bytes.toBytes("row" + i));
    put.setDurability(Durability.SKIP_WAL);
    put.addColumn(famAndQf[0], famAndQf[1], k);
    put.setCellVisibility(new CellVisibility("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!"
        + TOPSECRET));
    puts.add(put);
  }
  try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
    table.put(puts);
  }
  return puts.size();
}
 

private RowMutations rowToMutation(final GeoWaveRow row) {
  final RowMutations mutation = new RowMutations(row.getDataId());
  for (final GeoWaveValue value : row.getFieldValues()) {
    final Put put = new Put(row.getDataId());
    // visibility is in the visibility column so no need to serialize it with the value
    put.addColumn(
        StringUtils.stringToBinary(ByteArrayUtils.shortToString(row.getAdapterId())),
        new byte[0],
        DataIndexUtils.serializeDataIndexValue(value, false));
    if ((value.getVisibility() != null) && (value.getVisibility().length > 0)) {
      put.setCellVisibility(
          new CellVisibility(StringUtils.stringFromBinary(value.getVisibility())));
    }
    try {
      mutation.add(put);
    } catch (final IOException e) {
      LOGGER.error("Error creating HBase row mutation: " + e.getMessage());
    }
  }

  return mutation;
}
 

public static TPut putFromHBase(Put in) {
  TPut out = new TPut();
  out.setRow(in.getRow());
  if (in.getTimestamp() != HConstants.LATEST_TIMESTAMP) {
    out.setTimestamp(in.getTimestamp());
  }
  if (in.getDurability() != Durability.USE_DEFAULT) {
    out.setDurability(durabilityFromHBase(in.getDurability()));
  }
  for (Map.Entry<byte [], List<Cell>> entry : in.getFamilyCellMap().entrySet()) {
    byte[] family = entry.getKey();
    for (Cell cell : entry.getValue()) {
      TColumnValue columnValue = new TColumnValue();
      columnValue.setFamily(family)
          .setQualifier(CellUtil.cloneQualifier(cell))
          .setType(cell.getType().getCode())
          .setTimestamp(cell.getTimestamp())
          .setValue(CellUtil.cloneValue(cell));
      if (cell.getTagsLength() != 0) {
        columnValue.setTags(PrivateCellUtil.cloneTags(cell));
      }
      out.addToColumnValues(columnValue);
    }
  }
  for (Map.Entry<String, byte[]> attribute : in.getAttributesMap().entrySet()) {
    out.putToAttributes(ByteBuffer.wrap(Bytes.toBytes(attribute.getKey())),
        ByteBuffer.wrap(attribute.getValue()));
  }
  try {
    CellVisibility cellVisibility = in.getCellVisibility();
    if (cellVisibility != null) {
      TCellVisibility tCellVisibility = new TCellVisibility();
      tCellVisibility.setExpression(cellVisibility.getExpression());
      out.setCellVisibility(tCellVisibility);
    }
  } catch (DeserializationException e) {
    throw new RuntimeException(e);
  }
  return out;
}
 

public static TAppend appendFromHBase(Append in) throws IOException {
  TAppend out = new TAppend();
  out.setRow(in.getRow());

  if (in.getDurability() != Durability.USE_DEFAULT) {
    out.setDurability(durabilityFromHBase(in.getDurability()));
  }
  for (Map.Entry<byte [], List<Cell>> entry : in.getFamilyCellMap().entrySet()) {
    byte[] family = entry.getKey();
    for (Cell cell : entry.getValue()) {
      TColumnValue columnValue = new TColumnValue();
      columnValue.setFamily(family)
          .setQualifier(CellUtil.cloneQualifier(cell))
          .setType(cell.getType().getCode())
          .setTimestamp(cell.getTimestamp())
          .setValue(CellUtil.cloneValue(cell));
      if (cell.getTagsLength() != 0) {
        columnValue.setTags(PrivateCellUtil.cloneTags(cell));
      }
      out.addToColumns(columnValue);
    }
  }
  for (Map.Entry<String, byte[]> attribute : in.getAttributesMap().entrySet()) {
    out.putToAttributes(ByteBuffer.wrap(Bytes.toBytes(attribute.getKey())),
        ByteBuffer.wrap(attribute.getValue()));
  }
  try {
    CellVisibility cellVisibility = in.getCellVisibility();
    if (cellVisibility != null) {
      TCellVisibility tCellVisibility = new TCellVisibility();
      tCellVisibility.setExpression(cellVisibility.getExpression());
      out.setCellVisibility(tCellVisibility);
    }
  } catch (DeserializationException e) {
    throw new RuntimeException(e);
  }
  out.setReturnResults(in.isReturnResults());
  return out;
}
 

public static TIncrement incrementFromHBase(Increment in) throws IOException {
  TIncrement out = new TIncrement();
  out.setRow(in.getRow());

  if (in.getDurability() != Durability.USE_DEFAULT) {
    out.setDurability(durabilityFromHBase(in.getDurability()));
  }
  for (Map.Entry<byte [], List<Cell>> entry : in.getFamilyCellMap().entrySet()) {
    byte[] family = entry.getKey();
    for (Cell cell : entry.getValue()) {
      TColumnIncrement columnValue = new TColumnIncrement();
      columnValue.setFamily(family).setQualifier(CellUtil.cloneQualifier(cell));
      columnValue.setAmount(
          Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
      out.addToColumns(columnValue);
    }
  }
  for (Map.Entry<String, byte[]> attribute : in.getAttributesMap().entrySet()) {
    out.putToAttributes(ByteBuffer.wrap(Bytes.toBytes(attribute.getKey())),
        ByteBuffer.wrap(attribute.getValue()));
  }
  try {
    CellVisibility cellVisibility = in.getCellVisibility();
    if (cellVisibility != null) {
      TCellVisibility tCellVisibility = new TCellVisibility();
      tCellVisibility.setExpression(cellVisibility.getExpression());
      out.setCellVisibility(tCellVisibility);
    }
  } catch (DeserializationException e) {
    throw new RuntimeException(e);
  }
  out.setReturnResults(in.isReturnResults());
  return out;
}
 

@Override
protected void persist(org.apache.hadoop.mapreduce.Mapper.Context output, long count,
    byte[][] prev, byte[][] current, byte[] id) throws IOException {
  String visibilityExps = "";
  String[] split = labels.split(COMMA);
  for (int i = 0; i < current.length; i++) {
    for (int j = 0; j < DEFAULT_TABLES_COUNT; j++) {
      Put put = new Put(current[i]);
      byte[] value = prev == null ? NO_KEY : prev[i];
      put.addColumn(FAMILY_NAME, COLUMN_PREV, value);

      if (count >= 0) {
        put.addColumn(FAMILY_NAME, COLUMN_COUNT, Bytes.toBytes(count + i));
      }
      if (id != null) {
        put.addColumn(FAMILY_NAME, COLUMN_CLIENT, id);
      }
      visibilityExps = split[j * 2] + OR + split[(j * 2) + 1];
      put.setCellVisibility(new CellVisibility(visibilityExps));
      tables[j].mutate(put);
      try {
        Thread.sleep(1);
      } catch (InterruptedException e) {
        throw new IOException();
      }
    }
    if (i % 1000 == 0) {
      // Tickle progress every so often else maprunner will think us hung
      output.progress();
    }
  }
}
 

/**
 * Convert a protocol buffer CellVisibility bytes to a client CellVisibility
 *
 * @param protoBytes
 * @return the converted client CellVisibility
 * @throws DeserializationException
 */
public static CellVisibility toCellVisibility(byte[] protoBytes) throws DeserializationException {
  if (protoBytes == null) return null;
  ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
  ClientProtos.CellVisibility proto = null;
  try {
    ProtobufUtil.mergeFrom(builder, protoBytes);
    proto = builder.build();
  } catch (IOException e) {
    throw new DeserializationException(e);
  }
  return toCellVisibility(proto);
}
 

/**
 * Convert a protocol buffer CellVisibility bytes to a client CellVisibility
 *
 * @param protoBytes
 * @return the converted client CellVisibility
 * @throws DeserializationException
 */
private static CellVisibility toCellVisibility(byte[] protoBytes) throws DeserializationException {
  if (protoBytes == null) return null;
  ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
  ClientProtos.CellVisibility proto = null;
  try {
    ProtobufUtil.mergeFrom(builder, protoBytes);
    proto = builder.build();
  } catch (IOException e) {
    throw new DeserializationException(e);
  }
  return toCellVisibility(proto);
}
 

@Override
public void write(final GeoWaveMetadata metadata) {

  // we use a hashset of row IDs so that we can retain multiple versions
  // (otherwise timestamps will be applied on the server side in
  // batches and if the same row exists within a batch we will not
  // retain multiple versions)
  final Put put = new Put(metadata.getPrimaryId());

  final byte[] secondaryBytes =
      metadata.getSecondaryId() != null ? metadata.getSecondaryId() : new byte[0];

  put.addColumn(metadataTypeBytes, secondaryBytes, metadata.getValue());

  if ((metadata.getVisibility() != null) && (metadata.getVisibility().length > 0)) {
    put.setCellVisibility(
        new CellVisibility(StringUtils.stringFromBinary(metadata.getVisibility())));
  }

  try {
    synchronized (duplicateRowTracker) {
      final ByteArray primaryId = new ByteArray(metadata.getPrimaryId());
      if (!duplicateRowTracker.add(primaryId)) {
        writer.flush();
        duplicateRowTracker.clear();
        duplicateRowTracker.add(primaryId);
      }
    }
    writer.mutate(put);
  } catch (final IOException e) {
    LOGGER.error("Unable to write metadata", e);
  }
}
 

@Override
public void deleteCells(String tableName, List<DeleteRequest> deletes) throws IOException {
    List<Delete> deleteRequests = new ArrayList<>();
    for (int index = 0; index < deletes.size(); index++) {
        DeleteRequest req = deletes.get(index);
        Delete delete = new Delete(req.getRowId())
            .addColumn(req.getColumnFamily(), req.getColumnQualifier());
        if (!StringUtils.isEmpty(req.getVisibilityLabel())) {
            delete.setCellVisibility(new CellVisibility(req.getVisibilityLabel()));
        }
        deleteRequests.add(delete);
    }
    batchDelete(tableName, deleteRequests);
}
 

@Override
public void deleteCells(String tableName, List<DeleteRequest> deletes) throws IOException {
    List<Delete> deleteRequests = new ArrayList<>();
    for (int index = 0; index < deletes.size(); index++) {
        DeleteRequest req = deletes.get(index);
        Delete delete = new Delete(req.getRowId())
            .addColumn(req.getColumnFamily(), req.getColumnQualifier());
        if (!StringUtils.isEmpty(req.getVisibilityLabel())) {
            delete.setCellVisibility(new CellVisibility(req.getVisibilityLabel()));
        }
        deleteRequests.add(delete);
    }
    batchDelete(tableName, deleteRequests);
}
 

@Override
protected void addPutToKv(Put put, Cell kv) throws IOException {
  String visibilityExps = split[index * 2] + OR + split[(index * 2) + 1];
  put.setCellVisibility(new CellVisibility(visibilityExps));
  super.addPutToKv(put, kv);
}
 

@Override
public Put setCellVisibility(CellVisibility expression) {
  return (Put) super.setCellVisibility(expression);
}
 

@Override
public Delete setCellVisibility(CellVisibility expression) {
  return (Delete) super.setCellVisibility(expression);
}
 

/**
 * Sets the visibility expression associated with cells in this Mutation.
 * @param expression
 */
public Mutation setCellVisibility(CellVisibility expression) {
  this.setAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY,
      toCellVisibility(expression).toByteArray());
  return this;
}
 

/**
 * @return CellVisibility associated with cells in this Mutation.
 * @throws DeserializationException
 */
public CellVisibility getCellVisibility() throws DeserializationException {
  byte[] cellVisibilityBytes = this.getAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY);
  if (cellVisibilityBytes == null) return null;
  return toCellVisibility(cellVisibilityBytes);
}
 

@Override
public Append setCellVisibility(CellVisibility expression) {
  return (Append) super.setCellVisibility(expression);
}
 

@Override
public Increment setCellVisibility(CellVisibility expression) {
  return (Increment) super.setCellVisibility(expression);
}
 

/**
 * Convert a protocol buffer CellVisibility to a client CellVisibility
 *
 * @param proto
 * @return the converted client CellVisibility
 */
public static CellVisibility toCellVisibility(ClientProtos.CellVisibility proto) {
  if (proto == null) return null;
  return new CellVisibility(proto.getExpression());
}
 

/**
 * Create a protocol buffer CellVisibility based on a client CellVisibility.
 *
 * @param cellVisibility
 * @return a protocol buffer CellVisibility
 */
public static ClientProtos.CellVisibility toCellVisibility(CellVisibility cellVisibility) {
  ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
  builder.setExpression(cellVisibility.getExpression());
  return builder.build();
}
 

/**
 * Create a protocol buffer CellVisibility based on a client CellVisibility.
 *
 * @param cellVisibility
 * @return a protocol buffer CellVisibility
 */
static ClientProtos.CellVisibility toCellVisibility(CellVisibility cellVisibility) {
  ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder();
  builder.setExpression(cellVisibility.getExpression());
  return builder.build();
}
 

/**
 * Convert a protocol buffer CellVisibility to a client CellVisibility
 *
 * @param proto
 * @return the converted client CellVisibility
 */
private static CellVisibility toCellVisibility(ClientProtos.CellVisibility proto) {
  if (proto == null) return null;
  return new CellVisibility(proto.getExpression());
}