下面列出了org.apache.hadoop.hbase.Cell#getValueLength ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
/**
* Returns a {@link ServerName} from catalog table {@link Result}.
* (Copied from MetaTableAccessor)
* @param r Result to pull from
* @return A ServerName instance or null if necessary fields not found or empty.
*/
@InterfaceAudience.Private // for use by HMaster#getTableRegionRow which is used for testing only
public static ServerName getServerName(final Result r, final int replicaId) {
byte[] serverColumn = getServerColumn(replicaId);
Cell cell = r.getColumnLatestCell(CATALOG_FAMILY, serverColumn);
if (cell == null || cell.getValueLength() == 0) {
return null;
}
String hostAndPort = Bytes.toString(
cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
byte[] startcodeColumn = getStartCodeColumn(replicaId);
cell = r.getColumnLatestCell(CATALOG_FAMILY, startcodeColumn);
if (cell == null || cell.getValueLength() == 0) {
return null;
}
try {
return ServerName.valueOf(hostAndPort,
Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
} catch (IllegalArgumentException e) {
LOG.error("Ignoring invalid region for server " + hostAndPort + "; cell=" + cell, e);
return null;
}
}
public boolean returnValue(Result result) throws SQLException {
Cell statusKV = result.rawCells()[0];
if (statusKV.getValueLength() == 0) { // No error, but unable to return sequence values
return false;
}
long timestamp = statusKV.getTimestamp();
int statusCode = PInteger.INSTANCE.getCodec().decodeInt(statusKV.getValueArray(), statusKV.getValueOffset(), SortOrder.getDefault());
if (statusCode == SUCCESS) { // Success - update nextValue down to currentValue
SequenceValue value = findSequenceValue(timestamp);
if (value == null) {
throw new EmptySequenceCacheException(key.getSchemaName(),key.getSequenceName());
}
return true;
}
SQLExceptionCode code = SQLExceptionCode.fromErrorCode(statusCode);
// TODO: We could have the server return the timestamps of the
// delete markers and we could insert them here, but this seems
// like overkill.
// if (code == SQLExceptionCode.SEQUENCE_UNDEFINED) {
// }
throw new SQLExceptionInfo.Builder(code)
.setSchemaName(key.getSchemaName())
.setTableName(key.getSequenceName())
.build().buildException();
}
public void hashResult(Result result) {
if (!batchStarted) {
throw new RuntimeException("Cannot add to batch that has not been started.");
}
for (Cell cell : result.rawCells()) {
int rowLength = cell.getRowLength();
int familyLength = cell.getFamilyLength();
int qualifierLength = cell.getQualifierLength();
int valueLength = cell.getValueLength();
digest.update(cell.getRowArray(), cell.getRowOffset(), rowLength);
digest.update(cell.getFamilyArray(), cell.getFamilyOffset(), familyLength);
digest.update(cell.getQualifierArray(), cell.getQualifierOffset(), qualifierLength);
if (!ignoreTimestamps) {
long ts = cell.getTimestamp();
for (int i = 8; i > 0; i--) {
digest.update((byte) ts);
ts >>>= 8;
}
}
digest.update(cell.getValueArray(), cell.getValueOffset(), valueLength);
batchSize += rowLength + familyLength + qualifierLength + 8 + valueLength;
}
}
@Override
protected void map(ImmutableBytesWritable key, Result value, Context output) throws IOException, InterruptedException {
for (Cell c : value.rawCells()) {
Statement st = HalyardTableUtils.parseStatement(c, SVF);
if ((subj == null || subj.equals(st.getSubject())) && (pred == null || pred.equals(st.getPredicate())) && (obj == null || obj.equals(st.getObject())) && (ctx == null || ctx.contains(st.getContext()))) {
KeyValue kv = new KeyValue(c.getRowArray(), c.getRowOffset(), (int) c.getRowLength(),
c.getFamilyArray(), c.getFamilyOffset(), (int) c.getFamilyLength(),
c.getQualifierArray(), c.getQualifierOffset(), c.getQualifierLength(),
c.getTimestamp(), KeyValue.Type.DeleteColumn, c.getValueArray(), c.getValueOffset(),
c.getValueLength(), c.getTagsArray(), c.getTagsOffset(), c.getTagsLength());
output.write(new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength()), kv);
deleted++;
} else {
output.progress();
}
if (total++ % 10000l == 0) {
String msg = MessageFormat.format("{0} / {1} cells deleted", deleted, total);
output.setStatus(msg);
LOG.log(Level.INFO, msg);
}
}
}
public boolean returnValue(Result result) throws SQLException {
Cell statusKV = result.rawCells()[0];
if (statusKV.getValueLength() == 0) { // No error, but unable to return sequence values
return false;
}
long timestamp = statusKV.getTimestamp();
int statusCode = PInteger.INSTANCE.getCodec().decodeInt(statusKV.getValueArray(), statusKV.getValueOffset(), SortOrder.getDefault());
if (statusCode == SUCCESS) { // Success - update nextValue down to currentValue
SequenceValue value = findSequenceValue(timestamp);
if (value == null) {
throw new EmptySequenceCacheException(key.getSchemaName(),key.getSequenceName());
}
return true;
}
SQLExceptionCode code = SQLExceptionCode.fromErrorCode(statusCode);
// TODO: We could have the server return the timestamps of the
// delete markers and we could insert them here, but this seems
// like overkill.
// if (code == SQLExceptionCode.SEQUENCE_UNDEFINED) {
// }
throw new SQLExceptionInfo.Builder(code)
.setSchemaName(key.getSchemaName())
.setTableName(key.getSequenceName())
.build().buildException();
}
@SuppressWarnings("deprecation")
private static KeyValue addSaltByte(Cell keyValue, int nSaltBuckets) {
byte[] buf = keyValue.getRowArray();
int length = keyValue.getRowLength();
int offset = keyValue.getRowOffset();
boolean isViewSeq = length > SEQ_PREFIX_BYTES.length && Bytes.compareTo(SEQ_PREFIX_BYTES, 0, SEQ_PREFIX_BYTES.length, buf, offset, SEQ_PREFIX_BYTES.length) == 0;
if (!isViewSeq && nSaltBuckets == 0) {
return null;
}
byte[] newBuf;
if (isViewSeq) { // We messed up the name for the sequences for view indexes so we'll take this opportunity to fix it
if (buf[length-1] == 0) { // Global indexes on views have trailing null byte
length--;
}
byte[][] rowKeyMetaData = new byte[3][];
SchemaUtil.getVarChars(buf, offset, length, 0, rowKeyMetaData);
byte[] schemaName = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
byte[] unprefixedSchemaName = new byte[schemaName.length - MetaDataUtil.VIEW_INDEX_SEQUENCE_PREFIX_BYTES.length];
System.arraycopy(schemaName, MetaDataUtil.VIEW_INDEX_SEQUENCE_PREFIX_BYTES.length, unprefixedSchemaName, 0, unprefixedSchemaName.length);
byte[] tableName = rowKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
PName physicalName = PNameFactory.newName(unprefixedSchemaName);
// Reformulate key based on correct data
newBuf = MetaDataUtil.getViewIndexSequenceKey(tableName == null ? null : Bytes.toString(tableName),
physicalName, nSaltBuckets, false).getKey();
} else {
newBuf = new byte[length + 1];
System.arraycopy(buf, offset, newBuf, SaltingUtil.NUM_SALTING_BYTES, length);
newBuf[0] = SaltingUtil.getSaltingByte(newBuf, SaltingUtil.NUM_SALTING_BYTES, length, nSaltBuckets);
}
return new KeyValue(newBuf, 0, newBuf.length,
buf, keyValue.getFamilyOffset(), keyValue.getFamilyLength(),
buf, keyValue.getQualifierOffset(), keyValue.getQualifierLength(),
keyValue.getTimestamp(), KeyValue.Type.codeToType(keyValue.getTypeByte()),
buf, keyValue.getValueOffset(), keyValue.getValueLength());
}
@Override
public List<T> mapRow(Result result, int rowNum) throws Exception {
if (result.isEmpty()) {
return Collections.emptyList();
}
final byte[] distributedRowKey = result.getRow();
final String agentId = this.hbaseOperationFactory.getAgentId(distributedRowKey);
final long baseTimestamp = this.hbaseOperationFactory.getBaseTimestamp(distributedRowKey);
List<T> dataPoints = new ArrayList<>();
for (Cell cell : result.rawCells()) {
if (CellUtil.matchingFamily(cell, HbaseColumnFamily.AGENT_STAT_STATISTICS.getName())) {
Buffer qualifierBuffer = new OffsetFixedBuffer(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
Buffer valueBuffer = new OffsetFixedBuffer(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
long timestampDelta = this.decoder.decodeQualifier(qualifierBuffer);
AgentStatDecodingContext decodingContext = new AgentStatDecodingContext();
decodingContext.setAgentId(agentId);
decodingContext.setBaseTimestamp(baseTimestamp);
decodingContext.setTimestampDelta(timestampDelta);
List<T> candidates = this.decoder.decodeValue(valueBuffer, decodingContext);
for (T candidate : candidates) {
if (filter(candidate)) {
continue;
}
dataPoints.add(candidate);
}
}
}
// Reverse sort as timestamp is stored in a reversed order.
dataPoints.sort(REVERSE_TIMESTAMP_COMPARATOR);
return dataPoints;
}
@Override
public void readFields(ResultSet resultSet) throws SQLException {
Tuple row = resultSet.unwrap(PhoenixResultSet.class).getCurrentRow();
Cell kv = row.getValue(0);
ImmutableBytesWritable tmpPtr = new ImmutableBytesWritable(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength());
// A single Cell will be returned with the count(*) - we decode that here
rowCount = PLong.INSTANCE.getCodec().decodeLong(tmpPtr, SortOrder.getDefault());
}
private Cell getChangedKey(Cell next, boolean changeBottomKeys) {
// If it is a top store file change the StartKey with SplitKey in Key
//and produce the new value corresponding to the change in key
byte[] changedKey = getNewRowkeyByRegionStartKeyReplacedWithSplitKey(next, changeBottomKeys);
KeyValue changedKv =
new KeyValue(changedKey, 0, changedKey.length, next.getFamilyArray(),
next.getFamilyOffset(), next.getFamilyLength(), next.getQualifierArray(),
next.getQualifierOffset(), next.getQualifierLength(),
next.getTimestamp(), Type.codeToType(next.getTypeByte()),
next.getValueArray(), next.getValueOffset(), next.getValueLength(),
next.getTagsArray(), next.getTagsOffset(), next.getTagsLength());
return changedKv;
}
@Override
public List<JoinStatBo> mapRow(Result result, int rowNum) throws Exception {
if (result.isEmpty()) {
return Collections.emptyList();
}
final byte[] distributedRowKey = result.getRow();
final String applicationId = this.hbaseOperationFactory.getApplicationId(distributedRowKey);
final long baseTimestamp = this.hbaseOperationFactory.getBaseTimestamp(distributedRowKey);
List<JoinStatBo> dataPoints = new ArrayList<>();
for (Cell cell : result.rawCells()) {
if (CellUtil.matchingFamily(cell, HbaseColumnFamily.APPLICATION_STAT_STATISTICS.getName())) {
Buffer qualifierBuffer = new OffsetFixedBuffer(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
Buffer valueBuffer = new OffsetFixedBuffer(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
long timestampDelta = this.decoder.decodeQualifier(qualifierBuffer);
ApplicationStatDecodingContext decodingContext = new ApplicationStatDecodingContext();
decodingContext.setApplicationId(applicationId);
decodingContext.setBaseTimestamp(baseTimestamp);
decodingContext.setTimestampDelta(timestampDelta);
List<JoinStatBo> candidates = this.decoder.decodeValue(valueBuffer, decodingContext);
for (JoinStatBo candidate : candidates) {
long timestamp = candidate.getTimestamp();
if (this.filter.filter(timestamp)) {
continue;
}
dataPoints.add(candidate);
}
}
}
// Reverse sort as timestamp is stored in a reversed order.
dataPoints.sort(REVERSE_TIMESTAMP_COMPARATOR);
return dataPoints;
}
@Override
public void addDestinationTable(long txnId,byte[] destinationTable) throws IOException{
if(LOG.isTraceEnabled())
SpliceLogUtils.trace(LOG,"addDestinationTable txnId=%d, desinationTable",txnId,destinationTable);
Get get=new Get(getRowKey(txnId));
byte[] destTableQualifier=V2TxnDecoder.DESTINATION_TABLE_QUALIFIER_BYTES;
get.addColumn(FAMILY,destTableQualifier);
/*
* We only need to check the new transaction format, because we will never attempt to elevate
* a transaction created using the old transaction format.
*/
Result result=region.get(get);
//should never happen, this is in place to protect against programmer error
if(result==null||result==Result.EMPTY_RESULT)
throw new HReadOnlyModificationException("Transaction "+txnId+" is read-only, and was not properly elevated.");
Cell kv=result.getColumnLatestCell(FAMILY,destTableQualifier);
byte[] newBytes;
if(kv==null || kv.getValueLength()<=0){
newBytes=Encoding.encodeBytesUnsorted(destinationTable);
}else{
newBytes=new byte[destinationTable.length+kv.getValueLength()+1];
System.arraycopy(destinationTable,0,newBytes,0, destinationTable.length);
System.arraycopy(kv.getValueArray(),kv.getValueOffset(),newBytes, destinationTable.length+1,kv.getValueLength());
}
Put put=new Put(get.getRow());
put.addColumn(FAMILY,destTableQualifier,newBytes);
region.put(put);
}
@Override
public void cleanup() {
// actually delete the procedures if it is not the one with the max procedure id.
List<Cell> cells = new ArrayList<Cell>();
try (RegionScanner scanner =
region.getScanner(new Scan().addColumn(PROC_FAMILY, PROC_QUALIFIER).setReversed(true))) {
// skip the row with max procedure id
boolean moreRows = scanner.next(cells);
if (cells.isEmpty()) {
return;
}
cells.clear();
while (moreRows) {
moreRows = scanner.next(cells);
if (cells.isEmpty()) {
continue;
}
Cell cell = cells.get(0);
cells.clear();
if (cell.getValueLength() == 0) {
region.update(r -> r
.delete(new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())));
}
}
} catch (IOException e) {
LOG.warn("Failed to clean up delete procedures", e);
}
}
private void printCell(Cell cell) throws IOException {
out.print("K: " + CellUtil.getCellKeyAsString(cell,
c -> Long.toString(Bytes.toLong(c.getRowArray(), c.getRowOffset(), c.getRowLength()))));
if (cell.getType() == Cell.Type.Put) {
if (cell.getValueLength() == 0) {
out.println(" V: mark deleted");
} else {
Procedure<?> proc = ProcedureUtil.convertToProcedure(ProcedureProtos.Procedure.parser()
.parseFrom(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
out.println(" V: " + proc.toStringDetails());
}
} else {
out.println();
}
}
@Override
public void load(ProcedureLoader loader) throws IOException {
List<ProcedureProtos.Procedure> procs = new ArrayList<>();
long maxProcId = 0;
try (RegionScanner scanner =
region.getScanner(new Scan().addColumn(PROC_FAMILY, PROC_QUALIFIER))) {
List<Cell> cells = new ArrayList<>();
boolean moreRows;
do {
moreRows = scanner.next(cells);
if (cells.isEmpty()) {
continue;
}
Cell cell = cells.get(0);
cells.clear();
maxProcId = Math.max(maxProcId,
Bytes.toLong(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
if (cell.getValueLength() > 0) {
ProcedureProtos.Procedure proto = ProcedureProtos.Procedure.parser()
.parseFrom(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
procs.add(proto);
}
} while (moreRows);
}
loader.setMaxProcId(maxProcId);
ProcedureTree tree = ProcedureTree.build(procs);
loader.load(tree.getValidProcs());
loader.handleCorrupted(tree.getCorruptedProcs());
}
public static boolean matchingValue(Cell keyValue, byte[] value) {
return !(value == null || keyValue == null || value.length != keyValue.getValueLength()) && ArrayUtil.equals
(CellUtils.getBuffer(keyValue), keyValue.getValueOffset(), value, 0, keyValue.getValueLength());
}
private boolean isColumnDelete(Cell cell) {
return !TxUtils.isPreExistingVersion(cell.getTimestamp()) && cell.getValueLength() == 0 && !allowEmptyValues;
}
private boolean isColumnDelete(Cell cell) {
return !TxUtils.isPreExistingVersion(cell.getTimestamp()) && cell.getValueLength() == 0 && !allowEmptyValues;
}
@Override
protected int doWork() throws Exception {
Path path = new Path(file);
FileSystem fs = path.getFileSystem(conf);
try (WAL.Reader reader = WALFactory.createReader(fs, path, conf)) {
for (;;) {
WAL.Entry entry = reader.next();
if (entry == null) {
return 0;
}
WALKey key = entry.getKey();
WALEdit edit = entry.getEdit();
long sequenceId = key.getSequenceId();
long writeTime = key.getWriteTime();
out.println(
String.format(KEY_TMPL, sequenceId, FORMATTER.format(Instant.ofEpochMilli(writeTime))));
for (Cell cell : edit.getCells()) {
Map<String, Object> op = WALPrettyPrinter.toStringMap(cell);
if (!Bytes.equals(PROC_FAMILY, 0, PROC_FAMILY.length, cell.getFamilyArray(),
cell.getFamilyOffset(), cell.getFamilyLength())) {
// We could have cells other than procedure edits, for example, a flush marker
WALPrettyPrinter.printCell(out, op, false);
continue;
}
long procId = Bytes.toLong(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
out.println("pid=" + procId + ", type=" + op.get("type") + ", column=" +
op.get("family") + ":" + op.get("qualifier"));
if (cell.getType() == Cell.Type.Put) {
if (cell.getValueLength() > 0) {
// should be a normal put
Procedure<?> proc =
ProcedureUtil.convertToProcedure(ProcedureProtos.Procedure.parser()
.parseFrom(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
out.println("\t" + proc.toStringDetails());
} else {
// should be a 'delete' put
out.println("\tmark deleted");
}
}
out.println("cell total size sum: " + cell.heapSize());
}
out.println("edit heap size: " + edit.heapSize());
out.println("position: " + reader.getPosition());
}
}
}
private int compressSingleKeyValue(DataOutputStream out, Cell cell, Cell prevCell)
throws IOException {
int flag = 0; // Do not use more bits that can fit into a byte
int kLength = KeyValueUtil.keyLength(cell);
int vLength = cell.getValueLength();
long timestamp;
long diffTimestamp = 0;
int diffTimestampFitsInBytes = 0;
int timestampFitsInBytes;
int commonPrefix = 0;
if (prevCell == null) {
timestamp = cell.getTimestamp();
if (timestamp < 0) {
flag |= FLAG_TIMESTAMP_SIGN;
timestamp = -timestamp;
}
timestampFitsInBytes = ByteBufferUtils.longFitsIn(timestamp);
flag |= (timestampFitsInBytes - 1) << SHIFT_TIMESTAMP_LENGTH;
// put column family
byte familyLength = cell.getFamilyLength();
out.write(familyLength);
PrivateCellUtil.writeFamily(out, cell, familyLength);
} else {
// Finding common prefix
int preKeyLength = KeyValueUtil.keyLength(prevCell);
commonPrefix = PrivateCellUtil.findCommonPrefixInFlatKey(cell, prevCell, true, false);
if (kLength == preKeyLength) {
flag |= FLAG_SAME_KEY_LENGTH;
}
if (vLength == prevCell.getValueLength()) {
flag |= FLAG_SAME_VALUE_LENGTH;
}
if (cell.getTypeByte() == prevCell.getTypeByte()) {
flag |= FLAG_SAME_TYPE;
}
// don't compress timestamp and type using prefix encode timestamp
timestamp = cell.getTimestamp();
diffTimestamp = prevCell.getTimestamp() - timestamp;
boolean negativeTimestamp = timestamp < 0;
if (negativeTimestamp) {
timestamp = -timestamp;
}
timestampFitsInBytes = ByteBufferUtils.longFitsIn(timestamp);
boolean minusDiffTimestamp = diffTimestamp < 0;
if (minusDiffTimestamp) {
diffTimestamp = -diffTimestamp;
}
diffTimestampFitsInBytes = ByteBufferUtils.longFitsIn(diffTimestamp);
if (diffTimestampFitsInBytes < timestampFitsInBytes) {
flag |= (diffTimestampFitsInBytes - 1) << SHIFT_TIMESTAMP_LENGTH;
flag |= FLAG_TIMESTAMP_IS_DIFF;
if (minusDiffTimestamp) {
flag |= FLAG_TIMESTAMP_SIGN;
}
} else {
flag |= (timestampFitsInBytes - 1) << SHIFT_TIMESTAMP_LENGTH;
if (negativeTimestamp) {
flag |= FLAG_TIMESTAMP_SIGN;
}
}
}
out.write(flag);
if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
ByteBufferUtils.putCompressedInt(out, kLength);
}
if ((flag & FLAG_SAME_VALUE_LENGTH) == 0) {
ByteBufferUtils.putCompressedInt(out, vLength);
}
ByteBufferUtils.putCompressedInt(out, commonPrefix);
short rLen = cell.getRowLength();
if (commonPrefix < rLen + KeyValue.ROW_LENGTH_SIZE) {
// Previous and current rows are different. Copy the differing part of
// the row, skip the column family, and copy the qualifier.
PrivateCellUtil.writeRowKeyExcludingCommon(cell, rLen, commonPrefix, out);
PrivateCellUtil.writeQualifier(out, cell, cell.getQualifierLength());
} else {
// The common part includes the whole row. As the column family is the
// same across the whole file, it will automatically be included in the
// common prefix, so we need not special-case it here.
// What we write here is the non common part of the qualifier
int commonQualPrefix = commonPrefix - (rLen + KeyValue.ROW_LENGTH_SIZE)
- (cell.getFamilyLength() + KeyValue.FAMILY_LENGTH_SIZE);
PrivateCellUtil.writeQualifierSkippingBytes(out, cell, cell.getQualifierLength(),
commonQualPrefix);
}
if ((flag & FLAG_TIMESTAMP_IS_DIFF) == 0) {
ByteBufferUtils.putLong(out, timestamp, timestampFitsInBytes);
} else {
ByteBufferUtils.putLong(out, diffTimestamp, diffTimestampFitsInBytes);
}
if ((flag & FLAG_SAME_TYPE) == 0) {
out.write(cell.getTypeByte());
}
PrivateCellUtil.writeValue(out, cell, vLength);
return kLength + vLength + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE;
}
public boolean isFailedCommitTimestamp(Cell element) {
return element.getValueLength()==1 && element.getValueArray()[element.getValueOffset()]==SIConstants.SNAPSHOT_ISOLATION_FAILED_TIMESTAMP[0];
}