下面列出了org.apache.hadoop.hbase.Cell#getTagsLength ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
@Override
protected void map(ImmutableBytesWritable key, Result value, Context output) throws IOException, InterruptedException {
for (Cell c : value.rawCells()) {
Statement st = HalyardTableUtils.parseStatement(c, SVF);
if ((subj == null || subj.equals(st.getSubject())) && (pred == null || pred.equals(st.getPredicate())) && (obj == null || obj.equals(st.getObject())) && (ctx == null || ctx.contains(st.getContext()))) {
KeyValue kv = new KeyValue(c.getRowArray(), c.getRowOffset(), (int) c.getRowLength(),
c.getFamilyArray(), c.getFamilyOffset(), (int) c.getFamilyLength(),
c.getQualifierArray(), c.getQualifierOffset(), c.getQualifierLength(),
c.getTimestamp(), KeyValue.Type.DeleteColumn, c.getValueArray(), c.getValueOffset(),
c.getValueLength(), c.getTagsArray(), c.getTagsOffset(), c.getTagsLength());
output.write(new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength()), kv);
deleted++;
} else {
output.progress();
}
if (total++ % 10000l == 0) {
String msg = MessageFormat.format("{0} / {1} cells deleted", deleted, total);
output.setStatus(msg);
LOG.log(Level.INFO, msg);
}
}
}
public static Map<String, Object> toStringMap(Cell cell) {
Map<String, Object> stringMap = new HashMap<>();
stringMap.put("row",
Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
stringMap.put("type", cell.getType());
stringMap.put("family", Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(),
cell.getFamilyLength()));
stringMap.put("qualifier",
Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(),
cell.getQualifierLength()));
stringMap.put("timestamp", cell.getTimestamp());
stringMap.put("vlen", cell.getValueLength());
if (cell.getTagsLength() > 0) {
List<String> tagsString = new ArrayList<>();
Iterator<Tag> tagsIterator = PrivateCellUtil.tagsIterator(cell);
while (tagsIterator.hasNext()) {
Tag tag = tagsIterator.next();
tagsString
.add((tag.getType()) + ":" + Bytes.toStringBinary(Tag.cloneValue(tag)));
}
stringMap.put("tag", tagsString);
}
return stringMap;
}
public int write(Cell cell) throws IOException {
// We write tags seperately because though there is no tag in KV
// if the hfilecontext says include tags we need the tags length to be
// written
int size = KeyValueUtil.oswrite(cell, out, false);
// Write the additional tag into the stream
if (encodingCtx.getHFileContext().isIncludesTags()) {
int tagsLength = cell.getTagsLength();
out.writeShort(tagsLength);
if (tagsLength > 0) {
PrivateCellUtil.writeTags(out, cell, tagsLength);
}
size += tagsLength + KeyValue.TAGS_LENGTH_SIZE;
}
if (encodingCtx.getHFileContext().isIncludesMvcc()) {
WritableUtils.writeVLong(out, cell.getSequenceId());
size += WritableUtils.getVIntSize(cell.getSequenceId());
}
return size;
}
/**
* Creates a {@link TResult} (Thrift) from a {@link Result} (HBase).
*
* @param in the <code>Result</code> to convert
*
* @return converted result, returns an empty result if the input is <code>null</code>
*/
public static TResult resultFromHBase(Result in) {
Cell[] raw = in.rawCells();
TResult out = new TResult();
byte[] row = in.getRow();
if (row != null) {
out.setRow(in.getRow());
}
List<TColumnValue> columnValues = new ArrayList<>(raw.length);
for (Cell kv : raw) {
TColumnValue col = new TColumnValue();
col.setFamily(CellUtil.cloneFamily(kv));
col.setQualifier(CellUtil.cloneQualifier(kv));
col.setTimestamp(kv.getTimestamp());
col.setValue(CellUtil.cloneValue(kv));
col.setType(kv.getType().getCode());
if (kv.getTagsLength() > 0) {
col.setTags(PrivateCellUtil.cloneTags(kv));
}
columnValues.add(col);
}
out.setColumnValues(columnValues);
out.setStale(in.isStale());
out.setPartial(in.mayHaveMoreCellsInRow());
return out;
}
public static TPut putFromHBase(Put in) {
TPut out = new TPut();
out.setRow(in.getRow());
if (in.getTimestamp() != HConstants.LATEST_TIMESTAMP) {
out.setTimestamp(in.getTimestamp());
}
if (in.getDurability() != Durability.USE_DEFAULT) {
out.setDurability(durabilityFromHBase(in.getDurability()));
}
for (Map.Entry<byte [], List<Cell>> entry : in.getFamilyCellMap().entrySet()) {
byte[] family = entry.getKey();
for (Cell cell : entry.getValue()) {
TColumnValue columnValue = new TColumnValue();
columnValue.setFamily(family)
.setQualifier(CellUtil.cloneQualifier(cell))
.setType(cell.getType().getCode())
.setTimestamp(cell.getTimestamp())
.setValue(CellUtil.cloneValue(cell));
if (cell.getTagsLength() != 0) {
columnValue.setTags(PrivateCellUtil.cloneTags(cell));
}
out.addToColumnValues(columnValue);
}
}
for (Map.Entry<String, byte[]> attribute : in.getAttributesMap().entrySet()) {
out.putToAttributes(ByteBuffer.wrap(Bytes.toBytes(attribute.getKey())),
ByteBuffer.wrap(attribute.getValue()));
}
try {
CellVisibility cellVisibility = in.getCellVisibility();
if (cellVisibility != null) {
TCellVisibility tCellVisibility = new TCellVisibility();
tCellVisibility.setExpression(cellVisibility.getExpression());
out.setCellVisibility(tCellVisibility);
}
} catch (DeserializationException e) {
throw new RuntimeException(e);
}
return out;
}
public static TAppend appendFromHBase(Append in) throws IOException {
TAppend out = new TAppend();
out.setRow(in.getRow());
if (in.getDurability() != Durability.USE_DEFAULT) {
out.setDurability(durabilityFromHBase(in.getDurability()));
}
for (Map.Entry<byte [], List<Cell>> entry : in.getFamilyCellMap().entrySet()) {
byte[] family = entry.getKey();
for (Cell cell : entry.getValue()) {
TColumnValue columnValue = new TColumnValue();
columnValue.setFamily(family)
.setQualifier(CellUtil.cloneQualifier(cell))
.setType(cell.getType().getCode())
.setTimestamp(cell.getTimestamp())
.setValue(CellUtil.cloneValue(cell));
if (cell.getTagsLength() != 0) {
columnValue.setTags(PrivateCellUtil.cloneTags(cell));
}
out.addToColumns(columnValue);
}
}
for (Map.Entry<String, byte[]> attribute : in.getAttributesMap().entrySet()) {
out.putToAttributes(ByteBuffer.wrap(Bytes.toBytes(attribute.getKey())),
ByteBuffer.wrap(attribute.getValue()));
}
try {
CellVisibility cellVisibility = in.getCellVisibility();
if (cellVisibility != null) {
TCellVisibility tCellVisibility = new TCellVisibility();
tCellVisibility.setExpression(cellVisibility.getExpression());
out.setCellVisibility(tCellVisibility);
}
} catch (DeserializationException e) {
throw new RuntimeException(e);
}
out.setReturnResults(in.isReturnResults());
return out;
}
/**
* Whether the current cell is a mob reference cell.
* @param cell The current cell.
* @return True if the cell has a mob reference tag, false if it doesn't.
*/
public static boolean isMobReferenceCell(Cell cell) {
if (cell.getTagsLength() > 0) {
Optional<Tag> tag = PrivateCellUtil.getTag(cell, TagType.MOB_REFERENCE_TAG_TYPE);
if (tag.isPresent()) {
return true;
}
}
return false;
}
/**
* Gets the table name tag.
* @param cell The current cell.
* @return The table name tag.
*/
private static Optional<Tag> getTableNameTag(Cell cell) {
Optional<Tag> tag = Optional.empty();
if (cell.getTagsLength() > 0) {
tag = PrivateCellUtil.getTag(cell, TagType.MOB_TABLE_NAME_TAG_TYPE);
}
return tag;
}
@Override
public void write(Cell cell) throws IOException {
// We first write the KeyValue infrastructure as VInts.
StreamUtils.writeRawVInt32(out, KeyValueUtil.keyLength(cell));
StreamUtils.writeRawVInt32(out, cell.getValueLength());
// To support tags
int tagsLength = cell.getTagsLength();
StreamUtils.writeRawVInt32(out, tagsLength);
PrivateCellUtil.compressRow(out, cell,
compression.getDictionary(CompressionContext.DictionaryIndex.ROW));
PrivateCellUtil.compressFamily(out, cell,
compression.getDictionary(CompressionContext.DictionaryIndex.FAMILY));
PrivateCellUtil.compressQualifier(out, cell,
compression.getDictionary(CompressionContext.DictionaryIndex.QUALIFIER));
// Write timestamp, type and value as uncompressed.
StreamUtils.writeLong(out, cell.getTimestamp());
out.write(cell.getTypeByte());
PrivateCellUtil.writeValue(out, cell, cell.getValueLength());
if (tagsLength > 0) {
if (compression.tagCompressionContext != null) {
// Write tags using Dictionary compression
PrivateCellUtil.compressTags(out, cell, compression.tagCompressionContext);
} else {
// Tag compression is disabled within the WAL compression. Just write the tags bytes as
// it is.
PrivateCellUtil.writeTags(out, cell, tagsLength);
}
}
}
protected void updateMetaInfo(Cell cellToAdd, boolean succ, boolean mslabUsed,
MemStoreSizing memstoreSizing, boolean sizeAddedPreOperation) {
long delta = 0;
long cellSize = getCellLength(cellToAdd);
int cellsCount = succ ? 1 : 0;
// If there's already a same cell in the CellSet and we are using MSLAB, we must count in the
// MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger
// than the counted number)
if (succ || mslabUsed) {
delta = cellSize;
}
if (sizeAddedPreOperation) {
delta -= cellSize;
}
long heapSize = heapSizeChange(cellToAdd, succ || mslabUsed);
long offHeapSize = offHeapSizeChange(cellToAdd, succ || mslabUsed);
incMemStoreSize(delta, heapSize, offHeapSize, cellsCount);
if (memstoreSizing != null) {
memstoreSizing.incMemStoreSize(delta, heapSize, offHeapSize, cellsCount);
}
getTimeRangeTracker().includeTimestamp(cellToAdd);
minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId());
// In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call.
// When we use ACL CP or Visibility CP which deals with Tags during
// mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not
// parse the byte[] to identify the tags length.
if (cellToAdd.getTagsLength() > 0) {
tagsPresent = true;
}
}
/**
* Clone the passed cell by copying its data into the passed buf and create a cell with a chunkid
* out of it
* @see #copyBBECToChunkCell(ByteBufferExtendedCell, ByteBuffer, int, int)
*/
private static Cell copyToChunkCell(Cell cell, ByteBuffer buf, int offset, int len) {
int tagsLen = cell.getTagsLength();
if (cell instanceof ExtendedCell) {
((ExtendedCell) cell).write(buf, offset);
} else {
// Normally all Cell impls within Server will be of type ExtendedCell. Just considering the
// other case also. The data fragments within Cell is copied into buf as in KeyValue
// serialization format only.
KeyValueUtil.appendTo(cell, buf, offset, true);
}
return createChunkCell(buf, offset, len, tagsLen, cell.getSequenceId());
}
/**
* Add key/value to file. Keys must be added in an order that agrees with the
* Comparator passed on construction.
*
* @param cell
* Cell to add. Cannot be empty nor null.
*/
@Override
public void append(final Cell cell) throws IOException {
// checkKey uses comparator to check we are writing in order.
boolean dupKey = checkKey(cell);
if (!dupKey) {
checkBlockBoundary();
}
if (!blockWriter.isWriting()) {
newBlock();
}
blockWriter.write(cell);
totalKeyLength += PrivateCellUtil.estimatedSerializedSizeOfKey(cell);
totalValueLength += cell.getValueLength();
// Are we the first key in this block?
if (firstCellInBlock == null) {
// If cell is big, block will be closed and this firstCellInBlock reference will only last
// a short while.
firstCellInBlock = cell;
}
// TODO: What if cell is 10MB and we write infrequently? We hold on to cell here indefinitely?
lastCell = cell;
entryCount++;
this.maxMemstoreTS = Math.max(this.maxMemstoreTS, cell.getSequenceId());
int tagsLength = cell.getTagsLength();
if (tagsLength > this.maxTagsLength) {
this.maxTagsLength = tagsLength;
}
}
/**
* @return unencoded size added
*/
protected final int afterEncodingKeyValue(Cell cell, DataOutputStream out,
HFileBlockDefaultEncodingContext encodingCtx) throws IOException {
int size = 0;
if (encodingCtx.getHFileContext().isIncludesTags()) {
int tagsLength = cell.getTagsLength();
ByteBufferUtils.putCompressedInt(out, tagsLength);
// There are some tags to be written
if (tagsLength > 0) {
TagCompressionContext tagCompressionContext = encodingCtx.getTagCompressionContext();
// When tag compression is enabled, tagCompressionContext will have a not null value. Write
// the tags using Dictionary compression in such a case
if (tagCompressionContext != null) {
// Not passing tagsLength considering that parsing of the tagsLength is not costly
PrivateCellUtil.compressTags(out, cell, tagCompressionContext);
} else {
PrivateCellUtil.writeTags(out, cell, tagsLength);
}
}
size += tagsLength + KeyValue.TAGS_LENGTH_SIZE;
}
if (encodingCtx.getHFileContext().isIncludesMvcc()) {
// Copy memstore timestamp from the byte buffer to the output stream.
long memstoreTS = cell.getSequenceId();
WritableUtils.writeVLong(out, memstoreTS);
// TODO use a writeVLong which returns the #bytes written so that 2 time parsing can be
// avoided.
size += WritableUtils.getVIntSize(memstoreTS);
}
return size;
}
private Cell getChangedKey(Cell next, boolean changeBottomKeys) {
// If it is a top store file change the StartKey with SplitKey in Key
//and produce the new value corresponding to the change in key
byte[] changedKey = getNewRowkeyByRegionStartKeyReplacedWithSplitKey(next, changeBottomKeys);
KeyValue changedKv =
new KeyValue(changedKey, 0, changedKey.length, next.getFamilyArray(),
next.getFamilyOffset(), next.getFamilyLength(), next.getQualifierArray(),
next.getQualifierOffset(), next.getQualifierLength(),
next.getTimestamp(), Type.codeToType(next.getTypeByte()),
next.getValueArray(), next.getValueOffset(), next.getValueLength(),
next.getTagsArray(), next.getTagsOffset(), next.getTagsLength());
return changedKv;
}
/**
* @return Sum of the lengths of all the elements in a Cell; does not count in any infrastructure
*/
private static int getSumOfCellElementLengths(final Cell cell) {
return getSumOfCellKeyElementLengths(cell) + cell.getValueLength() + cell.getTagsLength();
}
@Override
public boolean replicate(ReplicateContext replicateContext) {
if (!delegator.canReplicateToSameCluster()) {
// Only when the replication is inter cluster replication we need to
// convert the visibility tags to
// string based tags. But for intra cluster replication like region
// replicas it is not needed.
List<Entry> entries = replicateContext.getEntries();
List<Tag> visTags = new ArrayList<>();
List<Tag> nonVisTags = new ArrayList<>();
List<Entry> newEntries = new ArrayList<>(entries.size());
for (Entry entry : entries) {
WALEdit newEdit = new WALEdit();
ArrayList<Cell> cells = entry.getEdit().getCells();
for (Cell cell : cells) {
if (cell.getTagsLength() > 0) {
visTags.clear();
nonVisTags.clear();
Byte serializationFormat = VisibilityUtils.extractAndPartitionTags(cell, visTags,
nonVisTags);
if (!visTags.isEmpty()) {
try {
byte[] modifiedVisExpression = visibilityLabelsService
.encodeVisibilityForReplication(visTags, serializationFormat);
if (modifiedVisExpression != null) {
nonVisTags
.add(new ArrayBackedTag(TagType.STRING_VIS_TAG_TYPE, modifiedVisExpression));
}
} catch (Exception ioe) {
LOG.error(
"Exception while reading the visibility labels from the cell. The replication "
+ "would happen as per the existing format and not as " +
"string type for the cell "
+ cell + ".", ioe);
// just return the old entries as it is without applying the string type change
newEdit.add(cell);
continue;
}
// Recreate the cell with the new tags and the existing tags
Cell newCell = PrivateCellUtil.createCell(cell, nonVisTags);
newEdit.add(newCell);
} else {
newEdit.add(cell);
}
} else {
newEdit.add(cell);
}
}
newEntries.add(new Entry((entry.getKey()), newEdit));
}
replicateContext.setEntries(newEntries);
return delegator.replicate(replicateContext);
} else {
return delegator.replicate(replicateContext);
}
}
public static List<Permission> getCellPermissionsForUser(User user, Cell cell)
throws IOException {
// Save an object allocation where we can
if (cell.getTagsLength() == 0) {
return null;
}
List<Permission> results = Lists.newArrayList();
Iterator<Tag> tagsIterator = PrivateCellUtil.tagsIterator(cell);
while (tagsIterator.hasNext()) {
Tag tag = tagsIterator.next();
if (tag.getType() == ACL_TAG_TYPE) {
// Deserialize the table permissions from the KV
// TODO: This can be improved. Don't build UsersAndPermissions just to unpack it again,
// use the builder
AccessControlProtos.UsersAndPermissions.Builder builder =
AccessControlProtos.UsersAndPermissions.newBuilder();
if (tag.hasArray()) {
ProtobufUtil.mergeFrom(builder, tag.getValueArray(), tag.getValueOffset(),
tag.getValueLength());
} else {
ProtobufUtil.mergeFrom(builder, Tag.cloneValue(tag));
}
ListMultimap<String,Permission> kvPerms =
AccessControlUtil.toUsersAndPermissions(builder.build());
// Are there permissions for this user?
List<Permission> userPerms = kvPerms.get(user.getShortName());
if (userPerms != null) {
results.addAll(userPerms);
}
// Are there permissions for any of the groups this user belongs to?
String[] groupNames = user.getGroupNames();
if (groupNames != null) {
for (String group : groupNames) {
List<Permission> groupPerms = kvPerms.get(AuthUtil.toGroupEntry(group));
if (results != null) {
results.addAll(groupPerms);
}
}
}
}
}
return results;
}
@Override
public void write(Cell cell) throws IOException {
if (encryptor == null) {
super.write(cell);
return;
}
byte[] iv = nextIv();
encryptor.setIv(iv);
encryptor.reset();
// TODO: Check if this is a cell for an encrypted CF. If not, we can
// write a 0 here to signal an unwrapped cell and just dump the KV bytes
// afterward
StreamUtils.writeRawVInt32(out, iv.length);
out.write(iv);
// TODO: Add support for WAL compression
ByteArrayOutputStream baos = new ByteArrayOutputStream();
OutputStream cout = encryptor.createEncryptionStream(baos);
ByteBufferWriterOutputStream bos = new ByteBufferWriterOutputStream(cout);
int tlen = cell.getTagsLength();
// Write the KeyValue infrastructure as VInts.
StreamUtils.writeRawVInt32(bos, KeyValueUtil.keyLength(cell));
StreamUtils.writeRawVInt32(bos, cell.getValueLength());
// To support tags
StreamUtils.writeRawVInt32(bos, tlen);
// Write row, qualifier, and family
short rowLength = cell.getRowLength();
StreamUtils.writeRawVInt32(bos, rowLength);
PrivateCellUtil.writeRow(bos, cell, rowLength);
byte familyLength = cell.getFamilyLength();
StreamUtils.writeRawVInt32(bos, familyLength);
PrivateCellUtil.writeFamily(bos, cell, familyLength);
int qualifierLength = cell.getQualifierLength();
StreamUtils.writeRawVInt32(bos, qualifierLength);
PrivateCellUtil.writeQualifier(bos, cell, qualifierLength);
// Write the rest ie. ts, type, value and tags parts
StreamUtils.writeLong(bos, cell.getTimestamp());
bos.write(cell.getTypeByte());
PrivateCellUtil.writeValue(bos, cell, cell.getValueLength());
if (tlen > 0) {
PrivateCellUtil.writeTags(bos, cell, tlen);
}
bos.close();
StreamUtils.writeRawVInt32(out, baos.size());
baos.writeTo(out);
// Increment IV given the final payload length
incrementIv(baos.size());
}
protected void testSeekBeforeInternals(TagUsage tagUsage) throws IOException {
Path p = makeNewFile(tagUsage);
FileSystem fs = TEST_UTIL.getTestFileSystem();
Configuration conf = TEST_UTIL.getConfiguration();
HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), true, conf);
HFileScanner scanner = reader.getScanner(false, true);
assertFalse(scanner.seekBefore(toKV("a", tagUsage)));
assertFalse(scanner.seekBefore(toKV("c", tagUsage)));
assertTrue(scanner.seekBefore(toKV("d", tagUsage)));
assertEquals("c", toRowStr(scanner.getCell()));
assertTrue(scanner.seekBefore(toKV("e", tagUsage)));
assertEquals("c", toRowStr(scanner.getCell()));
assertTrue(scanner.seekBefore(toKV("f", tagUsage)));
assertEquals("e", toRowStr(scanner.getCell()));
assertTrue(scanner.seekBefore(toKV("g", tagUsage)));
assertEquals("e", toRowStr(scanner.getCell()));
assertTrue(scanner.seekBefore(toKV("h", tagUsage)));
assertEquals("g", toRowStr(scanner.getCell()));
assertTrue(scanner.seekBefore(toKV("i", tagUsage)));
assertEquals("g", toRowStr(scanner.getCell()));
assertTrue(scanner.seekBefore(toKV("j", tagUsage)));
assertEquals("i", toRowStr(scanner.getCell()));
Cell cell = scanner.getCell();
if (tagUsage != TagUsage.NO_TAG && cell.getTagsLength() > 0) {
Iterator<Tag> tagsIterator = PrivateCellUtil.tagsIterator(cell);
while (tagsIterator.hasNext()) {
Tag next = tagsIterator.next();
assertEquals("myTag1", Bytes.toString(Tag.cloneValue(next)));
}
}
assertTrue(scanner.seekBefore(toKV("k", tagUsage)));
assertEquals("i", toRowStr(scanner.getCell()));
assertTrue(scanner.seekBefore(toKV("l", tagUsage)));
assertEquals("k", toRowStr(scanner.getCell()));
reader.close();
deleteTestDir(fs);
}
protected Cell mergeList(final List<Cell> cells) {
synchronized (MUTEX) {
Mergeable currentMergeable = null;
final Cell firstCell = cells.get(0);
for (final Cell cell : cells) {
final Mergeable mergeable =
getMergeable(
cell,
// TODO consider avoiding extra byte array
// allocations (which would require
// persistence utils to be able to use
// bytebuffer instead of byte[])
CellUtil.cloneValue(cell));
if (mergeable != null) {
if (currentMergeable == null) {
currentMergeable = mergeable;
} else {
currentMergeable.merge(mergeable);
}
}
}
final byte[] valueBinary = getBinary(currentMergeable);
// this is basically a lengthy verbose form of cloning
// in-place (without allocating new byte arrays) and
// simply replacing the value with the new mergeable
// value
return new KeyValue(
firstCell.getRowArray(),
firstCell.getRowOffset(),
firstCell.getRowLength(),
firstCell.getFamilyArray(),
firstCell.getFamilyOffset(),
firstCell.getFamilyLength(),
firstCell.getQualifierArray(),
firstCell.getQualifierOffset(),
firstCell.getQualifierLength(),
firstCell.getTimestamp(),
Type.codeToType(firstCell.getTypeByte()),
valueBinary,
0,
valueBinary.length,
firstCell.getTagsArray(),
firstCell.getTagsOffset(),
firstCell.getTagsLength());
}
}