下面列出了org.apache.hadoop.hbase.fs.HFileSystem#get ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
@Before
public void setUp() throws IOException {
conf = TEST_UTIL.getConfiguration();
this.conf.set("dfs.datanode.data.dir.perm", "700");
conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, INDEX_BLOCK_SIZE);
conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE, BLOOM_BLOCK_SIZE);
conf.setBoolean(CacheConfig.CACHE_DATA_BLOCKS_COMPRESSED_KEY, cacheCompressedData);
cowType.modifyConf(conf);
conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, cowType.shouldBeCached(BlockType.DATA));
conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
cowType.shouldBeCached(BlockType.LEAF_INDEX));
conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
cowType.shouldBeCached(BlockType.BLOOM_CHUNK));
cacheConf = new CacheConfig(conf, blockCache);
fs = HFileSystem.get(conf);
}
@Override public int run(String[] args) throws Exception {
if (args.length < 1) {
System.err.println("Usage: Clean <output dir>");
return -1;
}
Path p = new Path(args[0]);
Configuration conf = getConf();
TableName tableName = getTableName(conf);
try (FileSystem fs = HFileSystem.get(conf);
Connection conn = ConnectionFactory.createConnection(conf);
Admin admin = conn.getAdmin()) {
if (admin.tableExists(tableName)) {
admin.disableTable(tableName);
admin.deleteTable(tableName);
}
if (fs.exists(p)) {
fs.delete(p, true);
}
}
return 0;
}
@Before
public void setUp() throws IOException {
// parameterized tests add [#] suffix get rid of [ and ].
table = Bytes.toBytes(name.getMethodName().replaceAll("[\\[\\]]", "_"));
conf = TEST_UTIL.getConfiguration();
conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION);
conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false);
conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, false);
conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, false);
fs = HFileSystem.get(conf);
// Create the schema
ColumnFamilyDescriptor hcd = cowType
.modifyFamilySchema(
ColumnFamilyDescriptorBuilder.newBuilder(family).setBloomFilterType(BloomType.ROWCOL))
.build();
TableDescriptor htd =
TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setColumnFamily(hcd).build();
// Create a store based on the schema
String id = TestCacheOnWriteInSchema.class.getName();
Path logdir =
new Path(CommonFSUtils.getRootDir(conf), AbstractFSWALProvider.getWALDirectoryName(id));
fs.delete(logdir, true);
RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
walFactory = new WALFactory(conf, id);
region = TEST_UTIL.createLocalHRegion(info, htd, walFactory.getWAL(info));
region.setBlockCache(BlockCacheFactory.createBlockCache(conf));
store = new HStore(region, hcd, conf, false);
}
@Before
public void setUp() throws IOException {
keys.clear();
rand = new Random(2389757);
firstKeyInFile = null;
conf = TEST_UTIL.getConfiguration();
// This test requires at least HFile format version 2.
conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION);
fs = HFileSystem.get(conf);
}
private void clear() throws IOException {
keys.clear();
rand = new Random(2389757);
firstKeyInFile = null;
conf = TEST_UTIL.getConfiguration();
// This test requires at least HFile format version 2.
conf.setInt(HFile.FORMAT_VERSION_KEY, 3);
fs = HFileSystem.get(conf);
}
@Before
public void setUp() throws IOException {
conf = TEST_UTIL.getConfiguration();
conf.setBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, true);
fs = HFileSystem.get(conf);
blockCache = BlockCacheFactory.createBlockCache(conf);
cacheConf = new CacheConfig(conf, blockCache);
}
public FileSystem getTestFileSystem() throws IOException {
return HFileSystem.get(conf);
}
/**
* Scanner.seekBefore() could fail because when seeking to a previous HFile data block, it needs
* to know the size of that data block, which it calculates using current data block offset and
* the previous data block offset. This fails to work when there are leaf-level index blocks in
* the scannable section of the HFile, i.e. starting in HFileV2. This test will try seekBefore()
* on a flat (single-level) and multi-level (2,3) HFile and confirm this bug is now fixed. This
* bug also happens for inline Bloom blocks for the same reasons.
*/
@Test
public void testMultiIndexLevelRandomHFileWithBlooms() throws IOException {
conf = TEST_UTIL.getConfiguration();
TEST_UTIL.getConfiguration().setInt(BloomFilterUtil.PREFIX_LENGTH_KEY, 10);
// Try out different HFile versions to ensure reverse scan works on each version
for (int hfileVersion = HFile.MIN_FORMAT_VERSION_WITH_TAGS;
hfileVersion <= HFile.MAX_FORMAT_VERSION; hfileVersion++) {
conf.setInt(HFile.FORMAT_VERSION_KEY, hfileVersion);
fs = HFileSystem.get(conf);
// Try out different bloom types because inline Bloom blocks break seekBefore()
for (BloomType bloomType : BloomType.values()) {
// Test out HFile block indices of various sizes/levels
for (int testI = 0; testI < INDEX_CHUNK_SIZES.length; testI++) {
int indexBlockSize = INDEX_CHUNK_SIZES[testI];
int expectedNumLevels = EXPECTED_NUM_LEVELS[testI];
LOG.info(String.format("Testing HFileVersion: %s, BloomType: %s, Index Levels: %s",
hfileVersion, bloomType, expectedNumLevels));
conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, indexBlockSize);
conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE, BLOOM_BLOCK_SIZE);
conf.setInt(BloomFilterUtil.PREFIX_LENGTH_KEY, 10);
Cell[] cells = new Cell[NUM_KV];
Path hfilePath = new Path(TEST_UTIL.getDataTestDir(),
String.format("testMultiIndexLevelRandomHFileWithBlooms-%s-%s-%s",
hfileVersion, bloomType, testI));
// Disable caching to prevent it from hiding any bugs in block seeks/reads
conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
CacheConfig cacheConf = new CacheConfig(conf);
// Write the HFile
{
HFileContext meta = new HFileContextBuilder()
.withBlockSize(DATA_BLOCK_SIZE)
.build();
StoreFileWriter storeFileWriter =
new StoreFileWriter.Builder(conf, cacheConf, fs)
.withFilePath(hfilePath)
.withFileContext(meta)
.withBloomType(bloomType)
.build();
for (int i = 0; i < NUM_KV; i++) {
byte[] row = RandomKeyValueUtil.randomOrderedKey(RAND, i);
byte[] qual = RandomKeyValueUtil.randomRowOrQualifier(RAND);
byte[] value = RandomKeyValueUtil.randomValue(RAND);
KeyValue kv = new KeyValue(row, FAM, qual, value);
storeFileWriter.append(kv);
cells[i] = kv;
}
storeFileWriter.close();
}
// Read the HFile
HFile.Reader reader = HFile.createReader(fs, hfilePath, cacheConf, true, conf);
// Sanity check the HFile index level
assertEquals(expectedNumLevels, reader.getTrailer().getNumDataIndexLevels());
// Check that we can seekBefore in either direction and with both pread
// enabled and disabled
for (boolean pread : new boolean[] { false, true }) {
HFileScanner scanner = reader.getScanner(true, pread);
checkNoSeekBefore(cells, scanner, 0);
for (int i = 1; i < NUM_KV; i++) {
checkSeekBefore(cells, scanner, i);
checkCell(cells[i-1], scanner.getCell());
}
assertTrue(scanner.seekTo());
for (int i = NUM_KV - 1; i >= 1; i--) {
checkSeekBefore(cells, scanner, i);
checkCell(cells[i-1], scanner.getCell());
}
checkNoSeekBefore(cells, scanner, 0);
scanner.close();
}
reader.close();
}
}
}
}
@Before
public void setUp() throws IOException {
fs = HFileSystem.get(TEST_UTIL.getConfiguration());
}
@Before
public void setUp() throws Exception {
fs = HFileSystem.get(TEST_UTIL.getConfiguration());
hfs = (HFileSystem)fs;
}