org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner#lowerBound ( )源码实例Demo

下面列出了org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner#lowerBound ( ) 实例代码，或者点击链接到github查看源代码，也可以在右侧发表评论。

源代码1 项目： hadoop 文件： TestTFileByteArrays.java

@Test
public void testFailureNegativeOffset_2() throws IOException {
  if (skip)
    return;
  closeOutput();

  Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Scanner scanner = reader.createScanner();
  try {
    scanner.lowerBound("keyX".getBytes(), -1, 4);
    Assert.fail("Error on handling negative offset.");
  } catch (Exception e) {
    // noop, expecting exceptions
  } finally {
    reader.close();
    scanner.close();
  }
  closeOutput();
}

源代码2 项目： hadoop 文件： TestTFileByteArrays.java

@Test
public void testFailureNegativeLength_2() throws IOException {
  if (skip)
    return;
  closeOutput();

  Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Scanner scanner = reader.createScanner();
  try {
    scanner.lowerBound("keyX".getBytes(), 0, -1);
    Assert.fail("Error on handling negative length.");
  } catch (Exception e) {
    // noop, expecting exceptions
  } finally {
    scanner.close();
    reader.close();
  }
  closeOutput();
}

源代码3 项目： big-c 文件： TestTFileByteArrays.java

@Test
public void testFailureNegativeOffset_2() throws IOException {
  if (skip)
    return;
  closeOutput();

  Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Scanner scanner = reader.createScanner();
  try {
    scanner.lowerBound("keyX".getBytes(), -1, 4);
    Assert.fail("Error on handling negative offset.");
  } catch (Exception e) {
    // noop, expecting exceptions
  } finally {
    reader.close();
    scanner.close();
  }
  closeOutput();
}

源代码4 项目： big-c 文件： TestTFileByteArrays.java

@Test
public void testFailureNegativeLength_2() throws IOException {
  if (skip)
    return;
  closeOutput();

  Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Scanner scanner = reader.createScanner();
  try {
    scanner.lowerBound("keyX".getBytes(), 0, -1);
    Assert.fail("Error on handling negative length.");
  } catch (Exception e) {
    // noop, expecting exceptions
  } finally {
    scanner.close();
    reader.close();
  }
  closeOutput();
}

源代码5 项目： RDFS 文件： TestTFileByteArrays.java

@Test
public void testFailureNegativeOffset_2() throws IOException {
  if (skip)
    return;
  closeOutput();

  Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Scanner scanner = reader.createScanner();
  try {
    scanner.lowerBound("keyX".getBytes(), -1, 4);
    Assert.fail("Error on handling negative offset.");
  } catch (Exception e) {
    // noop, expecting exceptions
  } finally {
    reader.close();
    scanner.close();
  }
  closeOutput();
}

源代码6 项目： RDFS 文件： TestTFileByteArrays.java

@Test
public void testFailureNegativeLength_2() throws IOException {
  if (skip)
    return;
  closeOutput();

  Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Scanner scanner = reader.createScanner();
  try {
    scanner.lowerBound("keyX".getBytes(), 0, -1);
    Assert.fail("Error on handling negative length.");
  } catch (Exception e) {
    // noop, expecting exceptions
  } finally {
    scanner.close();
    reader.close();
  }
  closeOutput();
}

源代码7 项目： hadoop-gpu 文件： TestTFileByteArrays.java

public void testFailureNegativeOffset_2() throws IOException {
  if (skip)
    return;
  closeOutput();

  Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Scanner scanner = reader.createScanner();
  try {
    scanner.lowerBound("keyX".getBytes(), -1, 4);
    Assert.fail("Error on handling negative offset.");
  }
  catch (Exception e) {
    // noop, expecting exceptions
  }
  finally {
    reader.close();
    scanner.close();
  }
  closeOutput();
}

源代码8 项目： hadoop-gpu 文件： TestTFileByteArrays.java

public void testFailureNegativeLength_2() throws IOException {
  if (skip)
    return;
  closeOutput();

  Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Scanner scanner = reader.createScanner();
  try {
    scanner.lowerBound("keyX".getBytes(), 0, -1);
    Assert.fail("Error on handling negative length.");
  }
  catch (Exception e) {
    // noop, expecting exceptions
  }
  finally {
    scanner.close();
    reader.close();
  }
  closeOutput();
}

源代码9 项目： hadoop 文件： TestTFileSeek.java

public void seekTFile() throws IOException {
  int miss = 0;
  long totalBytes = 0;
  FSDataInputStream fsdis = fs.open(path);
  Reader reader =
    new Reader(fsdis, fs.getFileStatus(path).getLen(), conf);
  KeySampler kSampler =
      new KeySampler(rng, reader.getFirstKey(), reader.getLastKey(),
          keyLenGen);
  Scanner scanner = reader.createScanner();
  BytesWritable key = new BytesWritable();
  BytesWritable val = new BytesWritable();
  timer.reset();
  timer.start();
  for (int i = 0; i < options.seekCount; ++i) {
    kSampler.next(key);
    scanner.lowerBound(key.get(), 0, key.getSize());
    if (!scanner.atEnd()) {
      scanner.entry().get(key, val);
      totalBytes += key.getSize();
      totalBytes += val.getSize();
    }
    else {
      ++miss;
    }
  }
  timer.stop();
  double duration = (double) timer.read() / 1000; // in us.
  System.out.printf(
      "time: %s...avg seek: %s...%d hit...%d miss...avg I/O size: %.2fKB\n",
      timer.toString(), NanoTimer.nanoTimeToString(timer.read()
          / options.seekCount), options.seekCount - miss, miss,
      (double) totalBytes / 1024 / (options.seekCount - miss));

}

源代码10 项目： big-c 文件： TestTFileSeek.java

public void seekTFile() throws IOException {
  int miss = 0;
  long totalBytes = 0;
  FSDataInputStream fsdis = fs.open(path);
  Reader reader =
    new Reader(fsdis, fs.getFileStatus(path).getLen(), conf);
  KeySampler kSampler =
      new KeySampler(rng, reader.getFirstKey(), reader.getLastKey(),
          keyLenGen);
  Scanner scanner = reader.createScanner();
  BytesWritable key = new BytesWritable();
  BytesWritable val = new BytesWritable();
  timer.reset();
  timer.start();
  for (int i = 0; i < options.seekCount; ++i) {
    kSampler.next(key);
    scanner.lowerBound(key.get(), 0, key.getSize());
    if (!scanner.atEnd()) {
      scanner.entry().get(key, val);
      totalBytes += key.getSize();
      totalBytes += val.getSize();
    }
    else {
      ++miss;
    }
  }
  timer.stop();
  double duration = (double) timer.read() / 1000; // in us.
  System.out.printf(
      "time: %s...avg seek: %s...%d hit...%d miss...avg I/O size: %.2fKB\n",
      timer.toString(), NanoTimer.nanoTimeToString(timer.read()
          / options.seekCount), options.seekCount - miss, miss,
      (double) totalBytes / 1024 / (options.seekCount - miss));

}

源代码11 项目： RDFS 文件： TestTFileSeek.java

public void seekTFile() throws IOException {
  int miss = 0;
  long totalBytes = 0;
  FSDataInputStream fsdis = fs.open(path);
  Reader reader =
    new Reader(fsdis, fs.getFileStatus(path).getLen(), conf);
  KeySampler kSampler =
      new KeySampler(rng, reader.getFirstKey(), reader.getLastKey(),
          keyLenGen);
  Scanner scanner = reader.createScanner();
  BytesWritable key = new BytesWritable();
  BytesWritable val = new BytesWritable();
  timer.reset();
  timer.start();
  for (int i = 0; i < options.seekCount; ++i) {
    kSampler.next(key);
    scanner.lowerBound(key.get(), 0, key.getSize());
    if (!scanner.atEnd()) {
      scanner.entry().get(key, val);
      totalBytes += key.getSize();
      totalBytes += val.getSize();
    }
    else {
      ++miss;
    }
  }
  timer.stop();
  double duration = (double) timer.read() / 1000; // in us.
  System.out.printf(
      "time: %s...avg seek: %s...%d hit...%d miss...avg I/O size: %.2fKB\n",
      timer.toString(), NanoTimer.nanoTimeToString(timer.read()
          / options.seekCount), options.seekCount - miss, miss,
      (double) totalBytes / 1024 / (options.seekCount - miss));

}

源代码12 项目： hadoop-gpu 文件： TestTFileSeek.java

public void seekTFile() throws IOException {
  int miss = 0;
  long totalBytes = 0;
  FSDataInputStream fsdis = fs.open(path);
  Reader reader =
    new Reader(fsdis, fs.getFileStatus(path).getLen(), conf);
  KeySampler kSampler =
      new KeySampler(rng, reader.getFirstKey(), reader.getLastKey(),
          keyLenGen);
  Scanner scanner = reader.createScanner();
  BytesWritable key = new BytesWritable();
  BytesWritable val = new BytesWritable();
  timer.reset();
  timer.start();
  for (int i = 0; i < options.seekCount; ++i) {
    kSampler.next(key);
    scanner.lowerBound(key.get(), 0, key.getSize());
    if (!scanner.atEnd()) {
      scanner.entry().get(key, val);
      totalBytes += key.getSize();
      totalBytes += val.getSize();
    }
    else {
      ++miss;
    }
  }
  timer.stop();
  double duration = (double) timer.read() / 1000; // in us.
  System.out.printf(
      "time: %s...avg seek: %s...%d hit...%d miss...avg I/O size: %.2fKB\n",
      timer.toString(), NanoTimer.nanoTimeToString(timer.read()
          / options.seekCount), options.seekCount - miss, miss,
      (double) totalBytes / 1024 / (options.seekCount - miss));

}

源代码13 项目： hadoop 文件： TestTFile.java

/**
 * test none codecs
 */
void basicWithSomeCodec(String codec) throws IOException {
  Path ncTFile = new Path(ROOT, "basic.tfile");
  FSDataOutputStream fout = createFSOutput(ncTFile);
  Writer writer = new Writer(fout, minBlockSize, codec, "memcmp", conf);
  writeRecords(writer);
  fout.close();
  FSDataInputStream fin = fs.open(ncTFile);
  Reader reader =
      new Reader(fs.open(ncTFile), fs.getFileStatus(ncTFile).getLen(), conf);

  Scanner scanner = reader.createScanner();
  readAllRecords(scanner);
  scanner.seekTo(getSomeKey(50));
  assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)));
  // read the key and see if it matches
  byte[] readKey = readKey(scanner);
  assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
      readKey));

  scanner.seekTo(new byte[0]);
  byte[] val1 = readValue(scanner);
  scanner.seekTo(new byte[0]);
  byte[] val2 = readValue(scanner);
  assertTrue(Arrays.equals(val1, val2));
  
  // check for lowerBound
  scanner.lowerBound(getSomeKey(50));
  assertTrue("locaton lookup failed", scanner.currentLocation
      .compareTo(reader.end()) < 0);
  readKey = readKey(scanner);
  assertTrue("seeked key does not match", Arrays.equals(readKey,
      getSomeKey(50)));

  // check for upper bound
  scanner.upperBound(getSomeKey(50));
  assertTrue("location lookup failed", scanner.currentLocation
      .compareTo(reader.end()) < 0);
  readKey = readKey(scanner);
  assertTrue("seeked key does not match", Arrays.equals(readKey,
      getSomeKey(51)));

  scanner.close();
  // test for a range of scanner
  scanner = reader.createScannerByKey(getSomeKey(10), getSomeKey(60));
  readAndCheckbytes(scanner, 10, 50);
  assertFalse(scanner.advance());
  scanner.close();
  reader.close();
  fin.close();
  fs.delete(ncTFile, true);
}

源代码14 项目： big-c 文件： TestTFile.java

/**
 * test none codecs
 */
void basicWithSomeCodec(String codec) throws IOException {
  Path ncTFile = new Path(ROOT, "basic.tfile");
  FSDataOutputStream fout = createFSOutput(ncTFile);
  Writer writer = new Writer(fout, minBlockSize, codec, "memcmp", conf);
  writeRecords(writer);
  fout.close();
  FSDataInputStream fin = fs.open(ncTFile);
  Reader reader =
      new Reader(fs.open(ncTFile), fs.getFileStatus(ncTFile).getLen(), conf);

  Scanner scanner = reader.createScanner();
  readAllRecords(scanner);
  scanner.seekTo(getSomeKey(50));
  assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)));
  // read the key and see if it matches
  byte[] readKey = readKey(scanner);
  assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
      readKey));

  scanner.seekTo(new byte[0]);
  byte[] val1 = readValue(scanner);
  scanner.seekTo(new byte[0]);
  byte[] val2 = readValue(scanner);
  assertTrue(Arrays.equals(val1, val2));
  
  // check for lowerBound
  scanner.lowerBound(getSomeKey(50));
  assertTrue("locaton lookup failed", scanner.currentLocation
      .compareTo(reader.end()) < 0);
  readKey = readKey(scanner);
  assertTrue("seeked key does not match", Arrays.equals(readKey,
      getSomeKey(50)));

  // check for upper bound
  scanner.upperBound(getSomeKey(50));
  assertTrue("location lookup failed", scanner.currentLocation
      .compareTo(reader.end()) < 0);
  readKey = readKey(scanner);
  assertTrue("seeked key does not match", Arrays.equals(readKey,
      getSomeKey(51)));

  scanner.close();
  // test for a range of scanner
  scanner = reader.createScannerByKey(getSomeKey(10), getSomeKey(60));
  readAndCheckbytes(scanner, 10, 50);
  assertFalse(scanner.advance());
  scanner.close();
  reader.close();
  fin.close();
  fs.delete(ncTFile, true);
}

源代码15 项目： RDFS 文件： TestTFile.java

/**
 * test none codecs
 */
void basicWithSomeCodec(String codec) throws IOException {
  Path ncTFile = new Path(ROOT, "basic.tfile");
  FSDataOutputStream fout = createFSOutput(ncTFile);
  Writer writer = new Writer(fout, minBlockSize, codec, "memcmp", conf);
  writeRecords(writer);
  fout.close();
  FSDataInputStream fin = fs.open(ncTFile);
  Reader reader =
      new Reader(fs.open(ncTFile), fs.getFileStatus(ncTFile).getLen(), conf);

  Scanner scanner = reader.createScanner();
  readAllRecords(scanner);
  scanner.seekTo(getSomeKey(50));
  assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)));
  // read the key and see if it matches
  byte[] readKey = readKey(scanner);
  assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
      readKey));

  scanner.seekTo(new byte[0]);
  byte[] val1 = readValue(scanner);
  scanner.seekTo(new byte[0]);
  byte[] val2 = readValue(scanner);
  assertTrue(Arrays.equals(val1, val2));
  
  // check for lowerBound
  scanner.lowerBound(getSomeKey(50));
  assertTrue("locaton lookup failed", scanner.currentLocation
      .compareTo(reader.end()) < 0);
  readKey = readKey(scanner);
  assertTrue("seeked key does not match", Arrays.equals(readKey,
      getSomeKey(50)));

  // check for upper bound
  scanner.upperBound(getSomeKey(50));
  assertTrue("location lookup failed", scanner.currentLocation
      .compareTo(reader.end()) < 0);
  readKey = readKey(scanner);
  assertTrue("seeked key does not match", Arrays.equals(readKey,
      getSomeKey(51)));

  scanner.close();
  // test for a range of scanner
  scanner = reader.createScannerByKey(getSomeKey(10), getSomeKey(60));
  readAndCheckbytes(scanner, 10, 50);
  assertFalse(scanner.advance());
  scanner.close();
  reader.close();
  fin.close();
  fs.delete(ncTFile, true);
}

源代码16 项目： hadoop-gpu 文件： TestTFile.java

/**
 * test none codecs
 */
void basicWithSomeCodec(String codec) throws IOException {
  Path ncTFile = new Path(ROOT, "basic.tfile");
  FSDataOutputStream fout = createFSOutput(ncTFile);
  Writer writer = new Writer(fout, minBlockSize, codec, "memcmp", conf);
  writeRecords(writer);
  fout.close();
  FSDataInputStream fin = fs.open(ncTFile);
  Reader reader =
      new Reader(fs.open(ncTFile), fs.getFileStatus(ncTFile).getLen(), conf);

  Scanner scanner = reader.createScanner();
  readAllRecords(scanner);
  scanner.seekTo(getSomeKey(50));
  assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)));
  // read the key and see if it matches
  byte[] readKey = readKey(scanner);
  assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
      readKey));

  scanner.seekTo(new byte[0]);
  byte[] val1 = readValue(scanner);
  scanner.seekTo(new byte[0]);
  byte[] val2 = readValue(scanner);
  assertTrue(Arrays.equals(val1, val2));
  
  // check for lowerBound
  scanner.lowerBound(getSomeKey(50));
  assertTrue("locaton lookup failed", scanner.currentLocation
      .compareTo(reader.end()) < 0);
  readKey = readKey(scanner);
  assertTrue("seeked key does not match", Arrays.equals(readKey,
      getSomeKey(50)));

  // check for upper bound
  scanner.upperBound(getSomeKey(50));
  assertTrue("location lookup failed", scanner.currentLocation
      .compareTo(reader.end()) < 0);
  readKey = readKey(scanner);
  assertTrue("seeked key does not match", Arrays.equals(readKey,
      getSomeKey(51)));

  scanner.close();
  // test for a range of scanner
  scanner = reader.createScanner(getSomeKey(10), getSomeKey(60));
  readAndCheckbytes(scanner, 10, 50);
  assertFalse(scanner.advance());
  scanner.close();
  reader.close();
  fin.close();
  fs.delete(ncTFile, true);
}

方法所在类

org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner