org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner#advance ( )源码实例Demo

下面列出了org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner#advance ( ) 实例代码，或者点击链接到github查看源代码，也可以在右侧发表评论。

源代码1 项目： hadoop 文件： TestTFileByteArrays.java

private void readValueWithoutKey(int recordIndex)
    throws IOException {
  Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);

  Scanner scanner =
      reader.createScannerByKey(composeSortedKey(KEY, recordIndex)
          .getBytes(), null);

  byte[] vbuf1 = new byte[BUF_SIZE];
  int vlen1 = scanner.entry().getValueLength();
  scanner.entry().getValue(vbuf1);
  Assert.assertEquals(new String(vbuf1, 0, vlen1), VALUE + recordIndex);

  if (scanner.advance() && !scanner.atEnd()) {
    byte[] vbuf2 = new byte[BUF_SIZE];
    int vlen2 = scanner.entry().getValueLength();
    scanner.entry().getValue(vbuf2);
    Assert.assertEquals(new String(vbuf2, 0, vlen2), VALUE
        + (recordIndex + 1));
  }

  scanner.close();
  reader.close();
}

源代码2 项目： hadoop-gpu 文件： TestTFileByteArrays.java

private void readValueWithoutKey(int count, int recordIndex)
    throws IOException {
  Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);

  Scanner scanner =
      reader.createScanner(composeSortedKey(KEY, count, recordIndex)
          .getBytes(), null);

  byte[] vbuf1 = new byte[BUF_SIZE];
  int vlen1 = scanner.entry().getValueLength();
  scanner.entry().getValue(vbuf1);
  Assert.assertEquals(new String(vbuf1, 0, vlen1), VALUE + recordIndex);

  if (scanner.advance() && !scanner.atEnd()) {
    byte[] vbuf2 = new byte[BUF_SIZE];
    int vlen2 = scanner.entry().getValueLength();
    scanner.entry().getValue(vbuf2);
    Assert.assertEquals(new String(vbuf2, 0, vlen2), VALUE
        + (recordIndex + 1));
  }

  scanner.close();
  reader.close();
}

源代码3 项目： big-c 文件： TestTFile.java

private int readPrepWithUnknownLength(Scanner scanner, int start, int n)
    throws IOException {
  for (int i = start; i < start; i++) {
    String key = String.format(localFormatter, i);
    byte[] read = readKey(scanner);
    assertTrue("keys not equal", Arrays.equals(key.getBytes(), read));
    try {
      read = readValue(scanner);
      assertTrue(false);
    }
    catch (IOException ie) {
      // should have thrown exception
    }
    String value = "value" + key;
    read = readLongValue(scanner, value.getBytes().length);
    assertTrue("values nto equal", Arrays.equals(read, value.getBytes()));
    scanner.advance();
  }
  return (start + n);
}

源代码4 项目： RDFS 文件： TestTFile.java

private int readPrepWithUnknownLength(Scanner scanner, int start, int n)
    throws IOException {
  for (int i = start; i < start; i++) {
    String key = String.format(localFormatter, i);
    byte[] read = readKey(scanner);
    assertTrue("keys not equal", Arrays.equals(key.getBytes(), read));
    try {
      read = readValue(scanner);
      assertTrue(false);
    }
    catch (IOException ie) {
      // should have thrown exception
    }
    String value = "value" + key;
    read = readLongValue(scanner, value.getBytes().length);
    assertTrue("values nto equal", Arrays.equals(read, value.getBytes()));
    scanner.advance();
  }
  return (start + n);
}

源代码5 项目： hadoop 文件： TestTFile.java

private int readLargeRecords(Scanner scanner, int start, int n)
    throws IOException {
  for (int i = start; i < (start + n); i++) {
    byte[] key = readKey(scanner);
    String keyStr = String.format(localFormatter, i);
    assertTrue("btyes for keys do not match", Arrays.equals(
        keyStr.getBytes(), key));
    scanner.advance();
    key = readKey(scanner);
    assertTrue("btyes for keys do not match", Arrays.equals(
        keyStr.getBytes(), key));
    scanner.advance();
  }
  return (start + n);
}

源代码6 项目： hadoop-gpu 文件： TestTFile.java

private int readLargeRecords(Scanner scanner, int start, int n)
    throws IOException {
  for (int i = start; i < (start + n); i++) {
    byte[] key = readKey(scanner);
    String keyStr = String.format(localFormatter, i);
    assertTrue("btyes for keys do not match", Arrays.equals(
        keyStr.getBytes(), key));
    scanner.advance();
    key = readKey(scanner);
    assertTrue("btyes for keys do not match", Arrays.equals(
        keyStr.getBytes(), key));
    scanner.advance();
  }
  return (start + n);
}

源代码7 项目： hadoop-gpu 文件： TestTFileByteArrays.java

private void readKeyWithoutValue(int count, int recordIndex)
    throws IOException {
  Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Scanner scanner =
      reader.createScanner(composeSortedKey(KEY, count, recordIndex)
          .getBytes(), null);

  try {
    // read the indexed key
    byte[] kbuf1 = new byte[BUF_SIZE];
    int klen1 = scanner.entry().getKeyLength();
    scanner.entry().getKey(kbuf1);
    Assert.assertEquals(new String(kbuf1, 0, klen1), composeSortedKey(KEY,
        count, recordIndex));

    if (scanner.advance() && !scanner.atEnd()) {
      // read the next key following the indexed
      byte[] kbuf2 = new byte[BUF_SIZE];
      int klen2 = scanner.entry().getKeyLength();
      scanner.entry().getKey(kbuf2);
      Assert.assertEquals(new String(kbuf2, 0, klen2), composeSortedKey(KEY,
          count, recordIndex + 1));
    }
  }
  finally {
    scanner.close();
    reader.close();
  }
}

源代码8 项目： hadoop 文件： TestTFileByteArrays.java

static void readRecords(FileSystem fs, Path path, int count,
    Configuration conf) throws IOException {
  Reader reader =
      new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Scanner scanner = reader.createScanner();

  try {
    for (int nx = 0; nx < count; nx++, scanner.advance()) {
      Assert.assertFalse(scanner.atEnd());
      // Assert.assertTrue(scanner.next());

      byte[] kbuf = new byte[BUF_SIZE];
      int klen = scanner.entry().getKeyLength();
      scanner.entry().getKey(kbuf);
      Assert.assertEquals(new String(kbuf, 0, klen), composeSortedKey(KEY,
          nx));

      byte[] vbuf = new byte[BUF_SIZE];
      int vlen = scanner.entry().getValueLength();
      scanner.entry().getValue(vbuf);
      Assert.assertEquals(new String(vbuf, 0, vlen), VALUE + nx);
    }

    Assert.assertTrue(scanner.atEnd());
    Assert.assertFalse(scanner.advance());
  } finally {
    scanner.close();
    reader.close();
  }
}

源代码9 项目： hadoop 文件： TestTFileByteArrays.java

private void readKeyWithoutValue(int recordIndex)
    throws IOException {
  Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Scanner scanner =
      reader.createScannerByKey(composeSortedKey(KEY, recordIndex)
          .getBytes(), null);

  try {
    // read the indexed key
    byte[] kbuf1 = new byte[BUF_SIZE];
    int klen1 = scanner.entry().getKeyLength();
    scanner.entry().getKey(kbuf1);
    Assert.assertEquals(new String(kbuf1, 0, klen1), composeSortedKey(KEY,
        recordIndex));

    if (scanner.advance() && !scanner.atEnd()) {
      // read the next key following the indexed
      byte[] kbuf2 = new byte[BUF_SIZE];
      int klen2 = scanner.entry().getKeyLength();
      scanner.entry().getKey(kbuf2);
      Assert.assertEquals(new String(kbuf2, 0, klen2), composeSortedKey(KEY,
          recordIndex + 1));
    }
  } finally {
    scanner.close();
    reader.close();
  }
}

源代码10 项目： hadoop 文件： TestTFileUnsortedByteArrays.java

public void testScan() throws IOException {
  Reader reader =
      new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Assert.assertFalse(reader.isSorted());
  Assert.assertEquals((int) reader.getEntryCount(), 4);

  Scanner scanner = reader.createScanner();

  try {

    // read key and value
    byte[] kbuf = new byte[BUF_SIZE];
    int klen = scanner.entry().getKeyLength();
    scanner.entry().getKey(kbuf);
    Assert.assertEquals(new String(kbuf, 0, klen), "keyZ");

    byte[] vbuf = new byte[BUF_SIZE];
    int vlen = scanner.entry().getValueLength();
    scanner.entry().getValue(vbuf);
    Assert.assertEquals(new String(vbuf, 0, vlen), "valueZ");

    scanner.advance();

    // now try get value first
    vbuf = new byte[BUF_SIZE];
    vlen = scanner.entry().getValueLength();
    scanner.entry().getValue(vbuf);
    Assert.assertEquals(new String(vbuf, 0, vlen), "valueM");

    kbuf = new byte[BUF_SIZE];
    klen = scanner.entry().getKeyLength();
    scanner.entry().getKey(kbuf);
    Assert.assertEquals(new String(kbuf, 0, klen), "keyM");
  }
  finally {
    scanner.close();
    reader.close();
  }
}

源代码11 项目： hadoop-gpu 文件： TestTFileUnsortedByteArrays.java

public void testScan() throws IOException {
  Reader reader =
      new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Assert.assertFalse(reader.isSorted());
  Assert.assertEquals((int) reader.getEntryCount(), 4);

  Scanner scanner = reader.createScanner();

  try {

    // read key and value
    byte[] kbuf = new byte[BUF_SIZE];
    int klen = scanner.entry().getKeyLength();
    scanner.entry().getKey(kbuf);
    Assert.assertEquals(new String(kbuf, 0, klen), "keyZ");

    byte[] vbuf = new byte[BUF_SIZE];
    int vlen = scanner.entry().getValueLength();
    scanner.entry().getValue(vbuf);
    Assert.assertEquals(new String(vbuf, 0, vlen), "valueZ");

    scanner.advance();

    // now try get value first
    vbuf = new byte[BUF_SIZE];
    vlen = scanner.entry().getValueLength();
    scanner.entry().getValue(vbuf);
    Assert.assertEquals(new String(vbuf, 0, vlen), "valueM");

    kbuf = new byte[BUF_SIZE];
    klen = scanner.entry().getKeyLength();
    scanner.entry().getKey(kbuf);
    Assert.assertEquals(new String(kbuf, 0, klen), "keyM");
  }
  finally {
    scanner.close();
    reader.close();
  }
}

源代码12 项目： big-c 文件： TestTFile.java

private int readPrepWithKnownLength(Scanner scanner, int start, int n)
    throws IOException {
  for (int i = start; i < (start + n); i++) {
    String key = String.format(localFormatter, i);
    byte[] read = readKey(scanner);
    assertTrue("keys not equal", Arrays.equals(key.getBytes(), read));
    String value = "value" + key;
    read = readValue(scanner);
    assertTrue("values not equal", Arrays.equals(value.getBytes(), read));
    scanner.advance();
  }
  return (start + n);
}

源代码13 项目： hadoop-gpu 文件： TestTFile.java

private int readPrepWithKnownLength(Scanner scanner, int start, int n)
    throws IOException {
  for (int i = start; i < (start + n); i++) {
    String key = String.format(localFormatter, i);
    byte[] read = readKey(scanner);
    assertTrue("keys not equal", Arrays.equals(key.getBytes(), read));
    String value = "value" + key;
    read = readValue(scanner);
    assertTrue("values not equal", Arrays.equals(value.getBytes(), read));
    scanner.advance();
  }
  return (start + n);
}

源代码14 项目： big-c 文件： TestTFileByteArrays.java

private void readKeyWithoutValue(int recordIndex)
    throws IOException {
  Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Scanner scanner =
      reader.createScannerByKey(composeSortedKey(KEY, recordIndex)
          .getBytes(), null);

  try {
    // read the indexed key
    byte[] kbuf1 = new byte[BUF_SIZE];
    int klen1 = scanner.entry().getKeyLength();
    scanner.entry().getKey(kbuf1);
    Assert.assertEquals(new String(kbuf1, 0, klen1), composeSortedKey(KEY,
        recordIndex));

    if (scanner.advance() && !scanner.atEnd()) {
      // read the next key following the indexed
      byte[] kbuf2 = new byte[BUF_SIZE];
      int klen2 = scanner.entry().getKeyLength();
      scanner.entry().getKey(kbuf2);
      Assert.assertEquals(new String(kbuf2, 0, klen2), composeSortedKey(KEY,
          recordIndex + 1));
    }
  } finally {
    scanner.close();
    reader.close();
  }
}

源代码15 项目： RDFS 文件： TestTFileUnsortedByteArrays.java

public void testScanRange() throws IOException {
  Reader reader =
      new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Assert.assertFalse(reader.isSorted());
  Assert.assertEquals((int) reader.getEntryCount(), 4);

  Scanner scanner = reader.createScanner();

  try {

    // read key and value
    byte[] kbuf = new byte[BUF_SIZE];
    int klen = scanner.entry().getKeyLength();
    scanner.entry().getKey(kbuf);
    Assert.assertEquals(new String(kbuf, 0, klen), "keyZ");

    byte[] vbuf = new byte[BUF_SIZE];
    int vlen = scanner.entry().getValueLength();
    scanner.entry().getValue(vbuf);
    Assert.assertEquals(new String(vbuf, 0, vlen), "valueZ");

    scanner.advance();

    // now try get value first
    vbuf = new byte[BUF_SIZE];
    vlen = scanner.entry().getValueLength();
    scanner.entry().getValue(vbuf);
    Assert.assertEquals(new String(vbuf, 0, vlen), "valueM");

    kbuf = new byte[BUF_SIZE];
    klen = scanner.entry().getKeyLength();
    scanner.entry().getKey(kbuf);
    Assert.assertEquals(new String(kbuf, 0, klen), "keyM");
  }
  finally {
    scanner.close();
    reader.close();
  }
}

源代码16 项目： big-c 文件： TestTFileUnsortedByteArrays.java

public void testScan() throws IOException {
  Reader reader =
      new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Assert.assertFalse(reader.isSorted());
  Assert.assertEquals((int) reader.getEntryCount(), 4);

  Scanner scanner = reader.createScanner();

  try {

    // read key and value
    byte[] kbuf = new byte[BUF_SIZE];
    int klen = scanner.entry().getKeyLength();
    scanner.entry().getKey(kbuf);
    Assert.assertEquals(new String(kbuf, 0, klen), "keyZ");

    byte[] vbuf = new byte[BUF_SIZE];
    int vlen = scanner.entry().getValueLength();
    scanner.entry().getValue(vbuf);
    Assert.assertEquals(new String(vbuf, 0, vlen), "valueZ");

    scanner.advance();

    // now try get value first
    vbuf = new byte[BUF_SIZE];
    vlen = scanner.entry().getValueLength();
    scanner.entry().getValue(vbuf);
    Assert.assertEquals(new String(vbuf, 0, vlen), "valueM");

    kbuf = new byte[BUF_SIZE];
    klen = scanner.entry().getKeyLength();
    scanner.entry().getKey(kbuf);
    Assert.assertEquals(new String(kbuf, 0, klen), "keyM");
  }
  finally {
    scanner.close();
    reader.close();
  }
}

源代码17 项目： hadoop-gpu 文件： TestTFileUnsortedByteArrays.java

public void testScanRange() throws IOException {
  Reader reader =
      new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Assert.assertFalse(reader.isSorted());
  Assert.assertEquals((int) reader.getEntryCount(), 4);

  Scanner scanner = reader.createScanner();

  try {

    // read key and value
    byte[] kbuf = new byte[BUF_SIZE];
    int klen = scanner.entry().getKeyLength();
    scanner.entry().getKey(kbuf);
    Assert.assertEquals(new String(kbuf, 0, klen), "keyZ");

    byte[] vbuf = new byte[BUF_SIZE];
    int vlen = scanner.entry().getValueLength();
    scanner.entry().getValue(vbuf);
    Assert.assertEquals(new String(vbuf, 0, vlen), "valueZ");

    scanner.advance();

    // now try get value first
    vbuf = new byte[BUF_SIZE];
    vlen = scanner.entry().getValueLength();
    scanner.entry().getValue(vbuf);
    Assert.assertEquals(new String(vbuf, 0, vlen), "valueM");

    kbuf = new byte[BUF_SIZE];
    klen = scanner.entry().getKeyLength();
    scanner.entry().getKey(kbuf);
    Assert.assertEquals(new String(kbuf, 0, klen), "keyM");
  }
  finally {
    scanner.close();
    reader.close();
  }
}

源代码18 项目： RDFS 文件： TestTFileUnsortedByteArrays.java

public void testScan() throws IOException {
  Reader reader =
      new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Assert.assertFalse(reader.isSorted());
  Assert.assertEquals((int) reader.getEntryCount(), 4);

  Scanner scanner = reader.createScanner();

  try {

    // read key and value
    byte[] kbuf = new byte[BUF_SIZE];
    int klen = scanner.entry().getKeyLength();
    scanner.entry().getKey(kbuf);
    Assert.assertEquals(new String(kbuf, 0, klen), "keyZ");

    byte[] vbuf = new byte[BUF_SIZE];
    int vlen = scanner.entry().getValueLength();
    scanner.entry().getValue(vbuf);
    Assert.assertEquals(new String(vbuf, 0, vlen), "valueZ");

    scanner.advance();

    // now try get value first
    vbuf = new byte[BUF_SIZE];
    vlen = scanner.entry().getValueLength();
    scanner.entry().getValue(vbuf);
    Assert.assertEquals(new String(vbuf, 0, vlen), "valueM");

    kbuf = new byte[BUF_SIZE];
    klen = scanner.entry().getKeyLength();
    scanner.entry().getKey(kbuf);
    Assert.assertEquals(new String(kbuf, 0, klen), "keyM");
  }
  finally {
    scanner.close();
    reader.close();
  }
}

源代码19 项目： RDFS 文件： TestTFile.java

private int readPrepWithKnownLength(Scanner scanner, int start, int n)
    throws IOException {
  for (int i = start; i < (start + n); i++) {
    String key = String.format(localFormatter, i);
    byte[] read = readKey(scanner);
    assertTrue("keys not equal", Arrays.equals(key.getBytes(), read));
    String value = "value" + key;
    read = readValue(scanner);
    assertTrue("values not equal", Arrays.equals(value.getBytes(), read));
    scanner.advance();
  }
  return (start + n);
}

源代码20 项目： RDFS 文件： TestTFileByteArrays.java

static void readRecords(FileSystem fs, Path path, int count,
    Configuration conf) throws IOException {
  Reader reader =
      new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
  Scanner scanner = reader.createScanner();

  try {
    for (int nx = 0; nx < count; nx++, scanner.advance()) {
      Assert.assertFalse(scanner.atEnd());
      // Assert.assertTrue(scanner.next());

      byte[] kbuf = new byte[BUF_SIZE];
      int klen = scanner.entry().getKeyLength();
      scanner.entry().getKey(kbuf);
      Assert.assertEquals(new String(kbuf, 0, klen), composeSortedKey(KEY,
          nx));

      byte[] vbuf = new byte[BUF_SIZE];
      int vlen = scanner.entry().getValueLength();
      scanner.entry().getValue(vbuf);
      Assert.assertEquals(new String(vbuf, 0, vlen), VALUE + nx);
    }

    Assert.assertTrue(scanner.atEnd());
    Assert.assertFalse(scanner.advance());
  } finally {
    scanner.close();
    reader.close();
  }
}

方法所在类

org.apache.hadoop.io.file.tfile.TFile.Reader.Scanner