下面列出了怎么用org.apache.hadoop.io.compress.Lz4Codec的API类实例代码及写法，或者点击链接到github查看源代码。
@Test
public void testNativeCodeLoaded() {
if (requireTestJni() == false) {
LOG.info("TestNativeCodeLoader: libhadoop.so testing is not required.");
return;
}
if (!NativeCodeLoader.isNativeCodeLoaded()) {
fail("TestNativeCodeLoader: libhadoop.so testing was required, but " +
"libhadoop.so was not loaded.");
}
assertFalse(NativeCodeLoader.getLibraryName().isEmpty());
// library names are depended on platform and build envs
// so just check names are available
assertFalse(ZlibFactory.getLibraryName().isEmpty());
if (NativeCodeLoader.buildSupportsSnappy()) {
assertFalse(SnappyCodec.getLibraryName().isEmpty());
}
if (NativeCodeLoader.buildSupportsOpenssl()) {
assertFalse(OpensslCipher.getLibraryName().isEmpty());
}
assertFalse(Lz4Codec.getLibraryName().isEmpty());
LOG.info("TestNativeCodeLoader: libhadoop.so is loaded.");
}
@Test
public void testNativeCodeLoaded() {
if (requireTestJni() == false) {
LOG.info("TestNativeCodeLoader: libhadoop.so testing is not required.");
return;
}
if (!NativeCodeLoader.isNativeCodeLoaded()) {
fail("TestNativeCodeLoader: libhadoop.so testing was required, but " +
"libhadoop.so was not loaded.");
}
assertFalse(NativeCodeLoader.getLibraryName().isEmpty());
// library names are depended on platform and build envs
// so just check names are available
assertFalse(ZlibFactory.getLibraryName().isEmpty());
if (NativeCodeLoader.buildSupportsSnappy()) {
assertFalse(SnappyCodec.getLibraryName().isEmpty());
}
if (NativeCodeLoader.buildSupportsOpenssl()) {
assertFalse(OpensslCipher.getLibraryName().isEmpty());
}
assertFalse(Lz4Codec.getLibraryName().isEmpty());
LOG.info("TestNativeCodeLoader: libhadoop.so is loaded.");
}
@Override
public String toString() {
switch (this) {
case NONE: return "NONE";
case DEFAULT: return DefaultCodec.class.getName();
case BZIP: return BZip2Codec.class.getName();
case GZIP: return GzipCodec.class.getName();
case LZ4: return Lz4Codec.class.getName();
case SNAPPY: return SnappyCodec.class.getName();
case AUTOMATIC: return "Automatically Detected";
}
return null;
}
public HdfsSink2(Hdfs2SinkConfig config)
throws ClassNotFoundException
{
this.batchSize = config.getBatchBufferSize();
this.writerDir = config.getWriteDir();
switch (config.getZipType().trim().toLowerCase()) {
case "lzo":
codecClass = (Class<? extends CompressionCodec>) Class.forName("com.hadoop.compression.lzo.LzopCodec");
break;
case "lz4":
codecClass = Lz4Codec.class;
break;
case "snappy":
codecClass = SnappyCodec.class;
break;
case "gzip":
codecClass = GzipCodec.class;
break;
case "bzip2":
codecClass = BZip2Codec.class;
break;
case "default":
codecClass = DefaultCodec.class;
break;
default:
codecClass = NoneCodec.class;
}
}
private FileChannel createOutputStream(String rowKey, TextTimeParser timeParser, long split)
{
Configuration hadoopConf = new Configuration();
CompressionCodec codec = ReflectionUtils.newInstance(Lz4Codec.class, hadoopConf);
String outputPath = this.writeTableDir + timeParser.getPartitionPath() + "_partition_" + this.partition + "_split" + split + codec.getDefaultExtension();
logger.info("create {} text file {}", rowKey, outputPath);
try {
FileChannel fileChannel = new FileChannel(outputPath, split, codec, hadoopConf);
return fileChannel;
}
catch (IOException var11) {
throw new RuntimeException("textFile " + outputPath + " writer create failed", var11);
}
}
/**
* Used for returning the compression kind used in ORC
*
* @param codec
* @return
*/
private CompressionKind resolveCompression(CompressionCodec codec) {
if (codec instanceof Lz4Codec)
return CompressionKind.LZ4;
else if (codec instanceof SnappyCodec)
return CompressionKind.SNAPPY;
// although GZip and ZLIB are not same thing
// there is no better named codec for this case,
// use hadoop Gzip codec to enable ORC ZLIB compression
else if (codec instanceof GzipCodec)
return CompressionKind.ZLIB;
else
return CompressionKind.NONE;
}
@Override
Optional<String> getCodecName()
{
return Optional.of(Lz4Codec.class.getName());
}
/**
* A tool to test native library availability,
*/
public static void main(String[] args) {
String usage = "NativeLibraryChecker [-a|-h]\n"
+ " -a use -a to check all libraries are available\n"
+ " by default just check hadoop library (and\n"
+ " winutils.exe on Windows OS) is available\n"
+ " exit with error code 1 if check failed\n"
+ " -h print this message\n";
if (args.length > 1 ||
(args.length == 1 &&
!(args[0].equals("-a") || args[0].equals("-h")))) {
System.err.println(usage);
ExitUtil.terminate(1);
}
boolean checkAll = false;
if (args.length == 1) {
if (args[0].equals("-h")) {
System.out.println(usage);
return;
}
checkAll = true;
}
Configuration conf = new Configuration();
boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded();
boolean zlibLoaded = false;
boolean snappyLoaded = false;
// lz4 is linked within libhadoop
boolean lz4Loaded = nativeHadoopLoaded;
boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
boolean openSslLoaded = false;
boolean winutilsExists = false;
String openSslDetail = "";
String hadoopLibraryName = "";
String zlibLibraryName = "";
String snappyLibraryName = "";
String lz4LibraryName = "";
String bzip2LibraryName = "";
String winutilsPath = null;
if (nativeHadoopLoaded) {
hadoopLibraryName = NativeCodeLoader.getLibraryName();
zlibLoaded = ZlibFactory.isNativeZlibLoaded(conf);
if (zlibLoaded) {
zlibLibraryName = ZlibFactory.getLibraryName();
}
snappyLoaded = NativeCodeLoader.buildSupportsSnappy() &&
SnappyCodec.isNativeCodeLoaded();
if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) {
snappyLibraryName = SnappyCodec.getLibraryName();
}
if (OpensslCipher.getLoadingFailureReason() != null) {
openSslDetail = OpensslCipher.getLoadingFailureReason();
openSslLoaded = false;
} else {
openSslDetail = OpensslCipher.getLibraryName();
openSslLoaded = true;
}
if (lz4Loaded) {
lz4LibraryName = Lz4Codec.getLibraryName();
}
if (bzip2Loaded) {
bzip2LibraryName = Bzip2Factory.getLibraryName(conf);
}
}
// winutils.exe is required on Windows
winutilsPath = Shell.getWinUtilsPath();
if (winutilsPath != null) {
winutilsExists = true;
} else {
winutilsPath = "";
}
System.out.println("Native library checking:");
System.out.printf("hadoop: %b %s%n", nativeHadoopLoaded, hadoopLibraryName);
System.out.printf("zlib: %b %s%n", zlibLoaded, zlibLibraryName);
System.out.printf("snappy: %b %s%n", snappyLoaded, snappyLibraryName);
System.out.printf("lz4: %b %s%n", lz4Loaded, lz4LibraryName);
System.out.printf("bzip2: %b %s%n", bzip2Loaded, bzip2LibraryName);
System.out.printf("openssl: %b %s%n", openSslLoaded, openSslDetail);
if (Shell.WINDOWS) {
System.out.printf("winutils: %b %s%n", winutilsExists, winutilsPath);
}
if ((!nativeHadoopLoaded) || (Shell.WINDOWS && (!winutilsExists)) ||
(checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) {
// return 1 to indicated check failed
ExitUtil.terminate(1);
}
}
@Before
public void before() {
assumeTrue(Lz4Codec.isNativeCodeLoaded());
}
/**
* A tool to test native library availability,
*/
public static void main(String[] args) {
String usage = "NativeLibraryChecker [-a|-h]\n"
+ " -a use -a to check all libraries are available\n"
+ " by default just check hadoop library (and\n"
+ " winutils.exe on Windows OS) is available\n"
+ " exit with error code 1 if check failed\n"
+ " -h print this message\n";
if (args.length > 1 ||
(args.length == 1 &&
!(args[0].equals("-a") || args[0].equals("-h")))) {
System.err.println(usage);
ExitUtil.terminate(1);
}
boolean checkAll = false;
if (args.length == 1) {
if (args[0].equals("-h")) {
System.out.println(usage);
return;
}
checkAll = true;
}
Configuration conf = new Configuration();
boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded();
boolean zlibLoaded = false;
boolean snappyLoaded = false;
// lz4 is linked within libhadoop
boolean lz4Loaded = nativeHadoopLoaded;
boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
boolean openSslLoaded = false;
boolean winutilsExists = false;
String openSslDetail = "";
String hadoopLibraryName = "";
String zlibLibraryName = "";
String snappyLibraryName = "";
String lz4LibraryName = "";
String bzip2LibraryName = "";
String winutilsPath = null;
if (nativeHadoopLoaded) {
hadoopLibraryName = NativeCodeLoader.getLibraryName();
zlibLoaded = ZlibFactory.isNativeZlibLoaded(conf);
if (zlibLoaded) {
zlibLibraryName = ZlibFactory.getLibraryName();
}
snappyLoaded = NativeCodeLoader.buildSupportsSnappy() &&
SnappyCodec.isNativeCodeLoaded();
if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) {
snappyLibraryName = SnappyCodec.getLibraryName();
}
if (OpensslCipher.getLoadingFailureReason() != null) {
openSslDetail = OpensslCipher.getLoadingFailureReason();
openSslLoaded = false;
} else {
openSslDetail = OpensslCipher.getLibraryName();
openSslLoaded = true;
}
if (lz4Loaded) {
lz4LibraryName = Lz4Codec.getLibraryName();
}
if (bzip2Loaded) {
bzip2LibraryName = Bzip2Factory.getLibraryName(conf);
}
}
// winutils.exe is required on Windows
winutilsPath = Shell.getWinUtilsPath();
if (winutilsPath != null) {
winutilsExists = true;
} else {
winutilsPath = "";
}
System.out.println("Native library checking:");
System.out.printf("hadoop: %b %s%n", nativeHadoopLoaded, hadoopLibraryName);
System.out.printf("zlib: %b %s%n", zlibLoaded, zlibLibraryName);
System.out.printf("snappy: %b %s%n", snappyLoaded, snappyLibraryName);
System.out.printf("lz4: %b %s%n", lz4Loaded, lz4LibraryName);
System.out.printf("bzip2: %b %s%n", bzip2Loaded, bzip2LibraryName);
System.out.printf("openssl: %b %s%n", openSslLoaded, openSslDetail);
if (Shell.WINDOWS) {
System.out.printf("winutils: %b %s%n", winutilsExists, winutilsPath);
}
if ((!nativeHadoopLoaded) || (Shell.WINDOWS && (!winutilsExists)) ||
(checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) {
// return 1 to indicated check failed
ExitUtil.terminate(1);
}
}
@Before
public void before() {
assumeTrue(Lz4Codec.isNativeCodeLoaded());
}