下面列出了怎么用java.util.zip.CheckedOutputStream的API类实例代码及写法,或者点击链接到github查看源代码。
/**
* 压缩文件或目录
*
* @param fromPath 待压缩文件或路径
* @param toPath 压缩文件,如 xx.zip
*/
public static void compress(String fromPath, String toPath) throws IOException {
File fromFile = new File(fromPath);
File toFile = new File(toPath);
if (!fromFile.exists()) {
throw new FileNotFoundException(fromPath + "不存在!");
}
try (
FileOutputStream outputStream = new FileOutputStream(toFile);
CheckedOutputStream checkedOutputStream = new CheckedOutputStream(outputStream, new CRC32());
ZipOutputStream zipOutputStream = new ZipOutputStream(checkedOutputStream)
) {
String baseDir = "";
compress(fromFile, zipOutputStream, baseDir);
}
}
public static boolean compress(String srcFilePath, String destFilePath) {
File src = new File(srcFilePath);
if (!src.exists()) {
throw new RuntimeException(srcFilePath + "not exist");
}
File zipFile = new File(destFilePath);
try {
FileOutputStream fos = new FileOutputStream(zipFile);
CheckedOutputStream cos = new CheckedOutputStream(fos, new CRC32());
ZipOutputStream zos = new ZipOutputStream(cos);
String baseDir = "";
compressbyType(src, zos, baseDir);
zos.close();
return true;
} catch (Exception e) {
LOGGER.error("", e);
}
return false;
}
static byte[] getManifestAsBytes(int nchars) throws IOException {
crc.reset();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(baos, crc);
PrintStream ps = new PrintStream(cos);
ps.println("Manifest-Version: 1.0");
ps.print("Main-Class: ");
for (int i = 0 ; i < nchars - SOME_KLASS.length() ; i++) {
ps.print(i%10);
}
ps.println(SOME_KLASS);
cos.flush();
cos.close();
ps.close();
return baos.toByteArray();
}
static byte[] getManifestAsBytes(int nchars) throws IOException {
crc.reset();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(baos, crc);
PrintStream ps = new PrintStream(cos);
ps.println("Manifest-Version: 1.0");
ps.print("Main-Class: ");
for (int i = 0 ; i < nchars - SOME_KLASS.length() ; i++) {
ps.print(i%10);
}
ps.println(SOME_KLASS);
cos.flush();
cos.close();
ps.close();
return baos.toByteArray();
}
static byte[] getManifestAsBytes(int nchars) throws IOException {
crc.reset();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(baos, crc);
PrintStream ps = new PrintStream(cos);
ps.println("Manifest-Version: 1.0");
ps.print("Main-Class: ");
for (int i = 0 ; i < nchars - SOME_KLASS.length() ; i++) {
ps.print(i%10);
}
ps.println(SOME_KLASS);
cos.flush();
cos.close();
ps.close();
return baos.toByteArray();
}
static byte[] getManifestAsBytes(int nchars) throws IOException {
crc.reset();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(baos, crc);
PrintStream ps = new PrintStream(cos);
ps.println("Manifest-Version: 1.0");
ps.print("Main-Class: ");
for (int i = 0 ; i < nchars - SOME_KLASS.length() ; i++) {
ps.print(i%10);
}
ps.println(SOME_KLASS);
cos.flush();
cos.close();
ps.close();
return baos.toByteArray();
}
/**
* Compressed file or directory
*
* @param srcPath The address of the file or folder
* @param zipFilePath The address of the compressed package
*/
public static void zipFiles(String srcPath,String zipFilePath) {
File file = new File(srcPath);
if (!file.exists())
throw new RuntimeException(srcPath + "not exist!");
try {
FileOutputStream fileOutputStream = new FileOutputStream(zipFilePath);
CheckedOutputStream cos = new CheckedOutputStream(fileOutputStream,
new CRC32());
ZipOutputStream out = new ZipOutputStream(cos);
String baseDir="";
zip(file,out,baseDir);
out.close();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
static byte[] getManifestAsBytes(int nchars) throws IOException {
crc.reset();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(baos, crc);
PrintStream ps = new PrintStream(cos);
ps.println("Manifest-Version: 1.0");
ps.print("Main-Class: ");
for (int i = 0 ; i < nchars - SOME_KLASS.length() ; i++) {
ps.print(i%10);
}
ps.println(SOME_KLASS);
cos.flush();
cos.close();
ps.close();
return baos.toByteArray();
}
static byte[] getManifestAsBytes(int nchars) throws IOException {
crc.reset();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(baos, crc);
PrintStream ps = new PrintStream(cos);
ps.println("Manifest-Version: 1.0");
ps.print("Main-Class: ");
for (int i = 0 ; i < nchars - SOME_KLASS.length() ; i++) {
ps.print(i%10);
}
ps.println(SOME_KLASS);
cos.flush();
cos.close();
ps.close();
return baos.toByteArray();
}
private static void createIndexFile(File indexFile, Configuration conf)
throws IOException {
if (indexFile.exists()) {
System.out.println("Deleting existing file");
indexFile.delete();
}
indexFile.createNewFile();
FSDataOutputStream output = FileSystem.getLocal(conf).getRaw().append(
new Path(indexFile.getAbsolutePath()));
Checksum crc = new PureJavaCrc32();
crc.reset();
CheckedOutputStream chk = new CheckedOutputStream(output, crc);
String msg = "Writing new index file. This file will be used only " +
"for the testing.";
chk.write(Arrays.copyOf(msg.getBytes(),
MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH));
output.writeLong(chk.getChecksum().getValue());
output.close();
}
static byte[] getManifestAsBytes(int nchars) throws IOException {
crc.reset();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(baos, crc);
PrintStream ps = new PrintStream(cos);
ps.println("Manifest-Version: 1.0");
ps.print("Main-Class: ");
for (int i = 0 ; i < nchars - SOME_KLASS.length() ; i++) {
ps.print(i%10);
}
ps.println(SOME_KLASS);
cos.flush();
cos.close();
ps.close();
return baos.toByteArray();
}
static byte[] getManifestAsBytes(int nchars) throws IOException {
crc.reset();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(baos, crc);
PrintStream ps = new PrintStream(cos);
ps.println("Manifest-Version: 1.0");
ps.print("Main-Class: ");
for (int i = 0 ; i < nchars - SOME_KLASS.length() ; i++) {
ps.print(i%10);
}
ps.println(SOME_KLASS);
cos.flush();
cos.close();
ps.close();
return baos.toByteArray();
}
static byte[] getManifestAsBytes(int nchars) throws IOException {
crc.reset();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(baos, crc);
PrintStream ps = new PrintStream(cos);
ps.println("Manifest-Version: 1.0");
ps.print("Main-Class: ");
for (int i = 0 ; i < nchars - SOME_KLASS.length() ; i++) {
ps.print(i%10);
}
ps.println(SOME_KLASS);
cos.flush();
cos.close();
ps.close();
return baos.toByteArray();
}
private static void createIndexFile(File indexFile, Configuration conf)
throws IOException {
if (indexFile.exists()) {
System.out.println("Deleting existing file");
indexFile.delete();
}
indexFile.createNewFile();
FSDataOutputStream output = FileSystem.getLocal(conf).getRaw().append(
new Path(indexFile.getAbsolutePath()));
Checksum crc = new PureJavaCrc32();
crc.reset();
CheckedOutputStream chk = new CheckedOutputStream(output, crc);
String msg = "Writing new index file. This file will be used only " +
"for the testing.";
chk.write(Arrays.copyOf(msg.getBytes(),
MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH));
output.writeLong(chk.getChecksum().getValue());
output.close();
}
static byte[] getManifestAsBytes(int nchars) throws IOException {
crc.reset();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(baos, crc);
PrintStream ps = new PrintStream(cos);
ps.println("Manifest-Version: 1.0");
ps.print("Main-Class: ");
for (int i = 0 ; i < nchars - SOME_KLASS.length() ; i++) {
ps.print(i%10);
}
ps.println(SOME_KLASS);
cos.flush();
cos.close();
ps.close();
return baos.toByteArray();
}
static byte[] getManifestAsBytes(int nchars) throws IOException {
crc.reset();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(baos, crc);
PrintStream ps = new PrintStream(cos);
ps.println("Manifest-Version: 1.0");
ps.print("Main-Class: ");
for (int i = 0 ; i < nchars - SOME_KLASS.length() ; i++) {
ps.print(i%10);
}
ps.println(SOME_KLASS);
cos.flush();
cos.close();
ps.close();
return baos.toByteArray();
}
public void encode(OutputStream out) throws IOException {
java.util.zip.CRC32 crc32 = new java.util.zip.CRC32();
CheckedOutputStream outChecked = new CheckedOutputStream(out, crc32);
// Index Indicator
outChecked.write(0x00);
// Number of Records
EncoderUtil.encodeVLI(outChecked, recordCount);
// List of Records
for (IndexRecord record : records) {
EncoderUtil.encodeVLI(outChecked, record.unpadded);
EncoderUtil.encodeVLI(outChecked, record.uncompressed);
}
// Index Padding
for (int i = getIndexPaddingSize(); i > 0; --i)
outChecked.write(0x00);
// CRC32
long value = crc32.getValue();
for (int i = 0; i < 4; ++i)
out.write((byte)(value >>> (i * 8)));
}
static byte[] getManifestAsBytes(int nchars) throws IOException {
crc.reset();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(baos, crc);
PrintStream ps = new PrintStream(cos);
ps.println("Manifest-Version: 1.0");
ps.print("Main-Class: ");
for (int i = 0 ; i < nchars - SOME_KLASS.length() ; i++) {
ps.print(i%10);
}
ps.println(SOME_KLASS);
cos.flush();
cos.close();
ps.close();
return baos.toByteArray();
}
static byte[] getManifestAsBytes(int nchars) throws IOException {
crc.reset();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
CheckedOutputStream cos = new CheckedOutputStream(baos, crc);
PrintStream ps = new PrintStream(cos);
ps.println("Manifest-Version: 1.0");
ps.print("Main-Class: ");
for (int i = 0 ; i < nchars - SOME_KLASS.length() ; i++) {
ps.print(i%10);
}
ps.println(SOME_KLASS);
cos.flush();
cos.close();
ps.close();
return baos.toByteArray();
}
public void encode(OutputStream out) throws IOException {
java.util.zip.CRC32 crc32 = new java.util.zip.CRC32();
CheckedOutputStream outChecked = new CheckedOutputStream(out, crc32);
// Index Indicator
outChecked.write(0x00);
// Number of Records
EncoderUtil.encodeVLI(outChecked, recordCount);
// List of Records
for (Iterator i = records.iterator(); i.hasNext(); ) {
IndexRecord record = (IndexRecord)i.next();
EncoderUtil.encodeVLI(outChecked, record.unpadded);
EncoderUtil.encodeVLI(outChecked, record.uncompressed);
}
// Index Padding
for (int i = getIndexPaddingSize(); i > 0; --i)
outChecked.write(0x00);
// CRC32
long value = crc32.getValue();
for (int i = 0; i < 4; ++i)
out.write((byte)(value >>> (i * 8)));
}
private void flushBuffer() throws IOException {
final UnsafeByteArrayOutputStream compressedBuffer = new UnsafeByteArrayOutputStream(blockSize+4*numRecords);
final CheckedOutputStream checksumStream = new CheckedOutputStream(compressedBuffer, new Adler32());
final DataOutputStream compressorStream = new DataOutputStream(codec.createOutputStream(checksumStream));
compressorStream.writeInt(numRecords);
for (int i = 0; i < numRecords; i++) {
VIntUtils.writeVInt((OutputStream)compressorStream, lengthBuffer[i]);
}
compressorStream.write(currentBlockBytes.getByteArray(), 0, currentBlockBytes.size());
compressorStream.close();
out.writeInt(compressedBuffer.size());
final int checksum = (int)checksumStream.getChecksum().getValue();
out.writeInt(checksum);
out.write(compressedBuffer.getByteArray(), 0, compressedBuffer.size());
currentBlockBytes.reset();
numRecords = 0;
final int padLength = (int)(pad-out.position()%pad);
if (padLength != pad) {
for (int i = 0; i < padLength; i++) {
out.writeByte(0);
}
}
blockAddress = out.position()<<shift;
}
@Test
public void testChecksumStreams() throws IOException {
byte[] content = new byte[33333];
new Random().nextBytes(content);
Murmur3F murmur3F = new Murmur3F();
murmur3F.update(content);
String hash = murmur3F.getValueHexString();
murmur3F.reset();
CheckedOutputStream out = new CheckedOutputStream(new ByteArrayOutputStream(), murmur3F);
out.write(content);
Assert.assertEquals(hash, murmur3F.getValueHexString());
murmur3F.reset();
CheckedInputStream in = new CheckedInputStream(new ByteArrayInputStream(content), murmur3F);
IoUtils.readAllBytes(in);
Assert.assertEquals(hash, murmur3F.getValueHexString());
}
/**
* Compress a text file using the ZIP compressing algorithm.
*
* @param filename the path to the file to be compressed
*/
public static void zipCompress(String filename) throws IOException {
FileOutputStream fos = new FileOutputStream(filename + COMPRESSION_SUFFIX);
CheckedOutputStream csum = new CheckedOutputStream(fos, new CRC32());
ZipOutputStream out = new ZipOutputStream(new BufferedOutputStream(csum));
out.setComment("Failmon records.");
BufferedReader in = new BufferedReader(new FileReader(filename));
out.putNextEntry(new ZipEntry(new File(filename).getName()));
int c;
while ((c = in.read()) != -1)
out.write(c);
in.close();
out.finish();
out.close();
}
/**
* Compress a text file using the ZIP compressing algorithm.
*
* @param filename the path to the file to be compressed
*/
public static void zipCompress(String filename) throws IOException {
FileOutputStream fos = new FileOutputStream(filename + COMPRESSION_SUFFIX);
CheckedOutputStream csum = new CheckedOutputStream(fos, new CRC32());
ZipOutputStream out = new ZipOutputStream(new BufferedOutputStream(csum));
out.setComment("Failmon records.");
BufferedReader in = new BufferedReader(new FileReader(filename));
out.putNextEntry(new ZipEntry(new File(filename).getName()));
int c;
while ((c = in.read()) != -1)
out.write(c);
in.close();
out.finish();
out.close();
}
/**
* Create a {@link CompressedXContent} out of a {@link ToXContent} instance.
*/
public CompressedXContent(ToXContent xcontent, XContentType type, ToXContent.Params params) throws IOException {
BytesStreamOutput bStream = new BytesStreamOutput();
OutputStream compressedStream = CompressorFactory.COMPRESSOR.streamOutput(bStream);
CRC32 crc32 = new CRC32();
try (OutputStream checkedStream = new CheckedOutputStream(compressedStream, crc32)) {
try (XContentBuilder builder = XContentFactory.contentBuilder(type, checkedStream)) {
builder.startObject();
xcontent.toXContent(builder, params);
builder.endObject();
}
}
this.bytes = BytesReference.toBytes(bStream.bytes());
this.crc32 = (int) crc32.getValue();
assertConsistent();
}
public static void compress(final String rootDir, final String sourceDir, final String outputFile,
final Checksum checksum) throws IOException {
try (final FileOutputStream fos = new FileOutputStream(outputFile);
final CheckedOutputStream cos = new CheckedOutputStream(fos, checksum);
final ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(cos))) {
ZipUtil.compressDirectoryToZipFile(rootDir, sourceDir, zos);
zos.flush();
fos.getFD().sync();
}
}
/**
* compress
*
* @param srcFile
* @param destFile
* @throws Exception
*/
public static void compress(File srcFile, File destFile) throws Exception {
// CRC32 check
CheckedOutputStream cos = new CheckedOutputStream(new FileOutputStream(
destFile), new CRC32());
ZipOutputStream zos = new ZipOutputStream(cos);
zos.setComment(new String("comment"));
compress(srcFile, zos, BASE_DIR);
zos.flush();
zos.close();
}
/**
* Create a {@link CompressedXContent} out of a {@link ToXContent} instance.
*/
public CompressedXContent(ToXContent xcontent, XContentType type, ToXContent.Params params) throws IOException {
BytesStreamOutput bStream = new BytesStreamOutput();
OutputStream compressedStream = CompressorFactory.defaultCompressor().streamOutput(bStream);
CRC32 crc32 = new CRC32();
OutputStream checkedStream = new CheckedOutputStream(compressedStream, crc32);
try (XContentBuilder builder = XContentFactory.contentBuilder(type, checkedStream)) {
builder.startObject();
xcontent.toXContent(builder, params);
builder.endObject();
}
this.bytes = bStream.bytes().toBytes();
this.crc32 = (int) crc32.getValue();
assertConsistent();
}
public void testBadIndex() throws Exception {
final int parts = 30;
fs.delete(p, true);
conf.setInt(TTConfig.TT_INDEX_CACHE, 1);
IndexCache cache = new IndexCache(conf);
Path f = new Path(p, "badindex");
FSDataOutputStream out = fs.create(f, false);
CheckedOutputStream iout = new CheckedOutputStream(out, new CRC32());
DataOutputStream dout = new DataOutputStream(iout);
for (int i = 0; i < parts; ++i) {
for (int j = 0; j < MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; ++j) {
if (0 == (i % 3)) {
dout.writeLong(i);
} else {
out.writeLong(i);
}
}
}
out.writeLong(iout.getChecksum().getValue());
dout.close();
try {
cache.getIndexInformation("badindex", 7, f,
UserGroupInformation.getCurrentUser().getShortUserName());
fail("Did not detect bad checksum");
} catch (IOException e) {
if (!(e.getCause() instanceof ChecksumException)) {
throw e;
}
}
}
private static void writeFile(FileSystem fs, Path f, long fill, int parts)
throws IOException {
FSDataOutputStream out = fs.create(f, false);
CheckedOutputStream iout = new CheckedOutputStream(out, new CRC32());
DataOutputStream dout = new DataOutputStream(iout);
for (int i = 0; i < parts; ++i) {
for (int j = 0; j < MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; ++j) {
dout.writeLong(fill);
}
}
out.writeLong(iout.getChecksum().getValue());
dout.close();
}