下面列出了怎么用org.apache.commons.io.output.CountingOutputStream的API类实例代码及写法,或者点击链接到github查看源代码。
public S3TransportBuffer(long maxBytes, boolean useCompression, S3TransportSerializer serializer)
throws TransportException {
this.maxBytes = maxBytes;
this.serializer = serializer;
baos = new ByteArrayOutputStream();
cos = new CountingOutputStream(baos);
if (useCompression) {
this.isCompressed = true;
try {
os = new BZip2CompressorOutputStream(cos);
} catch (IOException e) {
throw new TransportException("unable to create BZip2CompressorOutputStream", e);
}
} else {
this.isCompressed = false;
os = cos;
}
}
VideoRecordingSession(
VideoSessionRegistry registry,
ChannelHandlerContext ctx,
VideoRecordingManager dao,
RecordingEventPublisher eventPublisher,
VideoStorageSession storage,
double precapture,
boolean stream,
CountingOutputStream mpegts,
double flushTimeInS
) throws IOException {
super(registry, ctx, storage, dao, eventPublisher, stream, precapture, true);
this.adapter = new Adapter(mpegts, flushTimeInS, storage.getRecordingId());
this.storage = storage;
this.mpegts = mpegts;
}
/**
* Custom serialization method.
*
* @param oos the object output stream
*/
private void writeObject(ObjectOutputStream oos) throws IOException {
// figure out size of the written network
CountingOutputStream cos = new CountingOutputStream(new NullOutputStream());
if (isInitializationFinished) {
ModelSerializer.writeModel(model, cos, false);
}
modelSize = cos.getByteCount();
// default serialization
oos.defaultWriteObject();
// Write layer configurations
String[] layerConfigs = new String[layers.length];
for (int i = 0; i < layers.length; i++) {
layerConfigs[i] =
layers[i].getClass().getName() + "::"
+ weka.core.Utils.joinOptions(layers[i].getOptions());
}
oos.writeObject(layerConfigs);
// actually write the network
if (isInitializationFinished) {
ModelSerializer.writeModel(model, oos, false);
}
}
/**
* Custom serialization method.
*
* @param oos the object output stream
*/
private void writeObject(ObjectOutputStream oos) throws IOException {
// figure out size of the written network
CountingOutputStream cos = new CountingOutputStream(new NullOutputStream());
if (isInitializationFinished) {
ModelSerializer.writeModel(model, cos, false);
}
modelSize = cos.getByteCount();
// default serialization
oos.defaultWriteObject();
// Write layer configurations
String[] layerConfigs = new String[layers.length];
for (int i = 0; i < layers.length; i++) {
layerConfigs[i] =
layers[i].getClass().getName() + "::"
+ weka.core.Utils.joinOptions(layers[i].getOptions());
}
oos.writeObject(layerConfigs);
// actually write the network
if (isInitializationFinished) {
ModelSerializer.writeModel(model, oos, false);
}
}
DataLakeDataGenerator(
String filePath,
OutputStreamHelper outputStreamHelper,
DataGeneratorFormatConfig dataFormatConfig,
long idleTimeSecs
) throws IOException, StageException {
this.filePath = filePath;
this.outputStreamHelper = outputStreamHelper;
this.cos = new CountingOutputStream(outputStreamHelper.getOutputStream(filePath));
this.generator = dataFormatConfig.getDataGeneratorFactory().getGenerator(
cos,
outputStreamHelper.getStreamCloseEventHandler()
);
this.idleTimeSecs = idleTimeSecs;
this.recordCount = new AtomicLong(0L);
this.idleClosed = false;
}
public void open(String loc) throws IOException {
channel = new RandomAccessFile(loc + ".dat", "rw").getChannel();
CountingInputStream countertmp = new CountingInputStream(new BufferedInputStream(Channels.newInputStream(channel), buffersize));
DataInputStream is = new DataInputStream(countertmp);
nrRows = is.readInt();
nrCols = is.readInt();
hashRows = DoubleMatrixDataset.loadIdentifiers(loc + ".rows.txt");
hashCols = DoubleMatrixDataset.loadIdentifiers(loc + ".cols.txt");
headerLen = 8;
currentPos = headerLen;
bytesPerRow = 8 * nrCols;
buffersize = bytesPerRow * 10;
channel.position(currentPos);
counter = new CountingOutputStream(new BufferedOutputStream(Channels.newOutputStream(channel), buffersize));
os = new DataOutputStream(counter);
System.out.println("Read header. current pos: " + channel.position());
System.out.println("Header: " + headerLen);
}
public static IDownloadIndicator downloadFile(URL url, Path out) {
DownloadTracker tracker = new DownloadTracker();
new Thread(() -> {
tracker.init(getFileSize(url));
try (InputStream input = url.openStream()) {
Path downloads = Paths.get("downloads");
Files.createDirectories(downloads);
Path temp = downloads.resolve(UUID.randomUUID().toString() + ".sbxdl");
int totalDataRead = 0;
try (CountingOutputStream output = new CountingOutputStream(Files.newOutputStream(temp))) {
byte[] data = new byte[1024];
int b;
while ((b = input.read(data, 0, 1024)) >= 0) {
output.write(data);
totalDataRead += b;
tracker.set(totalDataRead);
}
}
if (Files.exists(out))
Files.delete(out);
if (Files.notExists(out.getParent()))
Files.createDirectories(out.getParent());
Files.move(temp, out);
tracker.complete();
} catch (Exception e) {
e.printStackTrace();
} finally {
tracker.complete();
}
}).start();
return tracker;
}
@Override
protected VideoRecordingSession create(VideoSessionRegistry registry, ChannelHandlerContext ctx, VideoRecordingManager dao, RecordingEventPublisher eventPublisher, VideoStorageSession storage, double precapture, boolean stream) {
try {
return new VideoRecordingSession(registry, ctx, dao, eventPublisher, storage, precapture, stream, new CountingOutputStream(storage.output()), config.getVideoFlushFrequency());
} catch(Exception ioe) {
throw new RuntimeException(ioe);
}
}
/**
* Open the file and write its header.
*/
private void init() throws IOException {
FileSystem fs = this.path.getFileSystem(conf);
FSDataOutputStream fsOut = fs.create(this.path);
this.countingOut = new CountingOutputStream(
new BufferedOutputStream(fsOut));
this.out = new DataOutputStream(this.countingOut);
// put any necessary config strings into the header.
MetaBlock m = this.header.getMetaBlock();
if (isCharData) {
m.put(MetaBlock.ENTRY_ENCODING_KEY, MetaBlock.CLOB_ENCODING);
} else {
m.put(MetaBlock.ENTRY_ENCODING_KEY, MetaBlock.BLOB_ENCODING);
}
if (null != codec) {
m.put(MetaBlock.COMPRESSION_CODEC_KEY, this.codecName);
}
// Serialize the value of maxEntriesPerSegment as a VInt in a byte array
// and put that into the metablock as ENTRIES_PER_SEGMENT_KEY.
int segmentBufLen = WritableUtils.getVIntSize(this.maxEntriesPerSegment);
DataOutputBuffer entriesPerSegBuf = new DataOutputBuffer(segmentBufLen);
WritableUtils.writeVInt(entriesPerSegBuf, this.maxEntriesPerSegment);
byte [] entriesPerSegArray =
Arrays.copyOf(entriesPerSegBuf.getData(), segmentBufLen);
m.put(MetaBlock.ENTRIES_PER_SEGMENT_KEY,
new BytesWritable(entriesPerSegArray));
// Write the file header to the file.
this.header.write(out);
// Now we're ready to accept record data from the user.
}
@Override
/**
* {@inheritDoc}
*/
public OutputStream writeBlobRecord(long claimedLen) throws IOException {
finishRecord(); // finish any previous record.
checkForNull(this.out);
startRecordIndex();
this.header.getStartMark().write(out);
LOG.debug("Starting new record; id=" + curEntryId
+ "; claimedLen=" + claimedLen);
WritableUtils.writeVLong(out, curEntryId);
WritableUtils.writeVLong(out, claimedLen);
this.curClaimedLen = claimedLen;
this.userCountingOutputStream = new CountingOutputStream(
new CloseShieldOutputStream(out));
if (null == this.codec) {
// No codec; pass thru the same OutputStream to the user.
this.userOutputStream = this.userCountingOutputStream;
} else {
// Wrap our CountingOutputStream in a compressing OutputStream to
// give to the user.
this.compressor.reset();
this.userOutputStream = new CompressorStream(
this.userCountingOutputStream, compressor);
}
return this.userOutputStream;
}
/** Initialize the OutputStream to the next file to write to.
*/
private void openNextFile() throws IOException {
StringBuffer sb = new StringBuffer();
Formatter fmt = new Formatter(sb);
fmt.format("%05d", this.fileNum++);
String filename = filePrefix + fmt.toString();
if (codec != null) {
filename = filename + codec.getDefaultExtension();
}
Path destFile = new Path(destDir, filename);
FileSystem fs = destFile.getFileSystem(conf);
LOG.debug("Opening next output file: " + destFile);
if (fs.exists(destFile)) {
Path canonicalDest = destFile.makeQualified(fs);
throw new IOException("Destination file " + canonicalDest
+ " already exists");
}
OutputStream fsOut = fs.create(destFile);
// Count how many actual bytes hit HDFS.
this.countingFilterStream = new CountingOutputStream(fsOut);
if (codec != null) {
// Wrap that in a compressing stream.
this.writeStream = codec.createOutputStream(this.countingFilterStream);
} else {
// Write to the counting stream directly.
this.writeStream = this.countingFilterStream;
}
}
public FileHolder() throws IOException {
super();
this.plainTempFile = Files.createTempFile("", "");
this.plainStream = new CountingOutputStream(Files.newOutputStream(plainTempFile));
this.gzTempFile = Files.createTempFile("", "");
this.gzStream = new CountingOutputStream(Files.newOutputStream(gzTempFile));
this.bzTempFile = Files.createTempFile("", "");
this.bzStream = new CountingOutputStream(Files.newOutputStream(bzTempFile));
}
public FileHolder() throws IOException {
super();
this.plainTempFile = Files.createTempFile("", "");
this.plainStream = new CountingOutputStream(Files.newOutputStream(plainTempFile));
this.gzTempFile = Files.createTempFile("", "");
this.gzStream = new CountingOutputStream(Files.newOutputStream(gzTempFile));
this.bzTempFile = Files.createTempFile("", "");
this.bzStream = new CountingOutputStream(Files.newOutputStream(bzTempFile));
}
@Test
public void downloadMethodShouldCreateLocalFileStreamContainingProgressListener() throws IOException {
FTPFile file = new FTPFile("remote.file", 0l, "path/to", 0, false);
ftpConnection.setProgressListener(new ProgressListener());
ftpConnection.download(file, LOCAL_DIRECTORY);
verify(mockFileStreamFactory).createOutputStream(LOCAL_DIRECTORY + "/remote.file");
verify(mockFtpClient).retrieveFile(eq("path/to/remote.file"), any(CountingOutputStream.class));
}
public RecordWriter(Path path, long timeToLiveMillis, OutputStream textOutputStream,
DataGeneratorFactory generatorFactory, StreamCloseEventHandler streamCloseEventHandler) throws StageException, IOException {
this(path, timeToLiveMillis, generatorFactory);
this.textOutputStream = new CountingOutputStream(textOutputStream);
generator = generatorFactory.getGenerator(this.textOutputStream, streamCloseEventHandler);
textFile = true;
this.idleTimeout = -1L;
}
public void writeRow(int row, double[] cols) throws IOException {
long seekLoc = ((long) row * bytesPerRow) + headerLen;
if (seekLoc > channel.size()) {
throw new IllegalArgumentException("Seek location for row: " + row + ", " + seekLoc + " is outside file size: " + channel.size());
}
// if row is the next row, just write.
if (seekLoc - currentPos == 0) {
writeRow(cols);
} else {
// else, random access to new location
channel.position(seekLoc);
if (bytebuffer == null) {
bytebuffer = ByteBuffer.wrap(new byte[bytesPerRow]);
}
channel.write(bytebuffer);
// this is probably extremely slow?
counter = new CountingOutputStream(new BufferedOutputStream(Channels.newOutputStream(channel), buffersize));
os = new DataOutputStream(counter);
currentPos = seekLoc + bytesPerRow;
}
}
public void write(int row, int col, double val) throws IOException {
long seekLoc = ((long) row * bytesPerRow) + headerLen + (col * 8);
// System.out.println(row + "\t" + col + "\t" + seekLoc + "\t" + currentPos + "\t" + val);
if (seekLoc - currentPos == 0) {
os.writeDouble(val);
os.flush();
currentPos = seekLoc + 8;
} else {
if (seekLoc > channel.size()) {
throw new IllegalArgumentException("Seek location for row: " + row + ", " + seekLoc + " is outside file size: " + channel.size());
}
if (singledouble == null) {
singledouble = ByteBuffer.allocate(8);
}
singledouble.putDouble(val);
singledouble.flip();
// System.out.println("Seeking: " + seekLoc);
channel.position(seekLoc);
channel.write(singledouble);
currentPos = seekLoc + 8;
singledouble.compact();
// this is probably extremely slow?
counter = new CountingOutputStream(new BufferedOutputStream(Channels.newOutputStream(channel), buffersize));
os = new DataOutputStream(counter);
}
}
Adapter(CountingOutputStream mpegts, double flushTimeInS, UUID recId) throws IOException {
super(mpegts, flushTimeInS, recId);
}
protected MpegTsH264Adapter(CountingOutputStream mpegts, double flushTimeInS, UUID recId) throws IOException {
super(new DataOutputStream(mpegts), recId);
this.mpegts = mpegts;
this.flushFrequencyIn90KHz = (long)(flushTimeInS * 90000);
}
private CountingOutputStream listenOn(OutputStream outputStream) {
LOGGER.debug("Creating wrapping output stream for progress listener");
CountingOutputStream countingStream = new CountingOutputStream(outputStream) {
@Override
protected void beforeWrite(int n) {
super.beforeWrite(n);
progressListener.setBytesWritten(getByteCount());
}
};
return countingStream;
}