下面列出了com.google.common.io.ByteStreams#limit ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
/**
* Fetch the urlConnection stream by using the urlconnection and return its content To prevent DOS
* attack, limit the amount of the collected data
*
* @param urlConnection the URL connection to fetch
* @return the content of the file
* @throws IOException if fetch error occurs
*/
@VisibleForTesting
String fetch(@NotNull URLConnection urlConnection) throws IOException {
requireNonNull(urlConnection, "urlConnection parameter can't be null");
final String value;
try (InputStream inputStream = urlConnection.getInputStream();
BufferedReader reader =
new BufferedReader(
new InputStreamReader(ByteStreams.limit(inputStream, getLimit()), UTF_8))) {
value = reader.lines().collect(Collectors.joining("\n"));
} catch (IOException e) {
// we shouldn't fetch if check is done before
LOG.debug("Invalid URL", e);
throw e;
}
return value;
}
/**
* {@inheritDoc}
*/
@Override
public void write(T entity, OutputStream output, Long pos, Long length) throws IOException {
InputStream in = getInputStream(entity);
if (pos != null && pos > 0) {
in.skip(pos);
}
if (length != null && length > 0) {
in = ByteStreams.limit(in, length);
}
try {
ReaderWriter.writeTo(in, output);
} finally {
IOUtils.closeQuietly(in);
}
}
public InputStream doSeekLoad(long offset, int maxLength) throws IOException {
if ((type == NIOType.MMAP) && (maxLength >= 0)) {
ByteBuffer mapBuff = fc.map(MapMode.READ_ONLY, offset, maxLength);
bbis = new ByteBufferBackedInputStream(mapBuff, offset);
return ByteStreams.limit(bbis, maxLength);
} else {
fcis = new FileChannelInputStream(fc, offset, maxLength);
return fcis;
// if (maxLength > 0) {
// return new LimitInputStream(fcis, maxLength);
// } else {
// return fcis;
// }
}
}
@Override
public void read(
final AsnObjectSerializationContext context,
final AsnOctetStringBasedObjectCodec instance,
final InputStream inputStream
) throws IOException {
Objects.requireNonNull(context);
Objects.requireNonNull(instance);
Objects.requireNonNull(inputStream);
// WARNING: This length can be maliciously specified by the packet creator, so be careful not to use it for unsafe
// operations, such as creating a new array of initial size `length`. This usage is safe because it merely caps the
// InputStream to the specified packet-length, whereas the InputStream is authoritative for when it actually ends,
// and this limit may be well smaller than `length`.
int lengthToRead;
final AsnSizeConstraint sizeConstraint = instance.getSizeConstraint();
if (sizeConstraint.isFixedSize()) {
lengthToRead = sizeConstraint.getMax();
} else {
// Read the lengthToRead of the encoded OctetString...will never be > 127
lengthToRead = OerLengthSerializer.readLength(inputStream);
}
final byte[] bytes;
if (lengthToRead == 0) {
bytes = new byte[0];
} else {
// Use a limited input stream so we don't read too many bytes.
final InputStream limitedInputStream = ByteStreams.limit(inputStream, lengthToRead);
bytes = ByteStreams.toByteArray(limitedInputStream);
if (bytes.length != lengthToRead) {
throw new IOException(
format("Unable to properly decode %s bytes (could only read %s bytes)", lengthToRead, bytes.length)
);
}
}
instance.setBytes(bytes);
}
@Override
protected InputStream doSeekLoad(long offset, int maxLength)
throws IOException {
bbis.position(offset);
if (maxLength > 0) {
return ByteStreams.limit(bbis, maxLength);
} else {
return bbis;
}
}
/**
* Opens a stream to the first file archived in a TAR archive.
*
* <p>The result includes the file's metadata - the file name and modification time, as well as
* the full TAR header. Note that only the file name and modification times were actually set by
* {@link #openTarWriter}.
*
* @param input from where to read the TAR archive.
*/
@CheckReturnValue
static TarInputStream openTarReader(@WillNotClose InputStream input) {
try {
byte[] header = new byte[PosixTarHeader.HEADER_LENGTH];
ByteStreams.readFully(input, header, 0, header.length);
PosixTarHeader tarHeader = PosixTarHeader.from(header);
checkState(
tarHeader.getType() == PosixTarHeader.Type.REGULAR,
"Only support TAR archives with a single regular file");
return new TarInputStream(tarHeader, ByteStreams.limit(input, tarHeader.getSize()));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Extract the digest string from a stream, that carries a payload of a Maven sha1/md5 file. Method closes the
* passed in stream even in case of IO problem.
*
* @see {@link #extract(String)}
*/
@Nullable
public static String extract(final InputStream stream)
throws IOException
{
checkNotNull(stream);
try (InputStreamReader isr = new InputStreamReader(ByteStreams.limit(stream, MAX_CHARS_NEEDED),
StandardCharsets.UTF_8)) {
return extract(CharStreams.toString(isr));
}
finally {
stream.close();
}
}
@Override
@Nullable
@Guarded(by = STARTED)
public InputStream getLogFileStream(final String fileName, final long from, final long count) throws IOException {
log.debug("Retrieving log file: {}", fileName);
// checking for platform or normalized path-separator (on unix these are the same)
if (fileName.contains(File.pathSeparator) || fileName.contains("/")) {
log.warn("Cannot retrieve log files with path separators in their name");
return null;
}
File file = getLogFile(fileName);
if (file == null || !file.exists()) {
log.warn("Log file does not exist: {}", fileName);
return null;
}
long fromByte = from;
long bytesCount = count;
if (count < 0) {
bytesCount = Math.abs(count);
fromByte = Math.max(0, file.length() - bytesCount);
}
InputStream input = new BufferedInputStream(new FileInputStream(file));
if (fromByte == 0 && bytesCount >= file.length()) {
return input;
}
else {
input.skip(fromByte);
return ByteStreams.limit(input, bytesCount);
}
}
private static byte[] readFromBody(final ClientHttpResponse response) throws IOException {
@Nullable final InputStream stream = response.getBody();
// needed for spring versions prior to 4.3.14
if (stream == null) {
return new byte[0];
}
final InputStream input = ByteStreams.limit(stream, MAX_BODY_BYTES_TO_READ);
final ByteArrayOutputStream output = new ByteArrayOutputStream();
ByteStreams.copy(input, output);
return output.toByteArray();
}
/**
* @return a mutable list
* @throws IOException
*/
public static List<SourceFile> getDefaultExterns() throws IOException {
InputStream input = CommandLineRunner.class.getResourceAsStream(
"/externs.zip");
if (input == null) {
// In some environments, the externs.zip is relative to this class.
input = CommandLineRunner.class.getResourceAsStream("externs.zip");
}
Preconditions.checkNotNull(input);
ZipInputStream zip = new ZipInputStream(input);
Map<String, SourceFile> externsMap = Maps.newHashMap();
for (ZipEntry entry = null; (entry = zip.getNextEntry()) != null; ) {
BufferedInputStream entryStream = new BufferedInputStream(
ByteStreams.limit(zip, entry.getSize()));
externsMap.put(entry.getName(),
SourceFile.fromInputStream(
// Give the files an odd prefix, so that they do not conflict
// with the user's files.
"externs.zip//" + entry.getName(),
entryStream));
}
Preconditions.checkState(
externsMap.keySet().equals(Sets.newHashSet(DEFAULT_EXTERNS_NAMES)),
"Externs zip must match our hard-coded list of externs.");
// Order matters, so the resources must be added to the result list
// in the expected order.
List<SourceFile> externs = Lists.newArrayList();
for (String key : DEFAULT_EXTERNS_NAMES) {
externs.add(externsMap.get(key));
}
return externs;
}
/**
* @return a mutable list
* @throws IOException
*/
public static List<SourceFile> getDefaultExterns() throws IOException {
InputStream input = CommandLineRunner.class.getResourceAsStream(
"/externs.zip");
if (input == null) {
// In some environments, the externs.zip is relative to this class.
input = CommandLineRunner.class.getResourceAsStream("externs.zip");
}
Preconditions.checkNotNull(input);
ZipInputStream zip = new ZipInputStream(input);
Map<String, SourceFile> externsMap = Maps.newHashMap();
for (ZipEntry entry = null; (entry = zip.getNextEntry()) != null; ) {
BufferedInputStream entryStream = new BufferedInputStream(
ByteStreams.limit(zip, entry.getSize()));
externsMap.put(entry.getName(),
SourceFile.fromInputStream(
// Give the files an odd prefix, so that they do not conflict
// with the user's files.
"externs.zip//" + entry.getName(),
entryStream));
}
Preconditions.checkState(
externsMap.keySet().equals(Sets.newHashSet(DEFAULT_EXTERNS_NAMES)),
"Externs zip must match our hard-coded list of externs.");
// Order matters, so the resources must be added to the result list
// in the expected order.
List<SourceFile> externs = Lists.newArrayList();
for (String key : DEFAULT_EXTERNS_NAMES) {
externs.add(externsMap.get(key));
}
return externs;
}
private void logBody(MimeMessage message) throws MessagingException, IOException {
if (body && logger.isInfoEnabled()) {
try (InputStream inputStream = ByteStreams.limit(message.getDataHandler().getInputStream(), lengthToLog(message))) {
logger.info(IOUtils.toString(inputStream, StandardCharsets.UTF_8));
}
}
}
@Override
public InputStream read(int size, boolean extraCRLF) {
// Unset the next char.
nextSeen = false;
nextChar = 0;
InputStream limited = ByteStreams.limit(input, size);
if (extraCRLF) {
return new EolInputStream(this, limited);
} else {
return limited;
}
}
@Override
public InputStream getHeaderContent() throws IOException {
parseMessage();
long limit = getBodyStartOctet();
if (limit < 0) {
limit = 0;
}
return ByteStreams.limit(new FileInputStream(messageName.getFile()), limit);
}
public InputStream doSeekLoad(long offset, int maxLength) throws IOException {
fsdis.seek(offset);
if (maxLength >= 0) {
return ByteStreams.limit(fsdis, maxLength);
} else {
return fsdis;
}
}
static void handleMultiPartBody(Response response, RangeReceiver receiver, final Set<ContentRange> remaining,
HttpTransferListener listener, byte[] boundary) throws IOException {
try (InputStream in = inputStream(response, listener)) {
ContentRange range;
while ((range = nextPart(in, boundary)) != null) {
// technically it's OK for server to combine or re-order ranges. However, since we
// already combine and sort ranges, this should not happen
if (!remaining.remove(range)) {
throw new IOException("Received range " + range + " not one of requested " + remaining);
}
final InputStream part = ByteStreams.limit(in, range.length());
receiver.receive(range, part);
}
}
}
private static InputStream readPayloadStream(InputStream is) throws IOException {
final byte[] payloadLengthBytes = new byte[4];
ByteStreams.readFully(is, payloadLengthBytes);
final int payloadLength = Bytes.bytesToInt(payloadLengthBytes);
return ByteStreams.limit(is, payloadLength);
}
public ARCResource(MetaData metaData, ResourceContainer container,
ARCMetaData arcMetaData, InputStream raw) {
super(metaData.createChild(PAYLOAD_METADATA),container);
envelope = metaData;
this.arcMetaData = arcMetaData;
this.raw = raw;
metaData.putString(ENVELOPE_FORMAT, ENVELOPE_FORMAT_ARC);
metaData.putLong(ARC_HEADER_LENGTH, arcMetaData.getHeaderLength());
long leadingNL = arcMetaData.getLeadingNL();
if(leadingNL > 0) {
metaData.putLong(PAYLOAD_LEADING_SLOP_BYTES, leadingNL);
}
MetaData fields = metaData.createChild(ARC_HEADER_METADATA);
fields.putString(URL_KEY, arcMetaData.getUrl());
fields.putString(IP_KEY, arcMetaData.getIP());
fields.putString(DATE_STRING_KEY, arcMetaData.getDateString());
fields.putString(MIME_KEY, arcMetaData.getMime());
fields.putLong(DECLARED_LENGTH_KEY, arcMetaData.getLength());
countingIS = new CountingInputStream(
ByteStreams.limit(raw, arcMetaData.getLength()));
try {
digIS = new DigestInputStream(countingIS,
MessageDigest.getInstance("sha1"));
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
}
}
public InputStream doSeekLoad(long offset, int maxLength) throws IOException {
raf.seek(offset);
FileInputStream fis = new FileInputStream(raf.getFD());
if (maxLength > 0) {
return ByteStreams.limit(fis, maxLength);
} else {
return fis;
}
}
@Override
public void read(
final AsnObjectSerializationContext context,
final AsnCharStringBasedObjectCodec instance,
final InputStream inputStream
) throws IOException {
Objects.requireNonNull(context);
Objects.requireNonNull(instance);
Objects.requireNonNull(inputStream);
// WARNING: This length can be maliciously specified by the packet creator, so be careful not to use it for unsafe
// operations, such as creating a new array of initial size `length`. This usage is safe because it merely caps the
// InputStream to the specified packet-length, whereas the InputStream is authoritative for when it actually ends,
// and this limit may be well smaller than `length`.
int lengthToRead;
final AsnSizeConstraint sizeConstraint = instance.getSizeConstraint();
if (sizeConstraint.isFixedSize()) {
lengthToRead = sizeConstraint.getMax();
} else {
// Read the lengthToRead of the encoded OctetString...
lengthToRead = OerLengthSerializer.readLength(inputStream);
}
final String result;
/* beware the 0-lengthToRead string */
if (lengthToRead == 0) {
result = "";
} else {
// Use a limited input stream so we don't read too many bytes.
final InputStream limitedInputStream = ByteStreams.limit(inputStream, lengthToRead);
// WARNING: Don't close the InputStreamReader so that the underlying inputStream is not closed.
result = CharStreams.toString(new InputStreamReader(limitedInputStream, instance.getCharacterSet().name()));
// For UTF-8 characters, result.length() will report the viewable length (e.g., 3) but for certain encoded
// characters, the actual byte-length will be larger (e.g., the String 元元元 is 3 viewable bytes, but 9 encoded
// UTF-8 bytes). Thus, when we write the length-prefix, the code will write 9, so when we read, we need to
// validate that 9 bytes were read, and not 3 (in this example).
if (Utf8.encodedLength(result) != lengthToRead) {
throw new IOException(
format("Unable to properly decode %s bytes (could only read %s bytes)", lengthToRead, result.length())
);
}
}
instance.setCharString(result);
}