/**
 * Retrieve playlist data from a given URL.
 *
 * @param url URL to the remote playlist
 * @param maxByteSize maximum size of the response, in bytes, or 0 if unlimited
 * @param maxRedirects maximum number of redirects, or 0 if unlimited
 * @return the remote playlist data
 */
protected SpecificPlaylist retrievePlaylist(URL url, long maxByteSize, int maxRedirects) throws IOException, PlaylistException {

    SpecificPlaylist playlist;
    HttpURLConnection urlConnection = connectToURLWithRedirects(url, maxRedirects);
    try (InputStream in = urlConnection.getInputStream()) {
        String contentEncoding = urlConnection.getContentEncoding();
        if (maxByteSize > 0) {
            playlist = SpecificPlaylistFactory.getInstance().readFrom(new BoundedInputStream(in, maxByteSize), contentEncoding);
        } else {
            playlist = SpecificPlaylistFactory.getInstance().readFrom(in, contentEncoding);
        }
    } finally {
        urlConnection.disconnect();
    }
    if (playlist == null) {
        throw new PlaylistFormatUnsupported("Unsupported playlist format " + url.toString());
    }
    return playlist;
}
 

/**
 * Extracts the text content from the given API response.
 *
 * @param response the API (GetTextContent) response.
 * @return the text content from the given API response.
 * @throws IOException in case of I/O failure.
 */
private String textContentFrom(GetTextContentResponse response) throws IOException
{
    try (final InputStream ris = ofNullable(response.getContentEncoding())
            .map(c -> c.equals("gzip")).orElse(false)?
            new GZIPInputStream(response.getContent()) : response.getContent())
    {
        if (ris != null)
        {
            byte[] bytes = FileCopyUtils.copyToByteArray(new BoundedInputStream(ris, contentStreamLimit));
            return new String(bytes, StandardCharsets.UTF_8);
        }
        return "";
    }
    finally
    {
        response.release();
    }
}
 

@Override
protected InputStream getDownloadersInputStream() throws IOException {
    Request request = Request.createGET(sourcePath, connection);
    Response response = request.send();
    fileDetails = response.toFileDetails();

    // TODO: Manage the dependency which includes this class better?
    // Right now, I only needed the one class from apache commons.
    // There are countless classes online which provide this functionality,
    // including some which are available from the Android SDK - the only
    // problem is that they have a funky API which doesn't just wrap a
    // plain old InputStream (the class is ContentLengthInputStream -
    // whereas this BoundedInputStream is much more generic and useful
    // to us).
    BoundedInputStream stream = new BoundedInputStream(response.toContentStream(), fileDetails.getFileSize());
    stream.setPropagateClose(false);

    return stream;
}
 

public X509CRL requestCRL(String url) throws IOException, MalformedURLException, CertificateException, CRLException {
	HttpURLConnection con = (HttpURLConnection) new URL(url).openConnection();
	try {
		con.setUseCaches(false);

		InputStream in = new BoundedInputStream(con.getInputStream(), maxCrlSize);
		try {
			CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509");
			X509CRL crl = (X509CRL) certificateFactory.generateCRL(in);
			log.debug("CRL size: " + crl.getEncoded().length + " bytes");

			return crl;
		} finally {
			IOUtils.closeQuietly(in);
		}
	} catch (IOException ex) {
		log.error("Failed to download CRL from '" + url + "'", ex);
	} finally {
		if (con != null) {
			con.disconnect();
		}
	}
	
	return null;
}
 

/**
 *
 * @return path to server, unzipping it if necessary. Can be null is fragment is missing.
 */
private @Nullable File getServer() throws IOException {
	File serverPath = new File(AcutePlugin.getDefault().getStateLocation().toFile(), "omnisharp-roslyn"); //$NON-NLS-1$
	if (!serverPath.exists()) {
		serverPath.mkdirs();
		try (
			InputStream stream = FileLocator.openStream(AcutePlugin.getDefault().getBundle(), new Path("omnisharp-roslyn.tar"), true); //$NON-NLS-1$
			TarArchiveInputStream tarStream = new TarArchiveInputStream(stream);
		) {
			TarArchiveEntry entry = null;
			while ((entry = tarStream.getNextTarEntry()) != null) {
				if (!entry.isDirectory()) {
					File targetFile = new File(serverPath, entry.getName());
					targetFile.getParentFile().mkdirs();
					InputStream in = new BoundedInputStream(tarStream, entry.getSize()); // mustn't be closed
					try (
						FileOutputStream out = new FileOutputStream(targetFile);
					) {
						IOUtils.copy(in, out);
						if (!Platform.OS_WIN32.equals(Platform.getOS())) {
							int xDigit = entry.getMode() % 10;
							targetFile.setExecutable(xDigit > 0, (xDigit & 1) == 1);
							int wDigit = (entry.getMode() / 10) % 10;
							targetFile.setWritable(wDigit > 0, (wDigit & 1) == 1);
							int rDigit = (entry.getMode() / 100) % 10;
							targetFile.setReadable(rDigit > 0, (rDigit & 1) == 1);
						}
					}
				}
			}
		}
	}
	return serverPath;
}
 

protected InputStream normalize(final InputStream in, final TransferStatus status) throws ChecksumException {
    try {
        final InputStream bounded = status.getLength() > 0 ?
            new BoundedInputStream(in, status.getOffset() + status.getLength()) : in;
        return status.getOffset() > 0 ? StreamCopier.skip(bounded, status.getOffset()) : bounded;
    }
    catch(BackgroundException e) {
        throw new ChecksumException(LocaleFactory.localizedString("Checksum failure", "Error"), e.getMessage(), e);
    }
}
 

/**
 * @return Data input for the next message. Note that it does not automatically skip over the last message if it was
 *         not fully read, for that purpose, skipRemainingBytes() should be explicitly called after reading every
 *         message. A null return value indicates the position where MessageOutput#finish() had written the end
 *         marker.
 * @throws IOException On IO error
 */
public DataInput nextMessage() throws IOException {
  int value = dataInputStream.readInt();
  messageFlags = (int) ((value & 0xC0000000L) >> 30L);
  messageSize = value & 0x3FFFFFFF;

  if (messageSize == 0) {
    return null;
  }

  return new DataInputStream(new BoundedInputStream(countingInputStream, messageSize));
}
 

@Override
public RawPreview getRawSource(int maxLength, MultivaluedMap<String, String> previewParams) throws PipelineException {
  changeState(PreviewStatus.RUNNING, null);
  int bytesToRead = configuration.get(MAX_SOURCE_PREVIEW_SIZE_KEY, MAX_SOURCE_PREVIEW_SIZE_DEFAULT);
  bytesToRead = Math.min(bytesToRead, maxLength);

  PipelineConfiguration pipelineConf = pipelineStore.load(name, rev);
  if(pipelineConf.getStages().isEmpty()) {
    throw new PipelineRuntimeException(ContainerError.CONTAINER_0159, name);
  }

  //find the source stage in the pipeline configuration
  StageDefinition sourceStageDef = getSourceStageDef(pipelineConf);

  RawSourcePreviewer rawSourcePreviewer = createRawSourcePreviewer(sourceStageDef, previewParams);
  RawPreview rawPreview;
  ClassLoader classLoader = sourceStageDef.getStageClassLoader();
  ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
  try {
    Thread.currentThread().setContextClassLoader(classLoader);
    try(BoundedInputStream bIn = new BoundedInputStream(rawSourcePreviewer.preview(bytesToRead), bytesToRead)) {
      rawPreview = new RawPreviewImpl(IOUtils.toString(bIn), rawSourcePreviewer.getMimeType());
    }
  } catch (IOException ex) {
    throw new PipelineRuntimeException(PreviewError.PREVIEW_0003, ex.toString(), ex);
  } finally {
    Thread.currentThread().setContextClassLoader(contextClassLoader);
  }
  changeState(PreviewStatus.FINISHED, null);
  return rawPreview;
}
 

/**
 * Return a {@link ChannelBufferInputStream} if the wrapped
 * {@link ChannelBuffer} contains enough data. If not it will throw a
 * {@link NotEnoughDataException}
 */
@Override
public InputStream read(int size, boolean extraCRLF) throws DecodingException {
    int crlf = 0;
    if (extraCRLF) {
        crlf = 2;
    }
    
    if (maxLiteralSize > 0 && maxLiteralSize > size) {
        throw new DecodingException(HumanReadableText.FAILED, "Specified literal is greater then the allowed size");
    }
    // Check if we have enough data
    if (size + crlf > buffer.readableBytes()) {
        // ok let us throw a exception which till the decoder how many more
        // bytes we need
        throw new NotEnoughDataException(size + read + crlf);
    }

    // Unset the next char.
    nextSeen = false;
    nextChar = 0;

    // limit the size via commons-io as ChannelBufferInputStream size limiting is buggy
    InputStream in = new BoundedInputStream(new ChannelBufferInputStream(buffer), size); 
    if (extraCRLF) {
        return new EolInputStream(this, in);
    } else {
        return in;
    }
}
 

/**
 * Receive Files from {@code InputStream in} via in {@link AbstractStreamProtocol} defined
 * protocol.
 *
 * @throws IOException if any file or stream operation fails
 * @throws LocalCancellationException on local user cancellation
 */
public void receiveStream() throws IOException, LocalCancellationException {
  while (true) {
    String referencePointID = in.readUTF();

    /* check stream end */
    if (referencePointID.isEmpty()) break;

    String fileName = in.readUTF();
    IFile file = session.getReferencePoint(referencePointID).getFile(fileName);

    String message = "receiving " + displayName(file);
    log.debug(message);
    monitor.subTask(message);

    /*
     * folder creation is already done after file exchange, but in
     * case of future changes
     */
    FileSystem.createFolder(file);

    long fileSize = in.readLong();
    try (BoundedInputStream fileIn = new BoundedInputStream(in, fileSize)) {
      fileIn.setPropagateClose(false);

      if (file.exists()) file.setContents(fileIn);
      else file.create(fileIn);
    }

    if (monitor.isCanceled()) {
      throw new LocalCancellationException(
          "User canceled transmission", CancelOption.NOTIFY_PEER);
    }

    monitor.worked(1);
  }
}
 

@Override
public FetchedData fetch(Resource resource) throws Exception {
    LOG.info("HtmlUnit FETCHER {}", resource.getUrl());
    FetchedData fetchedData;
    try {
        String userAgent = getUserAgent();
        if (StringUtils.isNotBlank(userAgent)) {
            driver.removeRequestHeader(USER_AGENT);
            driver.addRequestHeader(USER_AGENT, userAgent);
        }
        Page page = driver.getPage(resource.getUrl());

        WebResponse response = page.getWebResponse();
        boolean truncated = false;
        try (InputStream stream = response.getContentAsStream()) {
            try (BoundedInputStream boundedStream = new BoundedInputStream(stream, CONTENT_LIMIT)) {
                try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
                    IOUtils.copy(boundedStream, out);
                    fetchedData = new FetchedData(out.toByteArray(), response.getContentType(), response.getStatusCode());
                    long contentLength = page.getWebResponse().getContentLength();
                    if (contentLength > 0 && contentLength < Integer.MAX_VALUE) {
                        fetchedData.setContentLength((int) contentLength);
                        truncated = (contentLength > fetchedData.getContentLength());
                        if (truncated) {
                            LOG.info("Content Truncated: {}, TotalSize={}", resource.getUrl(), contentLength);
                        }
                    }
                }
            }
        }
        resource.setStatus(ResourceStatus.FETCHED.toString());

        List<NameValuePair> respHeaders = page.getWebResponse().getResponseHeaders();
        Map<String, List<String>> headers = new HashMap<>();
        fetchedData.setHeaders(headers);
        if (respHeaders != null && !respHeaders.isEmpty()){
            respHeaders.forEach(item -> {
                if (!headers.containsKey(item.getName())) {
                    headers.put(item.getName(), new ArrayList<>());
                }
                headers.get(item.getName()).add(item.getValue());
            });
        }
        if (truncated){ //add truncated header
            headers.put(TRUNCATED, Collections.singletonList(Boolean.TRUE.toString()));
        }
    } catch (Exception e){
        LOG.warn(e.getMessage(), e);
        fetchedData = new FetchedData(new byte[0], "unknown/unknown", 0); // fixme: use proper status code
        resource.setStatus(ResourceStatus.ERROR.toString());
    }
    fetchedData.setResource(resource);
    return fetchedData;
}
 

BoundedInputStream boundedInputStream(InputStream inputStream, int length) {
    BoundedInputStream bis = new BoundedInputStream(inputStream, length); // No close() required/ not propagated.
    bis.setPropagateClose(false);
    return bis;
}
 

@Override
protected InputStream createSegment() throws IOException {
	InputStream res = new BoundedInputStream(Channels.newInputStream(this.randomAccessFile.getChannel().position(currentSegment * segmentationSize)), segmentationSize);
	((BoundedInputStream) res).setPropagateClose(false) ;
	return res ;                
}
 

@Override
public InputStream getInputStream() throws IOException {
  FSDataInputStream inputStream = localFS.open(inputFile);
  inputStream.seek(startOffset);
  return new BoundedInputStream(inputStream, getSize());
}
 

/**
 * Get a bounded inputstream.
 * @param stream the input stream
 * @param from the byte from which to start
 * @param to the byte to which to read
 * @throws IOException if an error occurs when skipping forward
 * @return the bounded inputstream
 */
public static InputStream getBoundedStream(final InputStream stream, final int from, final int to)
        throws IOException {
    final long skipped = stream.skip(from);
    LOGGER.debug("Skipped {} bytes", skipped);
    return new BoundedInputStream(stream, (long) to - from);
}