@Override
public synchronized void store(final URI uri, final HttpResponse response, boolean isDuplicate, final byte[] contentDigest, final String guessedCharset) throws IOException, InterruptedException {
	if (contentDigest == null) throw new NullPointerException("Content digest is null");
	final HttpResponseWarcRecord record = new HttpResponseWarcRecord(uri, response);
	HeaderGroup warcHeaders = record.getWarcHeaders();
	warcHeaders.updateHeader(new WarcHeader(WarcHeader.Name.WARC_PAYLOAD_DIGEST, "bubing:" + Hex.encodeHexString(contentDigest)));
	if (guessedCharset != null) warcHeaders.updateHeader(new WarcHeader(WarcHeader.Name.BUBING_GUESSED_CHARSET, guessedCharset));
	if (isDuplicate) warcHeaders.updateHeader(new WarcHeader(WarcHeader.Name.BUBING_IS_DUPLICATE, "true"));
	writer.write(record);
}
 

@Override
public void store(final URI uri, final HttpResponse response, final boolean isDuplicate, final byte[] contentDigest, final String guessedCharset) throws IOException, InterruptedException {
	if (contentDigest == null) throw new NullPointerException("Content digest is null");
	final HttpResponseWarcRecord record = new HttpResponseWarcRecord(uri, response);
	HeaderGroup warcHeaders = record.getWarcHeaders();
	warcHeaders.updateHeader(new WarcHeader(WarcHeader.Name.WARC_PAYLOAD_DIGEST, "bubing:" + Hex.encodeHexString(contentDigest)));
	if (guessedCharset != null) warcHeaders.updateHeader(new WarcHeader(WarcHeader.Name.BUBING_GUESSED_CHARSET, guessedCharset));
	if (isDuplicate) warcHeaders.updateHeader(new WarcHeader(WarcHeader.Name.BUBING_IS_DUPLICATE, "true"));
	writer.write(record);
}
 

protected static void writeHeaders(final HeaderGroup headers, final OutputStream output) throws IOException {
	for (final HeaderIterator it = headers.iterator(); it.hasNext();) {
		final org.apache.http.Header header = it.nextHeader();
		Util.toOutputStream(BasicLineFormatter.formatHeader(header, null), output);
		output.write(ByteArraySessionOutputBuffer.CRLF);
	}
}
 

private HttpRequestWarcRecord(final HeaderGroup warcHeaders, final URI targetURI, final HttpRequest request) {
	super(targetURI, warcHeaders);
	getWarcTargetURI(); // Check correct initialization
	this.warcHeaders.updateHeader(Type.warcHeader(Type.REQUEST));
	this.warcHeaders.updateHeader(new WarcHeader(WarcHeader.Name.CONTENT_TYPE, HTTP_REQUEST_MSGTYPE));
	this.protocolVersion = request.getProtocolVersion();
	this.requestLine = request.getRequestLine();
	this.setHeaders(request.getAllHeaders());
}
 

private HttpResponseWarcRecord(final HeaderGroup warcHeaders, final URI targetURI, final HttpResponse response, final HttpEntityFactory hef) throws IOException {
	super(targetURI, warcHeaders);
	getWarcTargetURI(); // Check correct initialization
	this.warcHeaders.updateHeader(Type.warcHeader(Type.RESPONSE));
	this.warcHeaders.updateHeader(new WarcHeader(WarcHeader.Name.CONTENT_TYPE, HTTP_RESPONSE_MSGTYPE));
	this.protocolVersion = response.getProtocolVersion();
	this.statusLine = response.getStatusLine();
	this.setHeaders(response.getAllHeaders());
	this.entity = (hef == null ? IdentityHttpEntityFactory.INSTANCE : hef).newEntity(response.getEntity());
}
 

public static Set<String> keys(HeaderGroup hg) {
	Set<String> ret = new HashSet<>();
	for (HeaderIterator it = hg.iterator(); it.hasNext();) {
		Header header = it.nextHeader();
		ret.add(header.getName().toLowerCase());
	}
	return ret;
}
 

public static List<Set<String>> diff(HeaderGroup expected, HeaderGroup actual) {
	Set<String> expecetdKeys = keys(expected);
	Set<String> actualKeys = keys(actual);
	Set<String> common = Sets.intersection(expecetdKeys, actualKeys);
	Set<String> symdiff = Sets.symmetricDifference(expecetdKeys, actualKeys);
	Set<String> diffval = new HashSet<>();
	for (String s : common)
		if (! expected.getCondensedHeader(s).getValue().equals(actual.getCondensedHeader(s).getValue())) diffval.add(s);
	return Arrays.asList(diffval, symdiff);
}
 

public static List<Set<String>> diff(Header[] expecteds, Header[] actuals) {
	HeaderGroup expected = new HeaderGroup();
	expected.setHeaders(expecteds);
	HeaderGroup actual = new HeaderGroup();
	expected.setHeaders(actuals);
	return diff(expected, actual);
}
 

public static HeaderGroup toHeaderGroup(String[] read) {
	HeaderGroup ret = new HeaderGroup();
	for (String s : read) {
		String[] p = s.split(":", 2);
		if (p.length != 2) continue;
		ret.addHeader(new BasicHeader(p[0].trim(), p[1].trim()));
	}
	return ret;
}
 

public WarcRecord(SessionInputBuffer buffer) throws IOException {
	this.warcHeaders = readCRLFSeparatedBlock(buffer);
	if (this.warcHeaders.length == 0) throw new EOFException();
	long contentLength = contentLength(this.warcHeaders);
	if (contentLength == -1) throw new WarcFormatException("Can't find Content-Length");
	final HeaderGroup hg = toHeaderGroup(this.warcHeaders);
	date = WarcHeader.parseDate(WarcHeader.getFirstHeader(hg, WarcHeader.Name.WARC_DATE).getValue());
	uuid = WarcHeader.parseId(WarcHeader.getFirstHeader(hg, WarcHeader.Name.WARC_RECORD_ID).getValue());
	this.payload = new byte[(int)contentLength];
	for (int read = 0; read < contentLength; read ++) this.payload[read] = (byte)buffer.read();
}
 

/**
 * Create a new {@link HttpCacheEntry}
 * 
 * @param requestDate
 *            Date/time when the request was made (Used for age
 *            calculations)
 * @param responseDate
 *            Date/time that the response came back (Used for age
 *            calculations)
 * @param statusLine
 *            HTTP status line
 * @param responseHeaders
 *            Header[] from original HTTP Response
 */
public HttpCacheEntry(final Date requestDate, final Date responseDate,
		final StatusLine statusLine, final Header[] responseHeaders,
		final Resource resource, final Set<String> variants) {
	super();
	if (requestDate == null) {
		throw new IllegalArgumentException("Request date may not be null");
	}
	if (responseDate == null) {
		throw new IllegalArgumentException("Response date may not be null");
	}
	if (statusLine == null) {
		throw new IllegalArgumentException("Status line may not be null");
	}
	if (responseHeaders == null) {
		throw new IllegalArgumentException(
				"Response headers may not be null");
	}
	if (resource == null) {
		throw new IllegalArgumentException("Resource may not be null");
	}
	this.requestDate = requestDate;
	this.responseDate = responseDate;
	this.statusLine = statusLine;
	this.responseHeaders = new HeaderGroup();
	this.responseHeaders.setHeaders(responseHeaders);
	this.resource = resource;
	this.variantURIs = variants != null ? new HashSet<String>(variants)
			: new HashSet<String>();
}
 

public static InfoWarcRecord fromPayload(final HeaderGroup warcHeaders, final BoundSessionInputBuffer payloadBuffer) throws IOException {
	return new InfoWarcRecord(warcHeaders, readPayload(payloadBuffer));
}
 

private InfoWarcRecord(final HeaderGroup warcHeaders, final Header[] info) {
	super(warcHeaders);
	this.warcHeaders.updateHeader(Type.warcHeader(Type.WARCINFO));
	this.info.setHeaders(info);
}
 

public HeaderGroup getInfo() {
	return info;
}
 

@Override
public HeaderGroup getWarcHeaders() {
	return this.warcHeaders;
}
 

public static HttpRequestWarcRecord fromPayload(final HeaderGroup warcHeaders, final BoundSessionInputBuffer payloadBuffer) throws IOException {
	return new HttpRequestWarcRecord(warcHeaders, null, readPayload(payloadBuffer));
}
 

public static HttpResponseWarcRecord fromPayload(final HeaderGroup warcHeaders, final BoundSessionInputBuffer payloadBuffer) throws IOException {
	return new HttpResponseWarcRecord(warcHeaders, null, readPayload(payloadBuffer), IdentityHttpEntityFactory.INSTANCE);
}
 

public static List<Set<String>> diff(Header[] expecteds, HeaderGroup actual) {
	HeaderGroup expected = new HeaderGroup();
	expected.setHeaders(expecteds);
	return diff(expected, actual);
}
 

public static List<Set<String>> diff(HeaderGroup expected, Header[] actuals) {
	HeaderGroup actual = new HeaderGroup();
	actual.setHeaders(actuals);
	return diff(expected, actual);
}
 

/**
 * Given access token, query IDP URL snowflake app to get SAML response
 * We also need to perform important client side validation:
 * validate the post back url come back with the SAML response
 * contains the same prefix as the Snowflake's server url, which is the
 * intended destination url to Snowflake.
 * Explanation:
 * This emulates the behavior of IDP initiated login flow in the user
 * browser where the IDP instructs the browser to POST the SAML
 * assertion to the specific SP endpoint.  This is critical in
 * preventing a SAML assertion issued to one SP from being sent to
 * another SP.
 *
 * @param loginInput   Login Info for the request
 * @param ssoUrl       URL to use for SSO
 * @param oneTimeToken The token used for SSO
 * @return The response in HTML form
 * @throws SnowflakeSQLException Will be thrown if the destination URL in
 *                               the SAML assertion does not match
 */
private static String federatedFlowStep4(
    SFLoginInput loginInput,
    String ssoUrl,
    String oneTimeToken) throws SnowflakeSQLException
{
  String responseHtml = "";
  try
  {

    final URL url = new URL(ssoUrl);
    URI oktaGetUri = new URIBuilder()
        .setScheme(url.getProtocol())
        .setHost(url.getHost())
        .setPath(url.getPath())
        .setParameter("RelayState", "%2Fsome%2Fdeep%2Flink")
        .setParameter("onetimetoken", oneTimeToken).build();
    HttpGet httpGet = new HttpGet(oktaGetUri);

    HeaderGroup headers = new HeaderGroup();
    headers.addHeader(new BasicHeader(HttpHeaders.ACCEPT, "*/*"));
    httpGet.setHeaders(headers.getAllHeaders());

    responseHtml = HttpUtil.executeGeneralRequest(
        httpGet,
        loginInput.getLoginTimeout(),
        loginInput.getOCSPMode());

    // step 5
    String postBackUrl = getPostBackUrlFromHTML(responseHtml);
    if (!isPrefixEqual(postBackUrl, loginInput.getServerUrl()))
    {
      logger.debug("The specified authenticator {} and the destination URL " +
                   "in the SAML assertion {} do not match.",
                   loginInput.getAuthenticator(), postBackUrl);
      throw new SnowflakeSQLException(
          SqlState.SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION,
          ErrorCode.IDP_INCORRECT_DESTINATION.getMessageCode());
    }
  }
  catch (IOException | URISyntaxException ex)
  {
    handleFederatedFlowError(loginInput, ex);
  }
  return responseHtml;
}
 

/**
 * Query IDP token url to authenticate and retrieve access token
 *
 * @param loginInput The login info for the request
 * @param tokenUrl   The URL used to retrieve the access token
 * @return Returns the one time token
 * @throws SnowflakeSQLException Will be thrown if the execute request fails
 */
private static String federatedFlowStep3(SFLoginInput loginInput, String tokenUrl)
throws SnowflakeSQLException

{
  String oneTimeToken = "";
  try
  {
    URL url = new URL(tokenUrl);
    URI tokenUri = url.toURI();
    final HttpPost postRequest = new HttpPost(tokenUri);

    String userName;
    if (Strings.isNullOrEmpty(loginInput.getOKTAUserName()))
    {
      userName = loginInput.getUserName();
    }
    else
    {
      userName = loginInput.getOKTAUserName();
    }
    StringEntity params = new StringEntity("{\"username\":\"" +
                                           userName + "\",\"password\":\"" +
                                           loginInput.getPassword() + "\"}");
    postRequest.setEntity(params);

    HeaderGroup headers = new HeaderGroup();
    headers.addHeader(new BasicHeader(HttpHeaders.ACCEPT, "application/json"));
    headers.addHeader(new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"));
    postRequest.setHeaders(headers.getAllHeaders());

    final String idpResponse = HttpUtil.executeRequestWithoutCookies(
        postRequest,
        loginInput.getLoginTimeout(),
        0,
        null,
        loginInput.getOCSPMode());

    logger.debug("user is authenticated against {}.",
                 loginInput.getAuthenticator());

    // session token is in the data field of the returned json response
    final JsonNode jsonNode = mapper.readTree(idpResponse);
    oneTimeToken = jsonNode.get("cookieToken").asText();
  }
  catch (IOException | URISyntaxException ex)
  {
    handleFederatedFlowError(loginInput, ex);
  }
  return oneTimeToken;
}
 

/**
 * Adds the given header, if not present (otherwise does nothing).
 *
 * @param headers the headers where to add the new one.
 * @param name the name of the header to add.
 * @param value the value of the header to add.
 */
public static void addIfNotPresent(final HeaderGroup headers, final WarcHeader.Name name, final String value) {
	if (! headers.containsHeader(name.value)) headers.addHeader(new WarcHeader(name, value));
}
 

/**
 * Returns the first header of given name.
 *
 * @param headers the headers to search from.
 * @param name the name of the header to lookup.
 * @return the header.
 */
public static Header getFirstHeader(final HeaderGroup headers, final WarcHeader.Name name) {
	return headers.getFirstHeader(name.value);
}
 

/** BUilds a record, optionally given the warcHeaders.
 *
 * @param warcHeaders the WARC headers, may be {@code null}.
 * @see AbstractWarcRecord#AbstractWarcRecord(URI,HeaderGroup)
 */
public AbstractWarcRecord(final HeaderGroup warcHeaders) {
	this(null, warcHeaders);
}
 

/** Returns the WARC headers.
 *
 * @return the WARC headers.
 */
public HeaderGroup getWarcHeaders();