@Override
public void process(HttpRequest request, HttpContext context) throws HttpException, IOException {
    AuthState authState = (AuthState) context.getAttribute(ClientContext.TARGET_AUTH_STATE);

    if (authState.getAuthScheme() == null) {
        AuthScheme authScheme = (AuthScheme) context.getAttribute("preemptive-auth");
        CredentialsProvider credsProvider = (CredentialsProvider) context
                .getAttribute(ClientContext.CREDS_PROVIDER);
        HttpHost targetHost = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST);
        if (authScheme != null) {
            Credentials creds = credsProvider
                    .getCredentials(new AuthScope(targetHost.getHostName(), targetHost.getPort()));
            if (creds == null) {
                throw new HttpException("No credentials for preemptive authentication");
            }
            authState.update(authScheme, creds);
        }
    }
}
 

@Test
public void executeAsyncDelegatesToConfiguredAsyncClient() {

    // given
    HttpClient client = Mockito.spy(createDefaultTestObject());
    CloseableHttpAsyncClient asyncClient = mockAsyncClient(client);

    BatchRequest request = createDefaultTestBatchRequest();

    // when
    client.executeAsync(request, createMockTestResultHandler());

    // then
    verify(client).getAsyncClient();
    verify(asyncClient).execute(
            any(HttpAsyncRequestProducer.class),
            any(HttpAsyncResponseConsumer.class),
            any(HttpContext.class),
            any(FutureCallback.class));

}
 

@Override
public CloseableHttpResponse execute(
    HttpHost target, HttpRequest request, HttpContext context) throws IOException, ClientProtocolException {
    Subsegment subsegment = getRecorder().beginSubsegment(target.getHostName());
    try {
        if (null != subsegment) {
            TracedHttpClient.addRequestInformation(subsegment, request, TracedHttpClient.getUrl(target, request));
        }
        CloseableHttpResponse response = super.execute(target, request, context);
        if (null != subsegment) {
            TracedResponseHandler.addResponseInformation(subsegment, response);
        }
        return response;
    } catch (Exception e) {
        if (null != subsegment) {
            subsegment.addException(e);
        }
        throw e;
    } finally {
        if (null != subsegment) {
            getRecorder().endSubsegment();
        }
    }
}
 

@Override
public HttpUriRequest getRedirect(HttpRequest request, HttpResponse response, HttpContext context) throws ProtocolException {
    URI uri = getLocationURI(request, response, context);
    String method = request.getRequestLine().getMethod();
    if (HttpConstants.POST.equalsIgnoreCase(method)) {
        try {
            HttpRequestWrapper httpRequestWrapper = (HttpRequestWrapper) request;
            httpRequestWrapper.setURI(uri);
            httpRequestWrapper.removeHeaders("Content-Length");
            return httpRequestWrapper;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return new HttpPost(uri);
    } else {
        return new HttpGet(uri);
    }
}
 

@Override
public void handle(HttpRequest httpRequest, HttpResponse httpResponse, HttpContext httpContext) throws IOException {
    lock.lock();
    try {
        HttpEntity entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity();
        String content = EntityUtils.toString(entity);

        replies.add(content);
        if (replies.size() == expected) {
            receivedExpectedMessages.signal();
        }

        httpResponse.setStatusCode(HttpStatus.SC_OK);
    } finally {
        lock.unlock();
    }
}
 

/**
 * UnixHttpClient can execute the HttpRequest with the given host
 * and context.
 * @throws IOException If something goes wrong.
 */
@Test
public void executesRequestWithHostAndContext() throws IOException {
    final HttpHost host = new HttpHost("127.0.0.1");
    final HttpRequest req = Mockito.mock(HttpRequest.class);
    final HttpContext context = Mockito.mock(HttpContext.class);
    final HttpResponse resp = Mockito.mock(HttpResponse.class);
    final HttpClient decorated = Mockito.mock(HttpClient.class);
    Mockito.when(
            decorated.execute(host, req, context)
    ).thenReturn(resp);

    final HttpClient unix = new UnixHttpClient(() -> decorated);
    MatcherAssert.assertThat(
        unix.execute(host, req, context), Matchers.is(resp)
    );
    Mockito.verify(
        decorated, Mockito.times(1)
    ).execute(host, req, context);
}
 

@Override
public HttpUriRequest getRedirect(final HttpRequest request, final HttpResponse response, final HttpContext context)
        throws ProtocolException
{
    final String method = request.getRequestLine().getMethod();
    HttpUriRequest redirect = super.getRedirect(request, response, context);
    if (HttpPost.METHOD_NAME.equalsIgnoreCase(method))
    {
        HttpPost httpPostRequest = new HttpPost(redirect.getURI());
        if (request instanceof HttpEntityEnclosingRequest)
        {
            httpPostRequest.setEntity(((HttpEntityEnclosingRequest) request).getEntity());
        }
        return httpPostRequest;
    }
    return redirect;
}
 

@Test(expected = AbortedException.class)
public void
clientExecutionTimeoutEnabled_aborted_exception_occurs_timeout_not_expired()
        throws Exception {
    ClientConfiguration config = new ClientConfiguration()
            .withClientExecutionTimeout(CLIENT_EXECUTION_TIMEOUT)
            .withMaxErrorRetry(0);
    ConnectionManagerAwareHttpClient rawHttpClient =
            createRawHttpClientSpy(config);

    doThrow(new AbortedException()).when(rawHttpClient).execute(any
            (HttpRequestBase.class), any(HttpContext.class));

    httpClient = new AmazonHttpClient(config, rawHttpClient, null);

    execute(httpClient, createMockGetRequest());
}
 

@Test
public void requestTimeoutEnabled_RequestCompletesWithinTimeout_EntityNotBufferedForStreamedResponse()
        throws Exception {
    ClientConfiguration config = new ClientConfiguration().withRequestTimeout(5 * 1000);
    ConnectionManagerAwareHttpClient rawHttpClient = createRawHttpClientSpy(config);

    HttpResponseProxy responseProxy = createHttpResponseProxySpy();
    doReturn(responseProxy).when(rawHttpClient).execute(any(HttpRequestBase.class), any(HttpContext.class));

    httpClient = new AmazonHttpClient(config, rawHttpClient, null);

    try {
        httpClient.requestExecutionBuilder().request(createMockGetRequest()).execute(new ErrorDuringUnmarshallingResponseHandler().leaveConnectionOpen());
        fail("Exception expected");
    } catch (AmazonClientException e) {
    }

    assertResponseWasNotBuffered(responseProxy);
}
 

/**
 * UnixHttpClient can execute the HttpRequest with the given host,
 * response handler and context.
 * @throws IOException If something goes wrong.
 */
@Test
public void executesRequestWithHostHandlerAndContext() throws IOException {
    final HttpHost host = new HttpHost("127.0.0.1");
    final HttpRequest req = Mockito.mock(HttpRequest.class);
    final ResponseHandler<String> handler = Mockito.mock(
        ResponseHandler.class
    );
    final HttpContext context = Mockito.mock(HttpContext.class);
    final HttpClient decorated = Mockito.mock(HttpClient.class);
    Mockito.when(
        decorated.execute(host, req, handler, context)
    ).thenReturn("executed");

    final HttpClient unix = new UnixHttpClient(() -> decorated);
    MatcherAssert.assertThat(
        unix.execute(host, req, handler, context),
        Matchers.equalTo("executed")
    );
    Mockito.verify(
        decorated, Mockito.times(1)
    ).execute(host, req, handler, context);
}
 

@Test
void testDoHttpHead() throws Exception
{
    CloseableHttpResponse closeableHttpResponse = mock(CloseableHttpResponse.class);
    HttpContext context = null;
    when(closeableHttpResponse.getEntity()).thenReturn(null);
    StatusLine statusLine = mock(StatusLine.class);
    int statusCode = HttpStatus.SC_MOVED_PERMANENTLY;
    Header[] headers = { header };
    when(closeableHttpResponse.getAllHeaders()).thenReturn(headers);
    when(statusLine.getStatusCode()).thenReturn(statusCode);
    when(closeableHttpResponse.getStatusLine()).thenReturn(statusLine);
    when(closeableHttpClient.execute(isA(HttpHead.class), eq(context)))
            .thenAnswer(getAnswerWithSleep(closeableHttpResponse));
    HttpResponse httpResponse = httpClient.doHttpHead(URI_TO_GO);
    assertEquals(HEAD, httpResponse.getMethod());
    assertEquals(URI_TO_GO, httpResponse.getFrom());
    assertEquals(statusCode, httpResponse.getStatusCode());
    assertThat(httpResponse.getResponseTimeInMs(), greaterThan(0L));
    assertThat(httpResponse.getResponseHeaders(), is(equalTo(headers)));
}
 

/**
 * {@inheritDoc}
 */
@Override
public HttpRoute determineRoute(final HttpHost host, final HttpRequest request,
        final HttpContext context) throws HttpException {

    Boolean ignoreNoProxy = (Boolean) context.getAttribute(IGNORE_NO_PROXY);
    URI requestUri = (URI) context.getAttribute(REQUEST_URI);

    if (proxyHost != null &&
            (Boolean.TRUE.equals(ignoreNoProxy) || useProxyFor(requestUri))) {
        if (log.isDebugEnabled()) {
            log.debug("Using proxy: " + proxyHost);
        }
        return super.determineRoute(host, request, context);
    }
    if (log.isDebugEnabled()) {
        log.debug("Using a direct connection (no proxy)");
    }
    // Return direct route
    return new HttpRoute(host);
}
 

@Test
void shouldSaveConnectionDetailsForSecuredConnectionAndDoNotCheckStalenessForResponsesWithEntity()
{
    String protocol = "TLSv1.3";
    SSLSession sslSession = mock(SSLSession.class);
    when(sslSession.getProtocol()).thenReturn(protocol);
    HttpContext context = mock(HttpContext.class);
    ManagedHttpClientConnection connection = mock(ManagedHttpClientConnection.class);
    when(context.getAttribute(HttpCoreContext.HTTP_CONNECTION)).thenReturn(connection);
    when(connection.isOpen()).thenReturn(true);
    when(connection.getSSLSession()).thenReturn(sslSession);
    HttpResponse response = mock(HttpResponse.class);
    when(response.getEntity()).thenReturn(mock(HttpEntity.class));
    interceptor.process(response, context);
    verify(httpTestContext).putConnectionDetails(argThat(connectionDetails -> connectionDetails.isSecure()
            && protocol.equals(connectionDetails.getSecurityProtocol())));
    verify(connection, never()).isStale();
}
 

@Override
public boolean retryRequest(final IOException exception, final int executionCount, final HttpContext context) {
    final Throwable cause = ExceptionUtils.getRootCause(exception);
    if(cause instanceof RuntimeException) {
        log.error(String.format("Cancel retry request with execution count %d for failure %s", executionCount, cause));
        return false;
    }
    final boolean retry = super.retryRequest(exception, executionCount, context);
    if(retry) {
        log.info(String.format("Retry request with failure %s", exception));
    }
    else {
        log.warn(String.format("Cancel retry request with execution count %d for failure %s", executionCount, exception));
    }
    return retry;
}
 

@Override
public HttpUriRequest getRedirect(HttpRequest request, HttpResponse response, HttpContext context) throws ProtocolException {
    URI uri = getLocationURI(request, response, context);
    String method = request.getRequestLine().getMethod();
    if ("post".equalsIgnoreCase(method)) {
        try {
            HttpRequestWrapper httpRequestWrapper = (HttpRequestWrapper) request;
            httpRequestWrapper.setURI(uri);
            httpRequestWrapper.removeHeaders("Content-Length");
            return httpRequestWrapper;
        } catch (Exception e) {

            log.error("强转为HttpRequestWrapper出错");
        }
        return new HttpPost(uri);
    } else {
        return new HttpGet(uri);
    }
}
 

@Override
public void process(final HttpRequest request, final HttpContext context) {
    if(StringUtils.contains(request.getRequestLine().getUri(), "b2_authorize_account")) {
        // Skip setting token for
        if(log.isDebugEnabled()) {
            log.debug("Skip setting token in b2_authorize_account");
        }
        return;
    }
    switch(request.getRequestLine().getMethod()) {
        case "POST":
            // Do not override Authorization header for upload requests with upload URL token
            if(StringUtils.contains(request.getRequestLine().getUri(), "b2_upload_part")
                || StringUtils.contains(request.getRequestLine().getUri(), "b2_upload_file")) {
                break;
            }
        default:
            if(StringUtils.isNotBlank(authorizationToken)) {
                request.removeHeaders(HttpHeaders.AUTHORIZATION);
                request.addHeader(HttpHeaders.AUTHORIZATION, authorizationToken);
            }
    }
}
 

@Test
public void clientExecutionTimeoutEnabled_RequestCompletesWithinTimeout_TaskCanceledAndEntityBuffered()
        throws Exception {
    ClientConfiguration config = new ClientConfiguration().withClientExecutionTimeout(CLIENT_EXECUTION_TIMEOUT)
            .withMaxErrorRetry(0);
    ConnectionManagerAwareHttpClient rawHttpClient = createRawHttpClientSpy(config);

    HttpResponseProxy responseProxy = createHttpResponseProxySpy();
    doReturn(responseProxy).when(rawHttpClient).execute(any(HttpRequestBase.class), any(HttpContext.class));

    httpClient = new AmazonHttpClient(config, rawHttpClient, null);

    try {
        execute(httpClient, createMockGetRequest());
        fail("Exception expected");
    } catch (AmazonClientException e) {
        NullResponseHandler.assertIsUnmarshallingException(e);
    }

    assertResponseIsBuffered(responseProxy);
    ScheduledThreadPoolExecutor requestTimerExecutor = httpClient.getClientExecutionTimer().getExecutor();
    assertTimerNeverTriggered(requestTimerExecutor);
    assertCanceledTasksRemoved(requestTimerExecutor);
    // Core threads should be spun up on demand. Since only one task was submitted only one
    // thread should exist
    assertEquals(1, requestTimerExecutor.getPoolSize());
    assertCoreThreadsShutDownAfterBeingIdle(requestTimerExecutor);
}
 

@Override
public void process(HttpResponse httpResponse, HttpContext httpContext) throws HttpException,
                                                                       IOException {
    SofaTraceContext sofaTraceContext = SofaTraceContextHolder.getSofaTraceContext();
    SofaTracerSpan httpClientSpan = sofaTraceContext.getCurrentSpan();
    //tag append
    super.appendHttpClientResponseSpanTags(httpResponse, httpClientSpan);
    //finish
    int statusCode = httpResponse.getStatusLine().getStatusCode();
    httpClientTracer.clientReceive(String.valueOf(statusCode));
}
 

private void configureRetryHandler(DefaultHttpClient httpClient) {
    httpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler() {
        public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
            return false;
        }
    });
}
 

/**
 * {@inheritDoc}
 */
@Override
public Socket createSocket(final HttpContext context) throws IOException {
    final HttpHost socksProxy = getSocksProxy(context);
    if (socksProxy != null) {
        return createSocketWithSocksProxy(socksProxy);
    }
    return super.createSocket(context);
}
 

@Override
public Proxy determineProxy(HttpHost host, HttpRequest request, HttpContext context, IPPool ipPool,
        CrawlerSession crawlerSession) {
    HttpClientContext httpClientContext = HttpClientContext.adapt(context);

    Proxy bind = (Proxy) context.getAttribute(VSCrawlerConstant.VSCRAWLER_AVPROXY_KEY);
    String accessUrl = null;
    if (request instanceof HttpRequestWrapper || request instanceof HttpGet) {
        accessUrl = HttpUriRequest.class.cast(request).getURI().toString();
    }
    if (!PoolUtil.isDungProxyEnabled(httpClientContext)) {
        log.info("{}不会被代理", accessUrl);
        return null;
    }

    if (bind == null || bind.isDisable()) {
        bind = ipPool.getIP(host.getHostName(), accessUrl);
    }

    if (bind == null) {
        return null;
    }

    log.info("{} 当前使用IP为:{}:{}", host.getHostName(), bind.getIp(), bind.getPort());
    // 将绑定IP放置到context,用于后置拦截器统计这个IP的使用情况
    return bind;
}
 

@Override
public HttpResponse execute(
    final HttpHost target, final HttpRequest request,
    final HttpContext context
) throws IOException {
    throw new UnsupportedOperationException();
}
 

@Override
public void process(HttpResponse httpResponse, HttpContext httpContext) throws HttpException,
                                                                       IOException {
    SofaTracerSpan httpClientSpan = (SofaTracerSpan) httpContext
        .getAttribute(CURRENT_ASYNC_HTTP_SPAN_KEY);
    //tag append
    super.appendHttpClientResponseSpanTags(httpResponse, httpClientSpan);
    //finish
    int statusCode = httpResponse.getStatusLine().getStatusCode();
    httpClientTracer.clientReceiveTagFinish(httpClientSpan, String.valueOf(statusCode));
}
 

@Test
public void testExecuteWithContext() throws IOException {
    HttpClient httpClient = HttpMocks.mockClient(builder -> builder.addResponse(OK_RESPONSE));
    HttpRequest request = Mockito.spy(new HttpGet());

    HttpResponse response = getResultFromExecution(httpClient, "dummy", client -> client.execute(null, request, (HttpContext) null));

    Mockito.verify(request, Mockito.never())
           .setHeader(Mockito.anyString(), Mockito.anyString());

    Assertions.assertEquals(200, response.getStatusLine()
                                         .getStatusCode());
}
 

public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException {
  HttpEntity entity = response.getEntity();
  if (entity != null) {
    Header ceheader = entity.getContentEncoding();
    if (ceheader != null) {
      HeaderElement[] codecs = ceheader.getElements();
      for (HeaderElement h : codecs) {
        if (h.getName().equalsIgnoreCase("deflate")) {
          response.setEntity(new DeflateDecompressingEntity(response.getEntity()));
          return;
        }
      }
    }
  }
}
 

@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
    boolean retry = true;

    Boolean b = (Boolean) context.getAttribute(ExecutionContext.HTTP_REQ_SENT);
    boolean sent = (b != null && b);

    if (executionCount > maxRetries) {
        // Do not retry if over max retry count
        retry = false;
    } else if (isInList(exceptionWhitelist, exception)) {
        // immediately retry if error is whitelisted
        retry = true;
    } else if (isInList(exceptionBlacklist, exception)) {
        // immediately cancel retry if the error is blacklisted
        retry = false;
    } else if (!sent) {
        // for most other errors, retry only if request hasn't been fully sent yet
        retry = true;
    }

    if (retry) {
        // resend all idempotent requests
        HttpUriRequest currentReq = (HttpUriRequest) context.getAttribute(ExecutionContext.HTTP_REQUEST);
        if (currentReq == null) {
            return false;
        }
    }

    if (retry) {
        SystemClock.sleep(retrySleepTimeMS);
    } else {
        exception.printStackTrace();
    }

    return retry;
}
 

@Override
public void process(HttpRequest httpRequest, HttpContext httpContext) throws HttpException,
                                                                     IOException {
    //lazy init
    RequestLine requestLine = httpRequest.getRequestLine();
    String methodName = requestLine.getMethod();
    //span generated
    SofaTracerSpan httpClientSpan = httpClientTracer.clientSend(methodName);
    super.appendHttpClientRequestSpanTags(httpRequest, httpClientSpan);
}
 

@Test
public void testUserAgentPrefixAndSuffixAreAdded() throws Exception {
    String prefix = "somePrefix", suffix = "someSuffix";
    Request<?> request = mockRequest(SERVER_NAME, HttpMethodName.PUT, URI_NAME, true);

    HttpResponseHandler<AmazonWebServiceResponse<Object>> handler = createStubResponseHandler();
    EasyMock.replay(handler);
    ClientConfiguration config =
            new ClientConfiguration().withUserAgentPrefix(prefix).withUserAgentSuffix(suffix);

    Capture<HttpRequestBase> capturedRequest = new Capture<HttpRequestBase>();

    EasyMock.reset(httpClient);
    EasyMock
            .expect(httpClient.execute(
                    EasyMock.capture(capturedRequest), EasyMock.<HttpContext>anyObject()))
            .andReturn(createBasicHttpResponse())
            .once();
    EasyMock.replay(httpClient);

    AmazonHttpClient client = new AmazonHttpClient(config, httpClient, null);

    client.requestExecutionBuilder().request(request).execute(handler);

    String userAgent = capturedRequest.getValue().getFirstHeader("User-Agent").getValue();
    Assert.assertTrue(userAgent.startsWith(prefix));
    Assert.assertTrue(userAgent.endsWith(suffix));
}
 

@Test
void shouldMarkValidationAsBrokenIfExceptionOccurs() throws IOException
{
    IOException ioException = new IOException();
    when(httpClient.execute(argThat(r -> HEAD.equals(r.getMethod())), any(HttpContext.class)))
        .thenThrow(ioException);
    ResourceValidation resourceValidation = new ResourceValidation(FIRST, CSS_SELECTOR);
    ResourceValidation result = resourceValidator.perform(resourceValidation);
    assertEquals(CheckStatus.BROKEN, result.getCheckStatus());
    verify(httpClient).execute(any(HttpUriRequest.class), any(HttpContext.class));
    verify(softAssert).recordFailedAssertion(eq("Exception occured during check of: https://vividus.org"),
            eq(ioException));
}
 

public final HttpContext createContext(URI uri,
        UsernamePasswordCredentials creds) throws Exception {
    CredentialsProvider credsProvider = new BasicCredentialsProvider();
    credsProvider.setCredentials(
            new AuthScope(uri.getHost(), uri.getPort()),
            creds);
    org.apache.http.HttpHost host = new org.apache.http.HttpHost(uri.getHost(), uri.getPort(), uri.getScheme());
    AuthCache authCache = new BasicAuthCache();
    BasicScheme basicAuth = new BasicScheme();
    authCache.put(host, basicAuth);
    HttpClientContext context1 = HttpClientContext.create();
    context1.setCredentialsProvider(credsProvider);
    context1.setAuthCache(authCache);
    return context1;
}