下面列出了org.apache.http.client.protocol.HttpClientContext#adapt ( ) 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
@Override
public void process(HttpRequest request, HttpContext context) throws HttpException, IOException {
HttpClientContext clientContext = HttpClientContext.adapt(context);
AuthState authState = clientContext.getTargetAuthState();
// If there's no auth scheme available yet, try to initialize it preemptively
if (authState.getAuthScheme() == null) {
CredentialsProvider credsProvider = clientContext.getCredentialsProvider();
HttpHost targetHost = clientContext.getTargetHost();
Credentials creds = credsProvider.getCredentials(
new AuthScope(targetHost.getHostName(), targetHost.getPort()));
if (creds != null) {
authState.update(new BasicScheme(), creds);
} else {
if (log.isDebugEnabled()) {
log.debug("PreemptiveAuthInterceptor: No credentials for preemptive authentication");
}
}
}
}
@Override
public boolean retryRequest(HttpResponse response, int executionCount, HttpContext ctx) {
log.fine(() -> String.format("retryRequest(responseCode='%s', executionCount='%d', ctx='%s'",
response.getStatusLine().getStatusCode(), executionCount, ctx));
HttpClientContext clientCtx = HttpClientContext.adapt(ctx);
if (!predicate.test(response, clientCtx)) {
log.fine(() -> String.format("Not retrying for '%s'", ctx));
return false;
}
if (executionCount > maxRetries) {
log.fine(() -> String.format("Max retries exceeded for '%s'", ctx));
retryFailedConsumer.onRetryFailed(response, executionCount, clientCtx);
return false;
}
Duration delay = delaySupplier.getDelay(executionCount);
log.fine(() -> String.format("Retrying after %s for '%s'", delay, ctx));
retryInterval.set(delay.toMillis());
retryConsumer.onRetry(response, delay, executionCount, clientCtx);
return true;
}
@Override
public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
HttpClientContext clientContext = HttpClientContext.adapt(context);
AuthState authState = clientContext.getTargetAuthState();
if (authState.getAuthScheme() == null) {
CredentialsProvider credsProvider = clientContext.getCredentialsProvider();
HttpHost targetHost = clientContext.getTargetHost();
Credentials creds = credsProvider.getCredentials(new AuthScope(targetHost.getHostName(), targetHost.getPort()));
if (creds != null) {
authState.update(new BasicScheme(), creds);
}
}
}
@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
if (executionCount >= 3) {// 如果已经重试了3次,就放弃
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接,那么就重试
return true;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof InterruptedIOException) {// 超时
return true;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return false;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
return false;
}
if (exception instanceof SSLException) {// ssl握手异常
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的,就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
return false;
}
@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
if (executionCount >= 3) {// 如果已经重试了3次,就放弃
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接,那么就重试
return true;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof InterruptedIOException) {// 超时
return true;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return false;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
return false;
}
if (exception instanceof SSLException) {// ssl握手异常
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的,就再次尝试
if (request instanceof HttpEntityEnclosingRequest) {
return false;
}
return false;
}
@Override
public boolean retryRequest(
IOException exception,
int executionCount,
HttpContext context) {
if (executionCount > retryExecutionCount) {
return false;
}
if (exception instanceof InterruptedIOException) {
return false;
}
if (exception instanceof UnknownHostException) {
return false;
}
if (exception instanceof ConnectTimeoutException) {
return true;
}
if (exception instanceof SSLException) {
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
if (idempotent) {
// Retry if the request is considered idempotent
return true;
}
return false;
}
@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
if (executionCount >= 3) {// 如果已经重试了3次,就放弃
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接,那么就重试
return true;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof InterruptedIOException) {// 超时
return true;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return false;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
return false;
}
if (exception instanceof SSLException) {// ssl握手异常
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的,就再次尝试
return !(request instanceof HttpEntityEnclosingRequest);
}
@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
if (executionCount >= 3) {// 如果已经重试了3次,就放弃
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接,那么就重试
return true;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof InterruptedIOException) {// 超时
return true;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return false;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
return false;
}
if (exception instanceof SSLException) {// ssl握手异常
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的,就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
return false;
}
@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
if (executionCount >= 5) {// 如果已经重试了5次,就放弃
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接,那么就重试
return true;
}
if (exception instanceof InterruptedIOException) {// 超时
return false;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return false;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
return false;
}
if (exception instanceof SSLException) {// SSL握手异常
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的,就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
return false;
}
@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
if (executionCount >= 3) {// 如果已经重试了3次,就放弃
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接,那么就重试
return true;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof InterruptedIOException) {// 超时
return true;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return false;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
return false;
}
if (exception instanceof SSLException) {// ssl握手异常
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的,就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
return false;
}
@Override
protected HttpHost determineProxy(HttpHost host, HttpRequest request, HttpContext context) throws HttpException {
HttpClientContext httpClientContext = HttpClientContext.adapt(context);
Proxy proxy = proxyPlanner.determineProxy(host, request, context, ipPool, crawlerSession);
if (proxy == null) {
return null;
}
if (log.isDebugEnabled()) {
log.debug("{} 当前使用IP为:{}:{}", host.getHostName(), proxy.getIp(), proxy.getPort());
}
context.setAttribute(VSCRAWLER_AVPROXY_KEY, proxy);
crawlerSession.setExtInfo(VSCRAWLER_AVPROXY_KEY, proxy);
if (proxy.getAuthenticationHeaders() != null) {
for (Header header : proxy.getAuthenticationHeaders()) {
request.addHeader(header);
}
}
if (StringUtils.isNotEmpty(proxy.getUsername()) && StringUtils.isNotEmpty(proxy.getPassword())) {
BasicCredentialsProvider credsProvider1 = new BasicCredentialsProvider();
httpClientContext.setCredentialsProvider(credsProvider1);
credsProvider1.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(proxy.getUsername(), proxy.getPassword()));
}
return new HttpHost(proxy.getIp(), proxy.getPort());
}
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
// Do not retry if over max retry count
if (executionCount >= maxRetries) {
return false;
}
// Do not retry calls to the github api
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
for (Header header : request.getHeaders("Host")) {
if (header.getValue().contains("github"))
return false;
}
// Do not retry calls that are not idempotent - posts in our case
// currently, we make 2 types of posts:
// 1) posting blobs - this is idempotent
// 2) posting commit of blob ids - this is not idempotent and we need to fall back to the logic
// in DeltaImporter2Publisher.commitBlobPostings
boolean idempotent = !(request.getRequestLine().getUri().contains("commit"));
if (idempotent) { // Retry if the request is considered idempotent
double wait = Math.pow(retryDelayFactor, executionCount);
System.err.println("Request failed. Retrying request in " + Math.round(wait) + " seconds");
try {
Thread.sleep((long) wait*1000);
return true;
} catch (InterruptedException e) {
e.printStackTrace();
return false;
}
}
return false;
}
/**
* 重试(如果请求是幂等的,就再次尝试)
*
* @param tryTimes 重试次数
* @param retryWhenInterruptedIO 连接拒绝时,是否重试
* @return 返回当前对象
*/
public HCB retry(final int tryTimes, final boolean retryWhenInterruptedIO){
// 请求重试处理
HttpRequestRetryHandler httpRequestRetryHandler = new HttpRequestRetryHandler() {
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
if (executionCount >= tryTimes) {// 如果已经重试了n次,就放弃
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接,那么就重试
return true;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof InterruptedIOException) {// 超时
//return false;
return retryWhenInterruptedIO;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return true;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
return false;
}
if (exception instanceof SSLException) {// SSL握手异常
return false;
}
HttpClientContext clientContext = HttpClientContext .adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的,就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
return false;
}
};
this.setRetryHandler(httpRequestRetryHandler);
return this;
}
/**
* This method sets a custom HttpRequestRetryHandler in order to enable a custom
* exception recovery mechanism.
*
* @return A HttpRequestRetryHandler representing handling of the retryHandler.
*/
private static HttpRequestRetryHandler retryHandler(AviCredentials creds) {
return (exception, executionCount, context) -> {
if (executionCount >= creds.getNumApiRetries()) {
// Do not retry if over max retry count
return false;
}
if (exception instanceof InterruptedIOException) {
// Timeout
return false;
}
if (exception instanceof UnknownHostException) {
// Unknown host
return false;
}
if (exception instanceof SSLException) {
// SSL handshake exception
return false;
}
if (exception instanceof HttpHostConnectException) {
return true;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
if (idempotent) {
// Retry if the request is considered idempotent
return true;
}
return false;
};
}
@Override
public boolean retryRequest(final IOException exception, final int executionCount, final HttpContext context) {
log.debug("Retry http request {} out of {}", executionCount, this.retryCount);
if (executionCount > this.retryCount) {
log.debug("Do not retry, over max retry count");
return false;
}
if (!isRetriable(exception)) {
if (log.isDebugEnabled()) {
log.debug("Do not retry, non retriable class {}", exception.getClass().getName());
}
return false;
}
final HttpClientContext clientContext = HttpClientContext.adapt(context);
final HttpRequest request = clientContext.getRequest();
if (requestIsAborted(request)) {
log.debug("Do not retry, request was aborted");
return false;
}
if (handleAsIdempotent(clientContext)) {
log.debug("Retry, request should be idempotent");
return true;
}
log.debug("Do not retry, no allow rules matched");
return false;
}
@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
if (executionCount >= 5) {// 如果已经重试了5次,就放弃
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接,那么就重试
return true;
}
if (exception instanceof InterruptedIOException) {// 超时
return false;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return false;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
return false;
}
if (exception instanceof SSLException) {// SSL握手异常
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的,就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
return false;
}
public CloseableHttpClient createHttpClient(int maxTotal, int maxPerRoute, int maxRoute,
String hostname, int port) {
ConnectionSocketFactory plainsf = PlainConnectionSocketFactory
.getSocketFactory();
LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory
.getSocketFactory();
Registry<ConnectionSocketFactory> registry = RegistryBuilder
.<ConnectionSocketFactory>create().register("http", plainsf)
.register("https", sslsf).build();
_cm = new PoolingHttpClientConnectionManager(
registry);
// 将最大连接数增加
_cm.setMaxTotal(maxTotal);
// 将每个路由基础的连接增加
_cm.setDefaultMaxPerRoute(maxPerRoute);
HttpHost httpHost = new HttpHost(hostname, port);
// 将目标主机的最大连接数增加
_cm.setMaxPerRoute(new HttpRoute(httpHost), maxRoute);
// 请求重试处理
HttpRequestRetryHandler httpRequestRetryHandler = new HttpRequestRetryHandler() {
public boolean retryRequest(IOException exception,
int executionCount, HttpContext context) {
if (executionCount >= _maxRetryTimes) {
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接,那么就重试
return true;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof InterruptedIOException) {// 超时
return false;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return false;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
return false;
}
if (exception instanceof SSLException) {// SSL握手异常
return false;
}
HttpClientContext clientContext = HttpClientContext
.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的,就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
return false;
}
};
CloseableHttpClient httpClient = HttpClients.custom()
.setConnectionManager(_cm)
.setRetryHandler(httpRequestRetryHandler).build();
return httpClient;
}
public static CloseableHttpClient createHttpClient() {
ConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
ConnectionSocketFactory sslsf = new EasySSLConnectionSocketFactory();
//SSLConnectionSocketFactory.getSocketFactory();
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", plainsf)
.register("https", sslsf)
.build();
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
// 将最大连接数增加到200
cm.setMaxTotal(200);
// 将每个路由基础的连接增加到20
cm.setDefaultMaxPerRoute(20);
//请求重试处理
HttpRequestRetryHandler httpRequestRetryHandler = new HttpRequestRetryHandler() {
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
if (executionCount >= 5) {// 如果已经重试了5次,就放弃
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接,那么就重试
return true;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof InterruptedIOException) {// 超时
return false;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return false;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
return false;
}
if (exception instanceof SSLException) {// ssl握手异常
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的,就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
return false;
}
};
CloseableHttpClient httpClient = HttpClients.custom()
.setConnectionManager(cm)
.setRetryHandler(httpRequestRetryHandler)
.build();
return httpClient;
}
public static HttpClientBuilder cliBuilder(int timeout) {
HttpRequestRetryHandler retryHander = new HttpRequestRetryHandler() {
@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
if (executionCount > 3) {
// Do not retry if over max retry count
return false;
}
if (exception instanceof java.net.SocketTimeoutException) {
//特殊处理
return true;
}
if (exception instanceof InterruptedIOException) {
// Timeout
return true;
}
if (exception instanceof UnknownHostException) {
// Unknown host
return false;
}
if (exception instanceof SSLException) {
// SSL handshake exception
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
if (idempotent) {
// Retry if the request is considered idempotent
return true;
}
return false;
}
};
RedirectStrategy redirectStrategy = new SeimiRedirectStrategy();
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(timeout).setConnectionRequestTimeout(timeout).setSocketTimeout(timeout).build();
PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = HttpClientConnectionManagerProvider.getHcPoolInstance();
return HttpClients.custom().setDefaultRequestConfig(requestConfig).setConnectionManager(poolingHttpClientConnectionManager)
.setRedirectStrategy(redirectStrategy).setRetryHandler(retryHander);
}
/**
* Creates custom retry handler to be used if HTTP exception happens
*
* @return retry handler
*/
private HttpRequestRetryHandler getRetryHandler() {
final HttpRequestRetryHandler myRetryHandler = new HttpRequestRetryHandler() {
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
if (executionCount >= connectionConfiguration.getExecutionCount()) {
// Do not retry if over max retry count
LOG.debug("Execution count {} is bigger than threshold. Stop", executionCount);
return false;
}
if (exception instanceof NoHttpResponseException) {
LOG.debug("NoHttpResponseException exception. Retry count {}", executionCount);
return true;
}
if (exception instanceof UnknownHostException) {
LOG.debug("UnknownHostException. Retry count {}", executionCount);
return true;
}
if (exception instanceof ConnectTimeoutException) {
LOG.debug("ConnectTimeoutException. Retry count {}", executionCount);
return true;
}
if (exception instanceof SocketTimeoutException
|| exception.getClass() == SocketTimeoutException.class
|| exception.getClass().isInstance(SocketTimeoutException.class)) {
// Connection refused
LOG.debug("socketTimeoutException Retry count {}", executionCount);
return true;
}
if (exception instanceof InterruptedIOException) {
// Timeout
LOG.debug("InterruptedIOException Retry count {}", executionCount);
return true;
}
if (exception instanceof SSLException) {
LOG.debug("SSLException Retry count {}", executionCount);
return true;
}
final HttpClientContext clientContext = HttpClientContext.adapt(context);
final HttpRequest request = clientContext.getRequest();
boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
if (idempotent) {
LOG.debug("HttpEntityEnclosingRequest. Retry count {}", executionCount);
return true;
}
LOG.debug("Retry stopped. Retry count {}", executionCount);
return false;
}
};
return myRetryHandler;
}