下面列出了org.apache.http.impl.client.HttpClientBuilder#setMaxConnPerRoute() 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
public HttpHelper(){
HttpClientBuilder builder = HttpClientBuilder.create();
builder.setMaxConnPerRoute(50);
builder.setMaxConnTotal(100);
// 创建支持忽略证书的https
try {
SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return true;
}
}).build();
httpclient = HttpClientBuilder.create()
.setSSLContext(sslContext)
.setConnectionManager(new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory> create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE))
.build()))
.build();
} catch (Throwable e) {
// ignore
}
}
/**
* Configures the http client.
*
* @param builder client builder to configure
*/
protected void configureClient(HttpClientBuilder builder)
{
if ( getRemoteHttpCacheAttributes().getMaxConnectionsPerHost() > 0 )
{
builder.setMaxConnTotal(getRemoteHttpCacheAttributes().getMaxConnectionsPerHost());
builder.setMaxConnPerRoute(getRemoteHttpCacheAttributes().getMaxConnectionsPerHost());
}
builder.setDefaultRequestConfig(RequestConfig.custom()
.setConnectTimeout(getRemoteHttpCacheAttributes().getConnectionTimeoutMillis())
.setSocketTimeout(getRemoteHttpCacheAttributes().getSocketTimeoutMillis())
// By default we instruct HttpClient to ignore cookies.
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.build());
}
public HttpClientBuilder clientBuilder() {
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
RequestConfig.Builder builder = RequestConfig.custom();
builder.setConnectionRequestTimeout(requestConnTimeout);
builder.setSocketTimeout(socketTimeout);
builder.setConnectTimeout(connTimeout);
httpClientBuilder.setDefaultRequestConfig(builder.build());
httpClientBuilder.setMaxConnPerRoute(maxConnPerRoute);
httpClientBuilder.setMaxConnTotal(maxConnTotal);
return httpClientBuilder;
}
public static byte[] getBytes(String url, int timeout) throws Exception {
long start = System.currentTimeMillis();
HttpClientBuilder builder = HttpClientBuilder.create();
builder.setMaxConnPerRoute(50);
builder.setMaxConnTotal(100);
CloseableHttpClient httpclient = builder.build();
URI uri = new URIBuilder(url).build();
RequestConfig config = custom().setConnectTimeout(timeout)
.setConnectionRequestTimeout(timeout)
.setSocketTimeout(timeout)
.build();
HttpGet httpGet = new HttpGet(uri);
HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(config);
CloseableHttpResponse response = httpclient.execute(httpGet, context);
try {
int statusCode = response.getStatusLine().getStatusCode();
long end = System.currentTimeMillis();
long cost = end - start;
if (logger.isWarnEnabled()) {
logger.warn("post " + url + ", cost : " + cost);
}
if (statusCode == HttpStatus.SC_OK) {
return EntityUtils.toByteArray(response.getEntity());
} else {
String errorMsg = EntityUtils.toString(response.getEntity());
throw new RuntimeException("requestGet remote error, url=" + uri.toString() + ", code=" + statusCode
+ ", error msg=" + errorMsg);
}
} finally {
response.close();
httpGet.releaseConnection();
}
}
/**
* Creates the <tt>HttpClientBuilder</tt> that will be used by this WebClient.
* Extensions may override this method in order to create a customized
* <tt>HttpClientBuilder</tt> instance (e.g. with a custom
* {@link org.apache.http.conn.ClientConnectionManager} to perform
* some tracking; see feature request 1438216).
* @return the <tt>HttpClientBuilder</tt> that will be used by this WebConnection
*/
protected HttpClientBuilder createHttpClientBuilder() {
final HttpClientBuilder builder = HttpClientBuilder.create();
builder.setRedirectStrategy(new HtmlUnitRedirectStrategie());
configureTimeout(builder, getTimeout());
configureHttpsScheme(builder);
builder.setMaxConnPerRoute(6);
builder.setConnectionManagerShared(true);
return builder;
}
public static byte[] getBytes(String url, int timeout) throws Exception {
long start = System.currentTimeMillis();
HttpClientBuilder builder = HttpClientBuilder.create();
builder.setMaxConnPerRoute(50);
builder.setMaxConnTotal(100);
CloseableHttpClient httpclient = builder.build();
URI uri = new URIBuilder(url).build();
RequestConfig config = custom().setConnectTimeout(timeout)
.setConnectionRequestTimeout(timeout)
.setSocketTimeout(timeout)
.build();
HttpGet httpGet = new HttpGet(uri);
HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(config);
CloseableHttpResponse response = httpclient.execute(httpGet, context);
try {
int statusCode = response.getStatusLine().getStatusCode();
long end = System.currentTimeMillis();
long cost = end - start;
if (logger.isWarnEnabled()) {
logger.warn("post " + url + ", cost : " + cost);
}
if (statusCode == HttpStatus.SC_OK) {
return EntityUtils.toByteArray(response.getEntity());
} else {
String errorMsg = EntityUtils.toString(response.getEntity());
throw new RuntimeException("requestGet remote error, url=" + uri.toString() + ", code=" + statusCode
+ ", error msg=" + errorMsg);
}
} finally {
response.close();
httpGet.releaseConnection();
}
}
@Override
public IHttpClient buildHttpClient(HttpClientConfig config)
{
HttpClientBuilder builder = HttpClientBuilder.create();
builder.setDefaultHeaders(config.createHeaders());
if (config.hasCookieStore())
{
builder.setDefaultCookieStore(config.getCookieStore());
}
if (config.hasCredentials())
{
AuthScope authScope = config.hasAuthScope() ? config.getAuthScope()
: ClientBuilderUtils.DEFAULT_AUTH_SCOPE;
CredentialsProvider credProvider = ClientBuilderUtils.createCredentialsProvider(authScope,
config.getCredentials());
builder.setDefaultCredentialsProvider(credProvider);
}
sslContextFactory.getSslContext(SSLConnectionSocketFactory.SSL, !config.isSslCertificateCheckEnabled())
.ifPresent(builder::setSSLContext);
if (!config.isSslHostnameVerificationEnabled())
{
builder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
}
builder.setConnectionManager(config.getConnectionManager());
builder.setMaxConnTotal(config.getMaxTotalConnections());
builder.setMaxConnPerRoute(config.getMaxConnectionsPerRoute());
builder.addInterceptorLast(config.getLastRequestInterceptor());
builder.addInterceptorLast(config.getLastResponseInterceptor());
builder.setRedirectStrategy(config.getRedirectStrategy());
builder.setRetryHandler(config.getHttpRequestRetryHandler());
Builder requestConfigBuilder = RequestConfig.custom();
requestConfigBuilder.setConnectionRequestTimeout(config.getConnectionRequestTimeout());
requestConfigBuilder.setConnectTimeout(config.getConnectTimeout());
requestConfigBuilder.setCircularRedirectsAllowed(config.isCircularRedirectsAllowed());
requestConfigBuilder.setSocketTimeout(config.getSocketTimeout());
Optional.ofNullable(config.getCookieSpec()).ifPresent(requestConfigBuilder::setCookieSpec);
builder.setDefaultRequestConfig(requestConfigBuilder.build());
builder.setDefaultSocketConfig(SocketConfig.copy(SocketConfig.DEFAULT)
.setSoTimeout(config.getSocketTimeout())
.build());
builder.setDnsResolver(config.getDnsResolver());
HttpClient httpClient = new HttpClient();
httpClient.setCloseableHttpClient(builder.build());
if (config.hasBaseUrl())
{
httpClient.setHttpHost(HttpHost.create(config.getBaseUrl()));
}
httpClient.setSkipResponseEntity(config.isSkipResponseEntity());
return httpClient;
}
private void download(BinlogFile binlogFile) throws Throwable {
String downloadLink = binlogFile.getDownloadLink();
String fileName = binlogFile.getFileName();
downloadLink = downloadLink.trim();
CloseableHttpClient httpClient = null;
if (downloadLink.startsWith("https")) {
HttpClientBuilder builder = HttpClientBuilder.create();
builder.setMaxConnPerRoute(50);
builder.setMaxConnTotal(100);
// 创建支持忽略证书的https
final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return true;
}
}).build();
httpClient = HttpClientBuilder.create()
.setSSLContext(sslContext)
.setConnectionManager(new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory> create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE))
.build()))
.build();
} else {
httpClient = HttpClientBuilder.create().setMaxConnPerRoute(50).setMaxConnTotal(100).build();
}
HttpGet httpGet = new HttpGet(downloadLink);
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(TIMEOUT)
.setConnectionRequestTimeout(TIMEOUT)
.setSocketTimeout(TIMEOUT)
.build();
httpGet.setConfig(requestConfig);
HttpResponse response = httpClient.execute(httpGet);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpResponseStatus.OK.code()) {
throw new RuntimeException("download failed , url:" + downloadLink + " , statusCode:" + statusCode);
}
saveFile(new File(destDir), "mysql-bin." + fileName, response);
}
/**
* Initialize an {@link HttpClient} for performing requests. Proxy settings will
* be read from the configuration and applied transparently.
*/
public HttpClientAdapter() {
Optional<SSLConnectionSocketFactory> sslSocketFactory = Optional.empty();
try {
SSLContext sslContext = SSLContext.getDefault();
sslSocketFactory = Optional.of(new SSLConnectionSocketFactory(
sslContext,
new String[]{"TLSv1", "TLSv1.1", "TLSv1.2"},
null,
SSLConnectionSocketFactory.getDefaultHostnameVerifier()));
}
catch (NoSuchAlgorithmException e) {
log.warn("No such algorithm. Using default context", e);
}
HttpClientBuilder clientBuilder = HttpClientBuilder.create();
sslSocketFactory.ifPresent(sf -> clientBuilder.setSSLSocketFactory(sf));
clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
Builder requestConfigBuilder = RequestConfig.custom()
.setConnectTimeout(Config.get().getInt(HTTP_CONNECTION_TIMEOUT, 5) * TO_MILLISECONDS)
.setSocketTimeout(Config.get().getInt(HTTP_SOCKET_TIMEOUT, 5 * 60) * TO_MILLISECONDS)
.setCookieSpec(CookieSpecs.IGNORE_COOKIES);
// Store the proxy settings
String proxyHostname = ConfigDefaults.get().getProxyHost();
if (!StringUtils.isBlank(proxyHostname)) {
int proxyPort = ConfigDefaults.get().getProxyPort();
proxyHost = new HttpHost(proxyHostname, proxyPort);
clientBuilder.setProxy(proxyHost);
String proxyUsername = ConfigDefaults.get().getProxyUsername();
String proxyPassword = ConfigDefaults.get().getProxyPassword();
if (!StringUtils.isBlank(proxyUsername) &&
!StringUtils.isBlank(proxyPassword)) {
Credentials proxyCredentials = new UsernamePasswordCredentials(
proxyUsername, proxyPassword);
credentialsProvider.setCredentials(new AuthScope(proxyHostname, proxyPort),
proxyCredentials);
}
// Explicitly exclude the NTLM authentication scheme
requestConfigBuilder = requestConfigBuilder.setProxyPreferredAuthSchemes(
Arrays.asList(AuthSchemes.DIGEST, AuthSchemes.BASIC));
clientBuilder.setProxyAuthenticationStrategy(new ProxyAuthenticationStrategy());
clientBuilder.setRoutePlanner(new CustomProxyRoutePlanner(proxyHost));
}
// Read proxy exceptions from the "no_proxy" config option
String noProxy = Config.get().getString(NO_PROXY);
if (!StringUtils.isBlank(noProxy)) {
for (String domain : Arrays.asList(noProxy.split(","))) {
noProxyDomains.add(domain.toLowerCase().trim());
}
}
requestConfig = requestConfigBuilder.build();
clientBuilder.setMaxConnPerRoute(Config.get().getInt(MAX_CONNCECTIONS, 1));
clientBuilder.setMaxConnTotal(Config.get().getInt(MAX_CONNCECTIONS, 1));
httpClient = clientBuilder.build();
}
protected FedX(Config config, Cache cache, Statistics statistics, EndpointListProvider endpointListProvider, SummaryProvider summaryProvider) {
this.config = config;
this.cache = cache;
this.statistics = statistics;
this.endpointListProvider = endpointListProvider;
this.summaryProvider = summaryProvider;
// initialize httpclient parameters
HttpClientBuilder httpClientBuilder = HttpClientBuilders.getSSLTrustAllHttpClientBuilder();
httpClientBuilder.setMaxConnTotal(config.getMaxHttpConnectionCount());
httpClientBuilder.setMaxConnPerRoute(config.getMaxHttpConnectionCountPerRoute());
//httpClientBuilder.evictExpiredConnections();
//httpClientBuilder.setConnectionReuseStrategy(new NoConnectionReuseStrategy());
//httpClientBuilder.setConnectionTimeToLive(1000, TimeUnit.MILLISECONDS);
//httpClientBuilder.disableAutomaticRetries();
// httpClientBuilder.setKeepAliveStrategy(new ConnectionKeepAliveStrategy(){
//
// @Override
// public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
// return 0;
// }});
httpClient = httpClientBuilder.build();
synchronized (log) {
if (monitoring == null) {
monitoring = MonitoringFactory.createMonitoring(config);
}
}
executor = Executors.newCachedThreadPool();
scheduler = new ControlledWorkerScheduler(config.getWorkerThreads(), "Evaluation Scheduler");
if (log.isDebugEnabled()) {
log.debug("Scheduler for async operations initialized with " + config.getWorkerThreads() + " worker threads.");
}
// initialize prefix declarations, if any
String prefixFile = config.getPrefixDeclarations();
if (prefixFile != null) {
prefixDeclarations = new Properties();
try {
prefixDeclarations.load(new FileInputStream(new File(prefixFile)));
} catch (IOException e) {
throw new FedXRuntimeException("Error loading prefix properties: " + e.getMessage());
}
}
open = true;
}
protected HttpClientBuilder getClientBuilder(boolean useSSL, String url,
Map<String, String> stringHeaders, int socketTimeout) {
HttpClientBuilder cb = HttpClientBuilder.create();
/*
* Setup SSL
*/
if (useSSL) {
/*
* All trusting SSL context
*/
try {
cb.setSSLContext(getSSLContext());
} catch (Exception e) {
throw new RuntimeException(e);
}
/*
* All trusting hostname verifier
*/
cb.setSSLHostnameVerifier(new HostnameVerifier() {
public boolean verify(String s, SSLSession sslSession) {
return true;
}
});
}
/*
* Add default headers
*/
ArrayList<BasicHeader> headers = new ArrayList<BasicHeader>(stringHeaders.size());
stringHeaders.forEach((k, v) -> headers.add(new BasicHeader(k, v)));
cb.setDefaultHeaders(headers);
/*
* Set socket timeout and transport threads
*/
SocketConfig sc = SocketConfig.custom().setSoTimeout(socketTimeout).build();
cb.setDefaultSocketConfig(sc);
cb.setMaxConnPerRoute(this.config.getThreads());
cb.setMaxConnTotal(this.config.getThreads());
return cb;
}
public HttpClient createClient() {
HttpClientBuilder clientBuilder;
if (connectionParams.useTlsConfigFromEnvironment()) {
clientBuilder = VespaHttpClientBuilder.create();
} else {
clientBuilder = HttpClientBuilder.create();
if (connectionParams.getSslContext() != null) {
setSslContext(clientBuilder, connectionParams.getSslContext());
} else {
SslContextBuilder builder = new SslContextBuilder();
if (connectionParams.getPrivateKey() != null && connectionParams.getCertificate() != null) {
builder.withKeyStore(connectionParams.getPrivateKey(), connectionParams.getCertificate());
}
if (connectionParams.getCaCertificates() != null) {
builder.withTrustStore(connectionParams.getCaCertificates());
}
setSslContext(clientBuilder, builder.build());
}
if (connectionParams.getHostnameVerifier() != null) {
clientBuilder.setSSLHostnameVerifier(connectionParams.getHostnameVerifier());
}
clientBuilder.setUserTokenHandler(context -> null); // https://stackoverflow.com/a/42112034/1615280
}
clientBuilder.setMaxConnPerRoute(1);
clientBuilder.setMaxConnTotal(1);
clientBuilder.setConnectionTimeToLive(connectionParams.getConnectionTimeToLive().getSeconds(), TimeUnit.SECONDS);
clientBuilder.setUserAgent(String.format("vespa-http-client (%s)", Vtag.currentVersion));
clientBuilder.setDefaultHeaders(Collections.singletonList(new BasicHeader(Headers.CLIENT_VERSION, Vtag.currentVersion)));
clientBuilder.disableContentCompression();
// Try to disable the disabling to see if system tests become stable again.
// clientBuilder.disableAutomaticRetries();
RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
requestConfigBuilder.setSocketTimeout(0);
if (connectionParams.getProxyHost() != null) {
requestConfigBuilder.setProxy(new HttpHost(connectionParams.getProxyHost(), connectionParams.getProxyPort()));
}
clientBuilder.setDefaultRequestConfig(requestConfigBuilder.build());
log.fine("Creating HttpClient: " + " ConnectionTimeout "
+ " SocketTimeout 0 secs "
+ " proxyhost (can be null) " + connectionParams.getProxyHost()
+ ":" + connectionParams.getProxyPort()
+ (useSsl ? " using ssl " : " not using ssl")
);
return clientBuilder.build();
}
@Override
public HttpClient createHttpClient(FilterConfig filterConfig) {
final String serviceRole = filterConfig.getInitParameter(PARAMETER_SERVICE_ROLE);
HttpClientBuilder builder;
GatewayConfig gatewayConfig = (GatewayConfig) filterConfig.getServletContext().getAttribute(GatewayConfig.GATEWAY_CONFIG_ATTRIBUTE);
GatewayServices services = (GatewayServices) filterConfig.getServletContext()
.getAttribute(GatewayServices.GATEWAY_SERVICES_ATTRIBUTE);
if (gatewayConfig != null && gatewayConfig.isMetricsEnabled()) {
MetricsService metricsService = services.getService(ServiceType.METRICS_SERVICE);
builder = metricsService.getInstrumented(HttpClientBuilder.class);
} else {
builder = HttpClients.custom();
}
// Conditionally set a custom SSLContext
SSLContext sslContext = createSSLContext(services, filterConfig, serviceRole);
if(sslContext != null) {
builder.setSSLSocketFactory(new SSLConnectionSocketFactory(sslContext));
}
if (Boolean.parseBoolean(System.getProperty(GatewayConfig.HADOOP_KERBEROS_SECURED))) {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UseJaasCredentials());
Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
.register(AuthSchemes.SPNEGO, new KnoxSpnegoAuthSchemeFactory(true))
.build();
builder.setDefaultAuthSchemeRegistry(authSchemeRegistry)
.setDefaultCookieStore(new HadoopAuthCookieStore(gatewayConfig))
.setDefaultCredentialsProvider(credentialsProvider);
} else {
builder.setDefaultCookieStore(new NoCookieStore());
}
builder.setKeepAliveStrategy( DefaultConnectionKeepAliveStrategy.INSTANCE );
builder.setConnectionReuseStrategy( DefaultConnectionReuseStrategy.INSTANCE );
builder.setRedirectStrategy( new NeverRedirectStrategy() );
builder.setRetryHandler( new NeverRetryHandler() );
int maxConnections = getMaxConnections( filterConfig );
builder.setMaxConnTotal( maxConnections );
builder.setMaxConnPerRoute( maxConnections );
builder.setDefaultRequestConfig(getRequestConfig(filterConfig, serviceRole));
// See KNOX-1530 for details
builder.disableContentCompression();
return builder.build();
}
private void download(BinlogFile binlogFile) throws Throwable {
String downloadLink = binlogFile.getDownloadLink();
String fileName = binlogFile.getFileName();
downloadLink = downloadLink.trim();
CloseableHttpClient httpClient = null;
if (downloadLink.startsWith("https")) {
HttpClientBuilder builder = HttpClientBuilder.create();
builder.setMaxConnPerRoute(50);
builder.setMaxConnTotal(100);
// 创建支持忽略证书的https
final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return true;
}
}).build();
httpClient = HttpClientBuilder.create()
.setSSLContext(sslContext)
.setConnectionManager(new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory> create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE))
.build()))
.build();
} else {
httpClient = HttpClientBuilder.create().setMaxConnPerRoute(50).setMaxConnTotal(100).build();
}
HttpGet httpGet = new HttpGet(downloadLink);
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(TIMEOUT)
.setConnectionRequestTimeout(TIMEOUT)
.setSocketTimeout(TIMEOUT)
.build();
httpGet.setConfig(requestConfig);
HttpResponse response = httpClient.execute(httpGet);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpResponseStatus.OK.code()) {
throw new RuntimeException("download failed , url:" + downloadLink + " , statusCode:" + statusCode);
}
saveFile(new File(destDir), "mysql-bin." + fileName, response);
}