下面列出了怎么用org.apache.http.config.SocketConfig的API类实例代码及写法，或者点击链接到github查看源代码。
@Override
protected CloseableHttpClient build(Payload payload) {
HttpClientBuilder httpClientBuilder = HttpClients.custom();
httpClientBuilder.setConnectionManager(poolingHttpClientConnectionManager);
if (payload.getUserAgent() != null) {
httpClientBuilder.setUserAgent(payload.getUserAgent());
} else {
httpClientBuilder.setUserAgent("");
}
httpClientBuilder.setConnectionManagerShared(true);
httpClientBuilder.setRedirectStrategy(new CustomRedirectStrategy());
SocketConfig.Builder socketConfigBuilder = SocketConfig.custom();
socketConfigBuilder.setSoKeepAlive(true).setTcpNoDelay(true);
socketConfigBuilder.setSoTimeout(payload.getSocketTimeout());
SocketConfig socketConfig = socketConfigBuilder.build();
httpClientBuilder.setDefaultSocketConfig(socketConfig);
poolingHttpClientConnectionManager.setDefaultSocketConfig(socketConfig);
httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(payload.getRetryTimes(), true));
reduceCookie(httpClientBuilder, payload);
return httpClientBuilder.build();
}
/**
* 创建httpclient连接池，并初始化httpclient
*/
public void init() {
try {
SSLContext sslcontext = SSLContexts.custom().loadTrustMaterial(null,
new TrustSelfSignedStrategy())
.build();
HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.getDefaultHostnameVerifier();
SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(
sslcontext, hostnameVerifier);
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.getSocketFactory())
.register("https", sslsf)
.build();
httpClientConnectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
// Increase max total connection to 200
httpClientConnectionManager.setMaxTotal(maxTotalPool);
// Increase default max connection per route to 20
httpClientConnectionManager.setDefaultMaxPerRoute(maxConPerRoute);
SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(socketTimeout).build();
httpClientConnectionManager.setDefaultSocketConfig(socketConfig);
} catch (Exception e) {
}
}
public SeedManager(final String joalConfFolder, final ObjectMapper mapper, final ApplicationEventPublisher publisher) throws IOException {
this.isSeeding = false;
this.joalFoldersPath = new JoalFoldersPath(Paths.get(joalConfFolder));
this.torrentFileProvider = new TorrentFileProvider(joalFoldersPath);
this.configProvider = new JoalConfigProvider(mapper, joalFoldersPath, publisher);
this.bitTorrentClientProvider = new BitTorrentClientProvider(configProvider, mapper, joalFoldersPath);
this.publisher = publisher;
this.connectionHandler = new ConnectionHandler();
final SocketConfig sc = SocketConfig.custom()
.setSoTimeout(30000)
.build();
final PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager();
connManager.setDefaultMaxPerRoute(100);
connManager.setMaxTotal(200);
connManager.setValidateAfterInactivity(1000);
connManager.setDefaultSocketConfig(sc);
this.httpClient = HttpClients.custom()
.setConnectionTimeToLive(1, TimeUnit.MINUTES)
.setConnectionManager(connManager)
.setConnectionManagerShared(true)
.build();
}
public static CrawlerHttpClientBuilder setDefault(CrawlerHttpClientBuilder proxyFeedBackDecorateHttpClientBuilder) {
SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).setSoLinger(-1).setSoReuseAddress(false)
.setSoTimeout(ProxyConstant.SOCKETSO_TIMEOUT).setTcpNoDelay(true).build();
proxyFeedBackDecorateHttpClientBuilder.setDefaultSocketConfig(socketConfig)
// .setSSLSocketFactory(sslConnectionSocketFactory)
// dungproxy0.0.6之后的版本,默认忽略https证书检查
.setRedirectStrategy(new LaxRedirectStrategy())
// 注意,这里使用ua生产算法自动产生ua,如果是mobile,可以使用
// com.virjar.vscrawler.core.net.useragent.UserAgentBuilder.randomAppUserAgent()
.setUserAgent(UserAgentBuilder.randomUserAgent())
// 对于爬虫来说,连接池没啥卵用,直接禁止掉(因为我们可能创建大量HttpClient,每个HttpClient一个连接池,会把系统socket资源撑爆)
// 测试开80个httpClient抓数据大概一个小时系统就会宕机
.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE)
// 现在有些网站的网络响应头部数据有非ASCII数据,httpclient默认只使用ANSI标准解析header,这可能导致带有中文的header数据无法解析
.setDefaultConnectionConfig(ConnectionConfig.custom().setCharset(Charsets.UTF_8).build());
return proxyFeedBackDecorateHttpClientBuilder;
}
public HttpRpcTransport build() {
initDefaults();
if (this.httpClient == null) {
httpClient = HttpClients.custom()
.setMaxConnTotal(maxConnections)
.setConnectionManagerShared(true)
.setSSLContext(sslContext)
.setDefaultSocketConfig(
SocketConfig.custom()
.setSoTimeout(1000)
.setSoReuseAddress(true)
.setSoKeepAlive(true)
.build()
)
.build();
}
return new HttpRpcTransport(target, rpcConverter, executorService, httpClient, context, onClose);
}
private void configureSocket(Socket socket, SocketConfig socketConfig) throws IOException {
socket.setSoTimeout(socketConfig.getSoTimeout());
socket.setReuseAddress(socketConfig.isSoReuseAddress());
socket.setTcpNoDelay(socketConfig.isTcpNoDelay());
socket.setKeepAlive(socketConfig.isSoKeepAlive());
if (socketConfig.getRcvBufSize() > 0) {
socket.setReceiveBufferSize(socketConfig.getRcvBufSize());
}
if (socketConfig.getSndBufSize() > 0) {
socket.setSendBufferSize(socketConfig.getSndBufSize());
}
if (socketConfig.getSoLinger() >= 0) {
socket.setSoLinger(true, socketConfig.getSoLinger());
}
}
public static RestOperations createCommonsHttpRestTemplate(int maxConnPerRoute, int maxConnTotal,
int connectTimeout, int soTimeout, int retryTimes, RetryPolicyFactory retryPolicyFactory) {
HttpClient httpClient = HttpClientBuilder.create()
.setMaxConnPerRoute(maxConnPerRoute)
.setMaxConnTotal(maxConnTotal)
.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(soTimeout).build())
.setDefaultRequestConfig(RequestConfig.custom().setConnectTimeout(connectTimeout).build())
.build();
ClientHttpRequestFactory factory = new HttpComponentsClientHttpRequestFactory(httpClient);
RestTemplate restTemplate = new RestTemplate(factory);
//set jackson mapper
for (HttpMessageConverter<?> hmc : restTemplate.getMessageConverters()) {
if (hmc instanceof MappingJackson2HttpMessageConverter) {
ObjectMapper objectMapper = createObjectMapper();
MappingJackson2HttpMessageConverter mj2hmc = (MappingJackson2HttpMessageConverter) hmc;
mj2hmc.setObjectMapper(objectMapper);
}
}
return (RestOperations) Proxy.newProxyInstance(RestOperations.class.getClassLoader(),
new Class[]{RestOperations.class},
new RetryableRestOperationsHandler(restTemplate, retryTimes, retryPolicyFactory));
}
protected AbstractMigrateUsersToIncludeUsernames() {
final String host = System.getProperty(COMMUNITY_HOST);
final int port = Integer.parseInt(System.getProperty(COMMUNITY_PORT));
final AciServerDetails.TransportProtocol transportProtocol = AciServerDetails.TransportProtocol.valueOf(System.getProperty(COMMUNITY_PROTOCOL));
serverDetails = new AciServerDetails(transportProtocol, host, port);
processorFactory = new ProcessorFactoryImpl(new Jaxb2MarshallerFactory());
final SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(HTTP_SOCKET_TIMEOUT).build();
final HttpClient httpClient = HttpClientBuilder.create()
.setMaxConnPerRoute(MAX_CONNECTIONS_PER_ROUTE)
.setMaxConnTotal(MAX_CONNECTIONS_TOTAL)
.setDefaultSocketConfig(socketConfig)
.build();
final AciHttpClient aciHttpClient = new AciHttpClientImpl(httpClient);
aciService = new AciServiceImpl(aciHttpClient, serverDetails);
}
protected final CloseableHttpClient getHttpClient(final boolean useSpnego) throws Exception {
final CredentialsProvider credsProvider = new BasicCredentialsProvider();
final HttpClientBuilder hcb = HttpClients.custom();
if (useSpnego) {
//SPNEGO/Kerberos setup
log.debug("SPNEGO activated");
final AuthSchemeProvider nsf = new SPNegoSchemeFactory(true);// new NegotiateSchemeProvider();
final Credentials jaasCreds = new JaasCredentials();
credsProvider.setCredentials(new AuthScope(null, -1, null, AuthSchemes.SPNEGO), jaasCreds);
credsProvider.setCredentials(new AuthScope(null, -1, null, AuthSchemes.NTLM), new NTCredentials("Guest", "Guest", "Guest",
"Guest"));
final Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider> create()
.register(AuthSchemes.SPNEGO, nsf).register(AuthSchemes.NTLM, new NTLMSchemeFactory()).build();
hcb.setDefaultAuthSchemeRegistry(authSchemeRegistry);
}
hcb.setDefaultCredentialsProvider(credsProvider);
hcb.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(10 * 1000).build());
final CloseableHttpClient httpClient = hcb.build();
return httpClient;
}
protected HttpClient getHttpClient(Map<String, Object> options) throws GeneralSecurityException {
SocketConfig socketConfig = SocketConfig.custom()
.setSoKeepAlive(true).build();
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
httpClientBuilder.setDefaultSocketConfig(socketConfig);
httpClientBuilder.disableAuthCaching();
httpClientBuilder.disableAutomaticRetries();
if(options.containsKey("sslVerify") && !Boolean.parseBoolean(options.get("sslVerify").toString())) {
log.debug("Disabling all SSL certificate verification.");
SSLContextBuilder sslContextBuilder = new SSLContextBuilder();
sslContextBuilder.loadTrustMaterial(null, new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return true;
}
});
httpClientBuilder.setSSLHostnameVerifier(new NoopHostnameVerifier());
httpClientBuilder.setSSLContext(sslContextBuilder.build());
}
return httpClientBuilder.build();
}
private HttpClientBuilder prepareHttpClientBuilderWithTimeoutsAndProxySettings(Proxy proxySettings) throws MojoExecutionException {
SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(FILE_DOWNLOAD_READ_TIMEOUT).build();
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(FILE_DOWNLOAD_CONNECT_TIMEOUT)
.setCookieSpec(CookieSpecs.IGNORE_COOKIES).build();
HttpClientBuilder httpClientBuilder = HttpClients.custom();
httpClientBuilder
.setDefaultSocketConfig(socketConfig)
.setDefaultRequestConfig(requestConfig)
.disableContentCompression();
HttpHost proxy = ProxyUtils.createProxyFromSettings(proxySettings);
if (proxy != null) {
httpClientBuilder.setProxy(proxy);
CredentialsProvider proxyCredentials = ProxyUtils.createProxyCredentialsFromSettings(proxySettings);
if (proxyCredentials != null) {
httpClientBuilder.setDefaultCredentialsProvider(proxyCredentials);
}
}
return httpClientBuilder;
}
private static PoolingHttpClientConnectionManager buildHttpClientManager(int socketTimeout)
{
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
cm.setMaxTotal(3);
cm.setDefaultMaxPerRoute(2);
cm.setValidateAfterInactivity(1);
cm.setDefaultSocketConfig(SocketConfig.custom()
.setSoTimeout(socketTimeout)
.setSoLinger(0)
.setTcpNoDelay(true)
.build());
Runtime.getRuntime().addShutdownHook(new Thread()
{
@Override
public void run()
{
cm.close();
}
});
return cm;
}
private CloseableHttpResponse execute(HttpPost post, CookieStore cookieStore) throws IOException {
try (CloseableHttpClient client = HttpClientBuilder
.create()
.useSystemProperties()
.setConnectionManager(connectionManager)
.setDefaultCookieStore(cookieStore)
.setConnectionManagerShared(true)
.setDefaultSocketConfig(
SocketConfig.custom()
.setSoTimeout(SO_TIMEOUT)
.setTcpNoDelay(true)
.build())
.build()) {
CloseableHttpResponse response = client.execute(post);
LOGGER.debug("Post response:\n"
+ "code: {}\n"
+ "headers: {}", response.getStatusLine().getStatusCode(), response.getAllHeaders());
return response;
}
}
private HttpClientConnectionManager createConnectionManager(final Http4FileSystemConfigBuilder builder,
final FileSystemOptions fileSystemOptions) throws FileSystemException {
final PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager();
connManager.setMaxTotal(builder.getMaxTotalConnections(fileSystemOptions));
connManager.setDefaultMaxPerRoute(builder.getMaxConnectionsPerHost(fileSystemOptions));
final SocketConfig socketConfig =
SocketConfig
.custom()
.setSoTimeout(builder.getSoTimeout(fileSystemOptions))
.build();
connManager.setDefaultSocketConfig(socketConfig);
return connManager;
}
private void runTest(int concurrency, HttpHandler handler) throws IOException, InterruptedException {
DefaultServer.setRootHandler(handler);
DefaultServer.startSSLServer();
try (CloseableHttpClient client = HttpClients.custom().disableConnectionState()
.setSSLContext(DefaultServer.getClientSSLContext())
.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(5000).build())
.setMaxConnPerRoute(1000)
.build()) {
ExecutorService executorService = Executors.newFixedThreadPool(concurrency);
AtomicBoolean failed = new AtomicBoolean();
Runnable task = new Runnable() {
@Override
public void run() {
if (failed.get()) {
return;
}
try (CloseableHttpResponse result = client.execute(new HttpGet(DefaultServer.getDefaultServerSSLAddress()))) {
Assert.assertEquals(StatusCodes.OK, result.getStatusLine().getStatusCode());
Header[] header = result.getHeaders("scheme");
Assert.assertEquals("https", header[0].getValue());
EntityUtils.consumeQuietly(result.getEntity());
} catch (Throwable t) {
if (failed.compareAndSet(false, true)) {
t.printStackTrace();
executorService.shutdownNow();
}
}
}
};
for (int i = 0; i < concurrency * 300; i++) {
executorService.submit(task);
}
executorService.shutdown();
Assert.assertTrue(executorService.awaitTermination(70, TimeUnit.SECONDS));
Assert.assertFalse(failed.get());
} finally {
DefaultServer.stopSSLServer();
}
}
private CloseableHttpClient generateClient(Site site) {
HttpClientBuilder httpClientBuilder = HttpClients.custom();
httpClientBuilder.setConnectionManager(connectionManager);
if (site.getUserAgent() != null) {
httpClientBuilder.setUserAgent(site.getUserAgent());
} else {
httpClientBuilder.setUserAgent("");
}
if (site.isUseGzip()) {
httpClientBuilder.addInterceptorFirst(new HttpRequestInterceptor() {
public void process(
final HttpRequest request,
final HttpContext context) throws HttpException, IOException {
if (!request.containsHeader("Accept-Encoding")) {
request.addHeader("Accept-Encoding", "gzip");
}
}
});
}
//解决post/redirect/post 302跳转问题
httpClientBuilder.setRedirectStrategy(new CustomRedirectStrategy());
SocketConfig.Builder socketConfigBuilder = SocketConfig.custom();
socketConfigBuilder.setSoKeepAlive(true).setTcpNoDelay(true);
socketConfigBuilder.setSoTimeout(site.getTimeOut());
SocketConfig socketConfig = socketConfigBuilder.build();
httpClientBuilder.setDefaultSocketConfig(socketConfig);
connectionManager.setDefaultSocketConfig(socketConfig);
httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));
generateCookie(httpClientBuilder, site);
return httpClientBuilder.build();
}
private CloseableHttpClient generateClient(Site site) {
HttpClientBuilder httpClientBuilder = HttpClients.custom();
httpClientBuilder.setConnectionManager(connectionManager);
if (site.getUserAgent() != null) {
httpClientBuilder.setUserAgent(site.getUserAgent());
} else {
httpClientBuilder.setUserAgent("");
}
if (site.isUseGzip()) {
httpClientBuilder.addInterceptorFirst(new HttpRequestInterceptor() {
@Override
public void process(
HttpRequest request,
HttpContext context) throws HttpException, IOException {
if (!request.containsHeader("Accept-Encoding")) {
request.addHeader("Accept-Encoding", "gzip");
}
}
});
}
//解决post/redirect/post 302跳转问题
httpClientBuilder.setRedirectStrategy(new CustomRedirectStrategy());
SocketConfig.Builder socketConfigBuilder = SocketConfig.custom();
socketConfigBuilder.setSoKeepAlive(true).setTcpNoDelay(true);
socketConfigBuilder.setSoTimeout(site.getTimeOut());
SocketConfig socketConfig = socketConfigBuilder.build();
httpClientBuilder.setDefaultSocketConfig(socketConfig);
connectionManager.setDefaultSocketConfig(socketConfig);
httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));
generateCookie(httpClientBuilder, site);
return httpClientBuilder.build();
}
public void testConnectionPooling() throws IOException {
PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = new PoolingHttpClientConnectionManager();
poolingHttpClientConnectionManager.setDefaultMaxPerRoute(5);
poolingHttpClientConnectionManager.setMaxTotal(5);
poolingHttpClientConnectionManager.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(1000 * 60).build());
HttpClientBuilder clientBuilder = HttpClientBuilder.create()
.setConnectionManager(poolingHttpClientConnectionManager);
try (CloseableHttpClient httpClient = clientBuilder.build()) {
for (int i = 0; i < 10; i++) {
HttpGet httpget = new HttpGet("http://davmail.sourceforge.net/version.txt");
try (CloseableHttpResponse response = httpClient.execute(httpget)) {
System.out.println("Pool stats after execute: " + poolingHttpClientConnectionManager.getTotalStats());
assertEquals(1, poolingHttpClientConnectionManager.getTotalStats().getLeased());
assertEquals(0, poolingHttpClientConnectionManager.getTotalStats().getAvailable());
String responseString = new BasicResponseHandler().handleResponse(response);
System.out.println(responseString);
System.out.println("Pool stats after response: " + poolingHttpClientConnectionManager.getTotalStats());
}
System.out.println("Pool stats after close response: " + poolingHttpClientConnectionManager.getTotalStats());
assertEquals(0, poolingHttpClientConnectionManager.getTotalStats().getLeased());
assertEquals(1, poolingHttpClientConnectionManager.getTotalStats().getAvailable());
}
}
System.out.println("Pool stats after close httpClient: " + poolingHttpClientConnectionManager.getTotalStats());
assertEquals(0, poolingHttpClientConnectionManager.getTotalStats().getLeased());
assertEquals(0, poolingHttpClientConnectionManager.getTotalStats().getAvailable());
}
public HttpClientConnectionManager createHttpClientConnectionManager() {
SSLContext sslContext = null;
try {
sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
return false;
}
}).build();
} catch (Exception e) {
throw new RuntimeException(e);
}
SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext,
NoopHostnameVerifier.INSTANCE);
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.getSocketFactory()).register("https", sslSocketFactory)
.build();
PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = new PoolingHttpClientConnectionManager(
socketFactoryRegistry);
// 最大连接数
poolingHttpClientConnectionManager.setMaxTotal(httpClientConfig.getMaxTotal());
// 单个站点最大连接数
poolingHttpClientConnectionManager.setDefaultMaxPerRoute(httpClientConfig.getMaxPerRoute());
// 长连接
poolingHttpClientConnectionManager.setDefaultSocketConfig(
SocketConfig.custom().setSoTimeout(httpClientConfig.getSocketTimeout()).setSoKeepAlive(true).build());
// 连接不活跃多久检查毫秒 并不是100 % 可信
poolingHttpClientConnectionManager.setValidateAfterInactivity(httpClientConfig.getValidateAfterInactivity());
// 空闲扫描线程
HttpClientIdleConnectionMonitor.registerConnectionManager(poolingHttpClientConnectionManager, httpClientConfig);
return poolingHttpClientConnectionManager;
}
private SocketConfig buildSocketConfig(AttributeMap standardOptions) {
return SocketConfig.custom()
// TODO do we want to keep SO keep alive
.setSoKeepAlive(false)
.setSoTimeout(
saturatedCast(standardOptions.get(SdkHttpConfigurationOption.READ_TIMEOUT)
.toMillis()))
.setTcpNoDelay(true)
.build();
}
private SocketConfig buildSocketConfig(HttpClientSettings settings) {
return SocketConfig.custom()
.setSoKeepAlive(settings.useTcpKeepAlive())
.setSoTimeout(settings.getSocketTimeout())
.setTcpNoDelay(true)
.build();
}
public static CloseableHttpClient createHttpClient(int concurrency) {
HttpClientBuilder builder = HttpClientBuilder.create();
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager();
connManager.setDefaultMaxPerRoute(concurrency);
connManager.setMaxTotal(concurrency);
RequestConfig requestConfig = RequestConfig.custom()//
.setAuthenticationEnabled(true)//
.setSocketTimeout(SOCKET_TIMEOUT)//
.setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT)//
.setConnectTimeout(CONNECT_TIMEOUT)//
.setRedirectsEnabled(true)//
.setRelativeRedirectsAllowed(true)//
.setMaxRedirects(15)//
.build();
SocketConfig socketConfig = SocketConfig.custom()//
.setSoKeepAlive(true)//
.setSoReuseAddress(true)//
.build();
builder.setConnectionManager(connManager);
builder.setDefaultSocketConfig(socketConfig);
builder.setDefaultRequestConfig(requestConfig);
return builder.build();
}
private CloseableHttpClient generateClient(Site site) {
HttpClientBuilder httpClientBuilder = HttpClients.custom();
httpClientBuilder.setConnectionManager(connectionManager);
if (site.getUserAgent() != null) {
httpClientBuilder.setUserAgent(site.getUserAgent());
} else {
httpClientBuilder.setUserAgent("");
}
if (site.isUseGzip()) {
httpClientBuilder.addInterceptorFirst(new HttpRequestInterceptor() {
public void process(
final HttpRequest request,
final HttpContext context) throws HttpException, IOException {
if (!request.containsHeader("Accept-Encoding")) {
request.addHeader("Accept-Encoding", "gzip");
}
}
});
}
//解决post/redirect/post 302跳转问题
httpClientBuilder.setRedirectStrategy(new CustomRedirectStrategy());
SocketConfig.Builder socketConfigBuilder = SocketConfig.custom();
socketConfigBuilder.setSoKeepAlive(true).setTcpNoDelay(true);
socketConfigBuilder.setSoTimeout(site.getTimeOut());
SocketConfig socketConfig = socketConfigBuilder.build();
httpClientBuilder.setDefaultSocketConfig(socketConfig);
connectionManager.setDefaultSocketConfig(socketConfig);
httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));
generateCookie(httpClientBuilder, site);
return httpClientBuilder.build();
}
public static CloseableHttpClient getHttpClient(HttpClientConnectionManager connectionManager, int maxConnections) {
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(30000).setSocketTimeout(30000).build();
ConnectionConfig connectionConfig = ConnectionConfig.custom().setCharset(StandardCharsets.UTF_8).build();
SocketConfig socketConfig = SocketConfig.custom().setTcpNoDelay(true).setSoKeepAlive(true)
.setSoReuseAddress(true).build();
//setup http client
return HttpClients.custom().setConnectionManager(connectionManager)
.setUserAgent(getHttpUserAgent()).setRetryHandler(new MCRRetryHandler(maxConnections))
.setDefaultRequestConfig(requestConfig).setDefaultConnectionConfig(connectionConfig)
.setDefaultSocketConfig(socketConfig).build();
}
@Inject
public SharedHttpClientConnectionManager(
final List<SSLContextSelector> sslContextSelectors,
@Named("${nexus.httpclient.connectionpool.size:-20}") final int connectionPoolSize,
@Named("${nexus.httpclient.connectionpool.maxSize:-200}") final int connectionPoolMaxSize,
@Named("${nexus.httpclient.connectionpool.idleTime:-30s}") final Time connectionPoolIdleTime,
@Named("${nexus.httpclient.connectionpool.evictingDelayTime:-5s}") final Time connectionPoolEvictingDelayTime,
@Named("${nexus.httpclient.connectionpool.validateAfterInactivityTime:-2s}") final Time connectionPoolValidateAfterInactivityTime,
@Named("${nexus.httpclient.connectionpool.default.requestTimeout:-20s}") final Time defaultSocketTimeout)
{
super(
new DefaultHttpClientConnectionOperator(createRegistry(sslContextSelectors), null, null),
null,
connectionPoolIdleTime.toMillis(),
TimeUnit.MILLISECONDS
);
setMaxTotal(connectionPoolMaxSize);
log.debug("Connection pool max-size: {}", connectionPoolMaxSize);
setDefaultMaxPerRoute(Math.min(connectionPoolSize, connectionPoolMaxSize));
log.debug("Connection pool size: {}", connectionPoolSize);
this.connectionPoolIdleTime = checkNotNull(connectionPoolIdleTime);
this.connectionPoolEvictingDelayTime = checkNotNull(connectionPoolEvictingDelayTime);
setValidateAfterInactivity(connectionPoolValidateAfterInactivityTime.toMillisI());
log.debug("Connection pool idle-time: {}, evicting delay: {}, validate after inactivity: {}",
connectionPoolIdleTime, connectionPoolEvictingDelayTime, connectionPoolValidateAfterInactivityTime);
setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(defaultSocketTimeout.toMillisI()).build());
log.debug("Default socket timeout {}", defaultSocketTimeout);
}
@Test
public void testPrepareUserAgentHeaderSetOnBuilder() {
// Setup
String expectedUserAgentHeader = "Nexus/Agent my user agent";
HttpClientPlan plan = mock(HttpClientPlan.class);
doReturn(expectedUserAgentHeader).when(plan).getUserAgent();
HttpClientBuilder builder = mock(HttpClientBuilder.class);
doReturn(builder).when(plan).getClient();
ConnectionConfig.Builder conn = mock(ConnectionConfig.Builder.class);
SocketConfig.Builder sock = mock(SocketConfig.Builder.class);
RequestConfig.Builder req = mock(RequestConfig.Builder.class);
doReturn(null).when(conn).build();
doReturn(null).when(sock).build();
doReturn(null).when(req).build();
doReturn(conn).when(plan).getConnection();
doReturn(sock).when(plan).getSocket();
doReturn(req).when(plan).getRequest();
HttpClientManagerImpl spy = spy(underTest);
doReturn(plan).when(spy).httpClientPlan();
// Execute
HttpClientBuilder returned = spy.prepare(null);
// Verify
assertNotNull("Returned builder must not be null.", returned);
assertEquals("Returned builder must be expected builder.", builder, returned);
verify(spy).setUserAgent(builder, expectedUserAgentHeader);
}
public HttpClientPlan() {
this.client = HttpClientBuilder.create();
this.connection = ConnectionConfig.copy(ConnectionConfig.DEFAULT);
this.socket = SocketConfig.copy(SocketConfig.DEFAULT);
this.request = RequestConfig.copy(RequestConfig.DEFAULT);
this.headers = new HashMap<>();
this.attributes = new HashMap<>();
}
private CloseableHttpClient generateClient(Site site) {
HttpClientBuilder httpClientBuilder = HttpClients.custom().setConnectionManager(connectionManager);
if (site != null && site.getUserAgent() != null) {
httpClientBuilder.setUserAgent(site.getUserAgent());
} else {
httpClientBuilder.setUserAgent("");
}
if (site == null || site.isUseGzip()) {
httpClientBuilder.addInterceptorFirst(new HttpRequestInterceptor() {
public void process(
final HttpRequest request,
final HttpContext context) throws HttpException, IOException {
if (!request.containsHeader("Accept-Encoding")) {
request.addHeader("Accept-Encoding", "gzip");
}
}
});
}
SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build();
httpClientBuilder.setDefaultSocketConfig(socketConfig);
if (site != null) {
httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));
}
generateCookie(httpClientBuilder, site);
return httpClientBuilder.build();
}
private TestWebServer(int port) throws IOException, InterruptedException {
this.port=port;
ServerBootstrap bootstrap = ServerBootstrap.bootstrap();
bootstrap.setListenerPort(port);
bootstrap.setServerInfo("Test/1.1");
bootstrap.setSocketConfig(SocketConfig.DEFAULT);
bootstrap.registerHandler("*", this::handleHttpRequest);
server = bootstrap.create();
server.start();
}
@BeforeAll
static void setupServer() throws MessagingException, IOException {
mapper = new UriHttpRequestHandlerMapper();
SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(50000).build();
server = ServerBootstrap.bootstrap().setListenerPort(0).setSocketConfig(socketConfig)
.setExceptionLogger(ExceptionLogger.NO_OP).setHandlerMapper(mapper).create();
server.start();
}