下面列出了org.apache.http.impl.auth.BasicSchemeFactory#org.apache.http.impl.NoConnectionReuseStrategy 实例代码，或者点击链接到github查看源代码，也可以在右侧发表评论。
public CloseableHttpClient createHttpClient(HttpClientConnectionManager httpClientConnectionManager,
RaptorHttpClientProperties httpClientProperties) {
RequestConfig defaultRequestConfig = RequestConfig.custom()
.setConnectTimeout(httpClientProperties.getConnectionTimeout())
.setSocketTimeout(httpClientProperties.getReadTimeout())
.setRedirectsEnabled(httpClientProperties.isFollowRedirects())
.build();
HttpRequestRetryHandler retryHandler = new DefaultHttpRequestRetryHandler(httpClientProperties.getRetryCount(),
httpClientProperties.isRequestSentRetryEnabled());
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create().disableContentCompression()
.disableCookieManagement()
.useSystemProperties()
.setRetryHandler(retryHandler)
.setConnectionManager(httpClientConnectionManager)
.setDefaultRequestConfig(defaultRequestConfig);
if(!keepAlive){
httpClientBuilder.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE);
}
return httpClientBuilder.build();
}
public static CrawlerHttpClientBuilder setDefault(CrawlerHttpClientBuilder proxyFeedBackDecorateHttpClientBuilder) {
SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).setSoLinger(-1).setSoReuseAddress(false)
.setSoTimeout(ProxyConstant.SOCKETSO_TIMEOUT).setTcpNoDelay(true).build();
proxyFeedBackDecorateHttpClientBuilder.setDefaultSocketConfig(socketConfig)
// .setSSLSocketFactory(sslConnectionSocketFactory)
// dungproxy0.0.6之后的版本,默认忽略https证书检查
.setRedirectStrategy(new LaxRedirectStrategy())
// 注意,这里使用ua生产算法自动产生ua,如果是mobile,可以使用
// com.virjar.vscrawler.core.net.useragent.UserAgentBuilder.randomAppUserAgent()
.setUserAgent(UserAgentBuilder.randomUserAgent())
// 对于爬虫来说,连接池没啥卵用,直接禁止掉(因为我们可能创建大量HttpClient,每个HttpClient一个连接池,会把系统socket资源撑爆)
// 测试开80个httpClient抓数据大概一个小时系统就会宕机
.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE)
// 现在有些网站的网络响应头部数据有非ASCII数据,httpclient默认只使用ANSI标准解析header,这可能导致带有中文的header数据无法解析
.setDefaultConnectionConfig(ConnectionConfig.custom().setCharset(Charsets.UTF_8).build());
return proxyFeedBackDecorateHttpClientBuilder;
}
@VisibleForTesting
HTTPConnectionFactory(final Configurator config) {
Registry<ConnectionSocketFactory> registry = null;
try {
final Configurator cfg = config == null ? ConfigUtil.getConfigInfo(HTTPConnectionFactory.class) : config;
// if someone doesn't want keep alives...
if (!cfg.findBooleanEntry(CFG_HTTP_KEEPALIVE, DFLT_KEEPALIVE)) {
this.connReuseStrategy = NoConnectionReuseStrategy.INSTANCE;
}
this.maxConns = cfg.findIntEntry(CFG_HTTP_MAXCONNS, DFLT_MAXCONNS);
this.userAgent = cfg.findStringEntry(CFG_HTTP_AGENT, DEFAULT_HTTP_AGENT);
final SSLContext sslContext = build(cfg);
// mainly for using in test environments where cert name may not match host name
final HostnameVerifier v = cfg.findBooleanEntry(CFG_NOOP_VERIFIER, false) ? new NoopHostnameVerifier() : new DefaultHostnameVerifier();
registry =
RegistryBuilder.<ConnectionSocketFactory>create().register(HTTP, PlainConnectionSocketFactory.getSocketFactory())
.register(HTTPS, new SSLConnectionSocketFactory(sslContext, v)).build();
} catch (IOException | GeneralSecurityException ex) {
log.error("Error configuring HTTPConnectionFactory. The connection factory will use HTTP Client default settings", ex);
}
if (registry == null) {
this.connMan = new PoolingHttpClientConnectionManager();
} else {
this.connMan = new PoolingHttpClientConnectionManager(registry);
}
this.connMan.setMaxTotal(this.maxConns);
}
@Bean
@ConditionalOnMissingBean(HttpClient.class)
public HttpClient httpClientOut() {
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(httpClientPropertiesOut.getConnectTimeout())
.setSocketTimeout(httpClientPropertiesOut.getSocketTimeout()).build();
HttpClient httpClient = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig)
.setUserAgent(httpClientPropertiesOut.getAgent()).setMaxConnPerRoute(httpClientPropertiesOut.getMaxPerRoute())
.setConnectionReuseStrategy(new NoConnectionReuseStrategy()).build();
return httpClient;
}
static CloseableHttpClient getHttpClient() {
if (httpclient == null) {
int timeout = 5 * 1000;
httpclient = HttpClientBuilder.create().
setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
setDefaultRequestConfig(RequestConfig.custom().
setConnectTimeout(timeout).
setConnectionRequestTimeout(timeout).
setCookieSpec(CookieSpecs.STANDARD).
setSocketTimeout(timeout).
build()).
build();
}
return httpclient;
}
/** Creates a new fetching thread.
*
* @param frontier a reference to the {@link Frontier}.
* @param index the index of this thread (only for logging purposes).
*/
public FetchingThread(final Frontier frontier, final int index) throws NoSuchAlgorithmException, IllegalArgumentException, IOException {
setName(this.getClass().getSimpleName() + '-' + index);
setPriority(Thread.MIN_PRIORITY); // Low priority; there will be thousands of this guys around.
this.frontier = frontier;
final BasicHttpClientConnectionManager connManager = new BasicHttpClientConnectionManagerWithAlternateDNS(frontier.rc.dnsResolver);
connManager.closeIdleConnections(0, TimeUnit.MILLISECONDS);
connManager.setConnectionConfig(ConnectionConfig.custom().setBufferSize(8 * 1024).build()); // TODO: make this configurable
cookieStore = new BasicCookieStore();
final BasicHeader[] headers = {
new BasicHeader("From", frontier.rc.userAgentFrom),
new BasicHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.95,text/*;q=0.9,*/*;q=0.8")
};
httpClient = HttpClients.custom()
.setSSLContext(frontier.rc.acceptAllCertificates ? TRUST_ALL_CERTIFICATES_SSL_CONTEXT : TRUST_SELF_SIGNED_SSL_CONTEXT)
.setConnectionManager(connManager)
.setConnectionReuseStrategy(frontier.rc.keepAliveTime == 0 ? NoConnectionReuseStrategy.INSTANCE : DefaultConnectionReuseStrategy.INSTANCE)
.setUserAgent(frontier.rc.userAgent)
.setDefaultCookieStore(cookieStore)
.setDefaultHeaders(ObjectArrayList.wrap(headers))
.build();
fetchData = new FetchData(frontier.rc);
}
public static CloseableHttpClient getHttpClient(final HttpHost proxy, final boolean redirects, final CookieStore cookieStore) {
final Builder builder = RequestConfig.custom()
.setRedirectsEnabled(redirects)
.setMaxRedirects(5);
if (proxy != null) builder.setProxy(proxy);
final RequestConfig requestConfig = builder.build();
return HttpClients.custom()
.setDefaultRequestConfig(requestConfig)
.setDefaultCookieStore(cookieStore)
.setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE)
.build();
}
public ListenerThread(final ApiServer requestHandler, final int port) {
try {
_serverSocket = new ServerSocket(port);
} catch (final IOException ioex) {
s_logger.error("error initializing api server", ioex);
return;
}
_params = new BasicHttpParams();
_params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 30000)
.setIntParameter(CoreConnectionPNames.SOCKET_BUFFER_SIZE, 8 * 1024)
.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK, false)
.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY, true)
.setParameter(CoreProtocolPNames.ORIGIN_SERVER, "HttpComponents/1.1");
// Set up the HTTP protocol processor
final BasicHttpProcessor httpproc = new BasicHttpProcessor();
httpproc.addInterceptor(new ResponseDate());
httpproc.addInterceptor(new ResponseServer());
httpproc.addInterceptor(new ResponseContent());
httpproc.addInterceptor(new ResponseConnControl());
// Set up request handlers
final HttpRequestHandlerRegistry reqistry = new HttpRequestHandlerRegistry();
reqistry.register("*", requestHandler);
// Set up the HTTP service
_httpService = new HttpService(httpproc, new NoConnectionReuseStrategy(), new DefaultHttpResponseFactory());
_httpService.setParams(_params);
_httpService.setHandlerResolver(reqistry);
}
/**
* Get a default HttpClient based on the HttpConfiguration object. If required the defaults can
* be altered to meet the requirements of the SDK user. The default client does not use connection
* pooling and does not reuse connections. Timeouts for connection and socket are taken from the
* {@link HttpConfiguration} object.
*
* @param httpConfiguration
* @return CloseableHttpClient
*/
public static CloseableHttpClient getDefaultClient(HttpConfiguration httpConfiguration) {
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(httpConfiguration.getTimeout())
.setSocketTimeout(httpConfiguration.getTimeout()).build();
HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager();
ConnectionReuseStrategy connectionResuseStrategy = new NoConnectionReuseStrategy();
logger.debug("Creating HttpClient with simple no pooling/no connection reuse default settings.");
CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(requestConfig).setConnectionManager(connectionManager)
.setConnectionReuseStrategy(connectionResuseStrategy).build();
return httpClient;
}
/**
* Default constructor.
* @param defaultFilterProcessesUrl the url of the filter
*/
public GitHubAuthFilter(final String defaultFilterProcessesUrl) {
super(defaultFilterProcessesUrl);
this.jreader = ParaObjectUtils.getJsonReader(Map.class);
int timeout = 30 * 1000;
this.httpclient = HttpClientBuilder.create().
setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
setDefaultRequestConfig(RequestConfig.custom().
setConnectTimeout(timeout).
setConnectionRequestTimeout(timeout).
setCookieSpec(CookieSpecs.STANDARD).
setSocketTimeout(timeout).
build()).
build();
}
/**
* Default constructor.
* @param defaultFilterProcessesUrl the url of the filter
*/
public LinkedInAuthFilter(final String defaultFilterProcessesUrl) {
super(defaultFilterProcessesUrl);
this.jreader = ParaObjectUtils.getJsonReader(Map.class);
int timeout = 30 * 1000;
this.httpclient = HttpClientBuilder.create().
setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
setDefaultRequestConfig(RequestConfig.custom().
setConnectTimeout(timeout).
setConnectionRequestTimeout(timeout).
setCookieSpec(CookieSpecs.STANDARD).
setSocketTimeout(timeout).
build()).
build();
}
/**
* Default constructor.
* @param defaultFilterProcessesUrl the url of the filter
*/
public AmazonAuthFilter(final String defaultFilterProcessesUrl) {
super(defaultFilterProcessesUrl);
this.jreader = ParaObjectUtils.getJsonReader(Map.class);
int timeout = 30 * 1000;
this.httpclient = HttpClientBuilder.create().
setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
setDefaultRequestConfig(RequestConfig.custom().
setConnectTimeout(timeout).
setConnectionRequestTimeout(timeout).
setCookieSpec(CookieSpecs.STANDARD).
setSocketTimeout(timeout).
build()).
build();
}
/**
* Default constructor.
* @param defaultFilterProcessesUrl the url of the filter
*/
public GoogleAuthFilter(final String defaultFilterProcessesUrl) {
super(defaultFilterProcessesUrl);
this.jreader = ParaObjectUtils.getJsonReader(Map.class);
int timeout = 30 * 1000;
this.httpclient = HttpClientBuilder.create().
setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
setDefaultRequestConfig(RequestConfig.custom().
setConnectTimeout(timeout).
setConnectionRequestTimeout(timeout).
setCookieSpec(CookieSpecs.STANDARD).
setSocketTimeout(timeout).
build()).
build();
}
/**
* Default constructor.
* @param defaultFilterProcessesUrl the url of the filter
*/
public TwitterAuthFilter(final String defaultFilterProcessesUrl) {
super(defaultFilterProcessesUrl);
this.jreader = ParaObjectUtils.getJsonReader(Map.class);
int timeout = 30 * 1000;
this.httpclient = HttpClientBuilder.create().
setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
setDefaultRequestConfig(RequestConfig.custom().
setConnectTimeout(timeout).
setConnectionRequestTimeout(timeout).
setCookieSpec(CookieSpecs.STANDARD).
setSocketTimeout(timeout).
build()).
build();
}
/**
* Default constructor.
* @param defaultFilterProcessesUrl the url of the filter
*/
public SlackAuthFilter(final String defaultFilterProcessesUrl) {
super(defaultFilterProcessesUrl);
this.jreader = ParaObjectUtils.getJsonReader(Map.class);
int timeout = 30 * 1000;
this.httpclient = HttpClientBuilder.create().
setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
setDefaultRequestConfig(RequestConfig.custom().
setConnectTimeout(timeout).
setConnectionRequestTimeout(timeout).
setCookieSpec(CookieSpecs.STANDARD).
setSocketTimeout(timeout).
build()).
build();
}
/**
* Default constructor.
* @param defaultFilterProcessesUrl the url of the filter
*/
public GenericOAuth2Filter(final String defaultFilterProcessesUrl) {
super(defaultFilterProcessesUrl);
this.jreader = ParaObjectUtils.getJsonReader(Map.class);
int timeout = 30 * 1000;
this.httpclient = HttpClientBuilder.create().
setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
setDefaultRequestConfig(RequestConfig.custom().
setConnectTimeout(timeout).
setConnectionRequestTimeout(timeout).
setCookieSpec(CookieSpecs.STANDARD).
setSocketTimeout(timeout).
build()).
build();
}
/**
* Default constructor.
* @param defaultFilterProcessesUrl the url of the filter
*/
public FacebookAuthFilter(String defaultFilterProcessesUrl) {
super(defaultFilterProcessesUrl);
this.jreader = ParaObjectUtils.getJsonReader(Map.class);
int timeout = 30 * 1000;
this.httpclient = HttpClientBuilder.create().
setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
setDefaultRequestConfig(RequestConfig.custom().
setConnectTimeout(timeout).
setConnectionRequestTimeout(timeout).
setCookieSpec(CookieSpecs.STANDARD).
setSocketTimeout(timeout).
build()).
build();
}
/**
* Default constructor.
* @param defaultFilterProcessesUrl the url of the filter
*/
public MicrosoftAuthFilter(final String defaultFilterProcessesUrl) {
super(defaultFilterProcessesUrl);
this.jreader = ParaObjectUtils.getJsonReader(Map.class);
int timeout = 30 * 1000;
this.httpclient = HttpClientBuilder.create().
setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
setDefaultRequestConfig(RequestConfig.custom().
setConnectTimeout(timeout).
setConnectionRequestTimeout(timeout).
setCookieSpec(CookieSpecs.STANDARD).
setSocketTimeout(timeout).
build()).
build();
}
/**
* Default constructor.
* @param dao dao
*/
@Inject
public OXRCurrencyConverter(DAO dao) {
this.dao = dao;
int timeout = 30 * 1000;
this.httpClient = HttpClientBuilder.create().
setConnectionReuseStrategy(new NoConnectionReuseStrategy()).
setDefaultRequestConfig(RequestConfig.custom().
setConnectTimeout(timeout).
setConnectionRequestTimeout(timeout).
setSocketTimeout(timeout).
build()).
build();
}
public ListenerThread(final ApiServer requestHandler, final int port) {
try {
_serverSocket = new ServerSocket(port);
} catch (final IOException ioex) {
s_logger.error("error initializing api server", ioex);
return;
}
_params = new BasicHttpParams();
_params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 30000)
.setIntParameter(CoreConnectionPNames.SOCKET_BUFFER_SIZE, 8 * 1024)
.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK, false)
.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY, true)
.setParameter(CoreProtocolPNames.ORIGIN_SERVER, "HttpComponents/1.1");
// Set up the HTTP protocol processor
final BasicHttpProcessor httpproc = new BasicHttpProcessor();
httpproc.addInterceptor(new ResponseDate());
httpproc.addInterceptor(new ResponseServer());
httpproc.addInterceptor(new ResponseContent());
httpproc.addInterceptor(new ResponseConnControl());
// Set up request handlers
final HttpRequestHandlerRegistry reqistry = new HttpRequestHandlerRegistry();
reqistry.register("*", requestHandler);
// Set up the HTTP service
_httpService = new HttpService(httpproc, new NoConnectionReuseStrategy(), new DefaultHttpResponseFactory());
_httpService.setParams(_params);
_httpService.setHandlerResolver(reqistry);
}
/**
* Create an {@link HttpClientBuilder} object. Invoked by {@link #createHttpClient(Http4FileSystemConfigBuilder, GenericFileName, FileSystemOptions)}.
*
* @param builder Configuration options builder for HTTP4 provider
* @param rootName The root path
* @param fileSystemOptions The FileSystem options
* @return an {@link HttpClientBuilder} object
* @throws FileSystemException if an error occurs
*/
protected HttpClientBuilder createHttpClientBuilder(final Http4FileSystemConfigBuilder builder, final GenericFileName rootName,
final FileSystemOptions fileSystemOptions) throws FileSystemException {
final List<Header> defaultHeaders = new ArrayList<>();
defaultHeaders.add(new BasicHeader(HTTP.USER_AGENT, builder.getUserAgent(fileSystemOptions)));
final ConnectionReuseStrategy connectionReuseStrategy = builder.isKeepAlive(fileSystemOptions)
? DefaultConnectionReuseStrategy.INSTANCE
: NoConnectionReuseStrategy.INSTANCE;
final HttpClientBuilder httpClientBuilder =
HttpClients.custom()
.setRoutePlanner(createHttpRoutePlanner(builder, fileSystemOptions))
.setConnectionManager(createConnectionManager(builder, fileSystemOptions))
.setSSLContext(createSSLContext(builder, fileSystemOptions))
.setSSLHostnameVerifier(createHostnameVerifier(builder, fileSystemOptions))
.setConnectionReuseStrategy(connectionReuseStrategy)
.setDefaultRequestConfig(createDefaultRequestConfig(builder, fileSystemOptions))
.setDefaultHeaders(defaultHeaders)
.setDefaultCookieStore(createDefaultCookieStore(builder, fileSystemOptions));
if (!builder.getFollowRedirect(fileSystemOptions)) {
httpClientBuilder.disableRedirectHandling();
}
return httpClientBuilder;
}
/**
* @param proxy Proxy configuration
* @param listener Log listener
* @param prompt Prompt for proxy credentials
* @return Builder for HTTP client
*/
public HttpClientBuilder build(final Proxy proxy, final TranscriptListener listener, final LoginCallback prompt) {
final HttpClientBuilder configuration = HttpClients.custom();
// Use HTTP Connect proxy implementation provided here instead of
// relying on internal proxy support in socket factory
switch(proxy.getType()) {
case HTTP:
case HTTPS:
final HttpHost h = new HttpHost(proxy.getHostname(), proxy.getPort(), Scheme.http.name());
if(log.isInfoEnabled()) {
log.info(String.format("Setup proxy %s", h));
}
configuration.setProxy(h);
configuration.setProxyAuthenticationStrategy(new CallbackProxyAuthenticationStrategy(ProxyCredentialsStoreFactory.get(), host, prompt));
break;
}
configuration.setUserAgent(new PreferencesUseragentProvider().get());
final int timeout = preferences.getInteger("connection.timeout.seconds") * 1000;
configuration.setDefaultSocketConfig(SocketConfig.custom()
.setTcpNoDelay(true)
.setSoTimeout(timeout)
.build());
configuration.setDefaultRequestConfig(this.createRequestConfig(timeout));
configuration.setDefaultConnectionConfig(ConnectionConfig.custom()
.setBufferSize(preferences.getInteger("http.socket.buffer"))
.setCharset(Charset.forName(host.getEncoding()))
.build());
if(preferences.getBoolean("http.connections.reuse")) {
configuration.setConnectionReuseStrategy(new DefaultClientConnectionReuseStrategy());
}
else {
configuration.setConnectionReuseStrategy(new NoConnectionReuseStrategy());
}
configuration.setRetryHandler(new ExtendedHttpRequestRetryHandler(preferences.getInteger("http.connections.retry")));
configuration.setServiceUnavailableRetryStrategy(new DisabledServiceUnavailableRetryStrategy());
if(!preferences.getBoolean("http.compression.enable")) {
configuration.disableContentCompression();
}
configuration.setRequestExecutor(new LoggingHttpRequestExecutor(listener));
// Always register HTTP for possible use with proxy. Contains a number of protocol properties such as the
// default port and the socket factory to be used to create the java.net.Socket instances for the given protocol
configuration.setConnectionManager(this.createConnectionManager(this.createRegistry()));
configuration.setDefaultAuthSchemeRegistry(RegistryBuilder.<AuthSchemeProvider>create()
.register(AuthSchemes.BASIC, new BasicSchemeFactory(
Charset.forName(preferences.getProperty("http.credentials.charset"))))
.register(AuthSchemes.DIGEST, new DigestSchemeFactory(
Charset.forName(preferences.getProperty("http.credentials.charset"))))
.register(AuthSchemes.NTLM, preferences.getBoolean("webdav.ntlm.windows.authentication.enable") && WinHttpClients.isWinAuthAvailable() ?
new BackportWindowsNTLMSchemeFactory(null) :
new NTLMSchemeFactory())
.register(AuthSchemes.SPNEGO, preferences.getBoolean("webdav.ntlm.windows.authentication.enable") && WinHttpClients.isWinAuthAvailable() ?
new BackportWindowsNegotiateSchemeFactory(null) :
new SPNegoSchemeFactory())
.register(AuthSchemes.KERBEROS, new KerberosSchemeFactory()).build());
return configuration;
}
protected ConnectionReuseStrategy createConnectionReuseStrategy() {
return NoConnectionReuseStrategy.INSTANCE;
}