/**
 * Get a HTTP client. With proxy if a proxy is provided in the constructor.
 * @return a HTTP client
 */
protected HttpClient getHttpClient(){

    HttpClientBuilder builder = HttpClientBuilder.create();

    if(proxyHost != null && proxyPort != 0) {
        HttpHost proxy = new HttpHost(proxyHost, proxyPort);
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        builder.setRoutePlanner(routePlanner);
    }

    RequestConfig.Builder config = RequestConfig.custom()
            .setConnectTimeout(timeout)
            .setConnectionRequestTimeout(timeout)
            .setSocketTimeout(timeout);

    builder.setDefaultRequestConfig(config.build());

    return builder.build();
}
 

/**
 * Get an async HTTP client. With proxy if a proxy is provided in the constructor.
 * @return an async HTTP client
 */
protected CloseableHttpAsyncClient getHttpAsyncClient(){

    HttpAsyncClientBuilder builder = HttpAsyncClientBuilder.create();

    if(proxyHost != null && proxyPort != 0) {
        HttpHost proxy = new HttpHost(proxyHost, proxyPort);
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        builder.setRoutePlanner(routePlanner);
    }

    RequestConfig.Builder config = RequestConfig.custom()
            .setConnectTimeout(timeout)
            .setConnectionRequestTimeout(timeout)
            .setSocketTimeout(timeout);

    builder.setDefaultRequestConfig(config.build());

    return builder.build();
}
 

/**
 * Creates a HttpClientBuilder with the default settings of GERBIL.
 * 
 * @return a HttpClientBuilder with the default settings of GERBIL.
 */
public HttpClientBuilder generateHttpClientBuilder() {
    HttpClientBuilder builder = HttpClientBuilder.create();
    builder.setUserAgent(userAgent);

    String proxyHost = GerbilConfiguration.getInstance().getString(PROXY_HOST_KEY);
    int proxyPort = GerbilConfiguration.getInstance().getInt(PROXY_PORT_KEY, DEFAULT_PROXY_PORT);

    if (proxyHost != null) {
        HttpHost proxy = new HttpHost(proxyHost, proxyPort);
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        builder.setRoutePlanner(routePlanner);
    }
    // Use a redirect strategy that allows the "direct redirect" of POST requests
    // without creating a completely new request
    builder.setRedirectStrategy(new SimpleRedirectStrategy()).build();

    return builder;
}
 

/**
 * プロパティの設定が終了したあとにHttpClientのインスタンスを生成し、プロキシの設定を行います。
 * {@inheritDoc}
 */
@Override
public void afterPropertiesSet() throws Exception {

    Assert.notNull(proxyHost, "プロキシホスト(proxyHost)は必須です。");
    Assert.notNull(proxyPort, "プロキシポート番号(proxyPort)は必須です。");

    PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
    connectionManager.setMaxTotal(maxTotal);
    connectionManager.setDefaultMaxPerRoute(defaultMaxPerRoute);

    HttpClientBuilder builder = HttpClients.custom();
    builder.setConnectionManager(connectionManager);
    if (authentication) {
        Assert.notNull(username, "ユーザ認証がtrueに設定された場合、ユーザ名(username)は必須です。");
        Assert.notNull(password, "ユーザ認証がtrueに設定された場合、パスワード(password)は必須です。");
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxyHost, Integer.parseInt(proxyPort)));
        builder.setRoutePlanner(routePlanner);
        CredentialsProvider credsProvider = new BasicCredentialsProvider();
        credsProvider.setCredentials(new AuthScope(proxyHost, Integer.parseInt(proxyPort)), new UsernamePasswordCredentials(username, password));
        builder.setDefaultCredentialsProvider(credsProvider);
    }
    builder.setDefaultRequestConfig(RequestConfig.custom().setSocketTimeout(readTimeout).build());
    CloseableHttpClient client = builder.build();
    setHttpClient(client);
}
 

@Test
public void givenServerThatIsBehindProxy_whenClientIsConfiguredToSendRequestViaProxy_shouldReturn200() throws IOException {
    //given
    proxyMock.stubFor(get(urlMatching(".*"))
      .willReturn(aResponse().proxiedFrom("http://localhost:8089/")));

    serviceMock.stubFor(get(urlEqualTo("/private"))
      .willReturn(aResponse().withStatus(200)));


    HttpHost proxy = new HttpHost("localhost", 8090);
    DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
    HttpClient httpclient = HttpClients.custom()
      .setRoutePlanner(routePlanner)
      .build();

    //when
    final HttpGet httpGet = new HttpGet("http://localhost:8089/private");
    HttpResponse response = httpclient.execute(httpGet);

    //then
    assertEquals(response.getStatusLine().getStatusCode(), 200);
    proxyMock.verify(getRequestedFor(urlEqualTo("/private")));
    serviceMock.verify(getRequestedFor(urlEqualTo("/private")));
}
 

public void init(BasicCookieStore basicCookieStore) {
    this.basicCookieStore = basicCookieStore;
    this.config = ApplicationContextHelper.getBean(Config.class);
    httpClient = HttpClients.custom().setDefaultCookieStore(basicCookieStore).build();
    if (config != null && config.getEnableProxy()) {
        HttpHost proxy = new HttpHost(config.getProxyIp().getIp(), config.getProxyIp().getPort());
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        httpClient = HttpClients.custom().setRoutePlanner(routePlanner)
                .setDefaultCookieStore(basicCookieStore).build();
    }
    httpPureClient = HttpClients.createDefault();
}
 

@Override
public final HttpClientBuilder apply(final HttpClientBuilder builder) {
    return builder.setRoutePlanner(
        new DefaultProxyRoutePlanner(
            new HttpHost(this.host, this.port, this.scheme)
        )
    );
}
 

public static CloseableHttpClient getNotValidatingClient(int timeout, String proxyServer, Integer proxyPort,
		String proxyUser, String proxyPassword) {
	try {
		HttpClientBuilder clientBuilder = HttpClients.custom();

		RequestConfig.Builder requestBuilder = RequestConfig.custom().setConnectTimeout(timeout * 1000)
				.setSocketTimeout(timeout * 1000).setConnectionRequestTimeout(timeout * 1000)
				.setRedirectsEnabled(true);

		if (StringUtils.isNotEmpty(proxyServer)) {
			HttpHost proxyHost = new HttpHost(proxyServer, proxyPort);
			requestBuilder.setProxy(proxyHost);

			DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxyHost);
			clientBuilder.setRoutePlanner(routePlanner);

			if (StringUtils.isNotEmpty(proxyUser)) {
				CredentialsProvider credentialProvider = new BasicCredentialsProvider();
				credentialProvider.setCredentials(AuthScope.ANY,
						new UsernamePasswordCredentials(proxyUser, proxyPassword));
				clientBuilder.setRoutePlanner(routePlanner);
			}
		}

		RequestConfig requestConfig = requestBuilder.build();

		CloseableHttpClient httpclient = clientBuilder.setHostnameVerifier(new AllowAllHostnameVerifier())
				.setSslcontext(new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
					public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
						return true;
					}
				}).build()).setDefaultRequestConfig(requestConfig).build();
		return httpclient;
	} catch (Throwable t) {
		return null;
	}
}
 

public ClientHttpRequestFactory createRequestFactory(HttpProxyConfiguration httpProxyConfiguration, boolean trustSelfSignedCerts) {
    HttpClientBuilder httpClientBuilder = HttpClients.custom()
                                                     .useSystemProperties();

    if (trustSelfSignedCerts) {
        httpClientBuilder.setSslcontext(buildSslContext());
        httpClientBuilder.setHostnameVerifier(BROWSER_COMPATIBLE_HOSTNAME_VERIFIER);
    }

    if (httpProxyConfiguration != null) {
        HttpHost proxy = new HttpHost(httpProxyConfiguration.getProxyHost(), httpProxyConfiguration.getProxyPort());
        httpClientBuilder.setProxy(proxy);

        if (httpProxyConfiguration.isAuthRequired()) {
            BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            credentialsProvider.setCredentials(new AuthScope(httpProxyConfiguration.getProxyHost(),
                                                             httpProxyConfiguration.getProxyPort()),
                                               new UsernamePasswordCredentials(httpProxyConfiguration.getUsername(),
                                                                               httpProxyConfiguration.getPassword()));
            httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        }

        HttpRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        httpClientBuilder.setRoutePlanner(routePlanner);
    }

    HttpClient httpClient = httpClientBuilder.build();

    return new HttpComponentsClientHttpRequestFactory(httpClient);
}
 

@Override
public HttpClientBuilder build(final Proxy proxy, final TranscriptListener listener, final LoginCallback prompt) {
    final HttpClientBuilder builder = super.build(proxy, listener, prompt);
    // Add filter to inject custom headers to authenticate with proxy
    builder.setRequestExecutor(
            new CustomHeaderHttpRequestExecutor(headers)
    );
    // Set proxy router planer
    builder.setRoutePlanner(new DefaultProxyRoutePlanner(
        new HttpHost(this.proxy.getHostname(), this.proxy.getPort(), this.proxy.getProtocol().getScheme().name()),
            new DefaultSchemePortResolver()));
    return builder;
}
 

private CloseableHttpClient getHttpClient(HttpHost proxy) {
    HttpClientBuilder builder = HttpClientBuilder.create()
            .setSSLHostnameVerifier(new NoopHostnameVerifier())
            .setConnectionTimeToLive(70, TimeUnit.SECONDS)
            .setMaxConnTotal(100);
    if (proxy != null) {
        builder.setProxy(proxy)
                .setRoutePlanner(new DefaultProxyRoutePlanner(proxy));
    }
    return builder.build();
}
 

private RequestConfig.Builder setupProxy(ProxyConfiguration proxy, HttpClientBuilder clientBuilder, RequestConfig.Builder reqconfigconbuilder) throws MalformedURLException
{
	HttpHost proxyHost = new HttpHost(proxy.name, proxy.port);
	DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxyHost);
	clientBuilder.setRoutePlanner(routePlanner);
	reqconfigconbuilder.setProxy(proxyHost);

	setupProxyAuth(proxy, clientBuilder, proxyHost);
	return reqconfigconbuilder;
}
 

private HttpRoutePlanner getRoutePlanner() {
    if (isUsingProxy()) {
        return new DefaultProxyRoutePlanner(new HttpHost(getProxyHost(), getProxyPort()));
    } else {
        return new DefaultRoutePlanner(null);
    }
}
 

private HttpClientBuilder configureProxy(HttpClientBuilder builder) {
    final String proxy = config.get("proxy");
    if (isNotBlank(proxy)) {
        final int proxyPort = Integer.parseInt(config.get("proxyPort"));
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(new HttpHost(proxy, proxyPort));
        builder.setRoutePlanner(routePlanner);
    }
    return builder;
}
 

@Override
public void customize(RestTemplate restTemplate) {
    HttpHost proxy = new HttpHost(PROXY_SERVER_HOST, PROXY_SERVER_PORT);
    HttpClient httpClient = HttpClientBuilder.create()
        .setRoutePlanner(new DefaultProxyRoutePlanner(proxy) {
            @Override
            public HttpHost determineProxy(HttpHost target, HttpRequest request, HttpContext context) throws HttpException {
                return super.determineProxy(target, request, context);
            }
        })
        .build();
    restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory(httpClient));
}
 

private HttpRoutePlanner createHttpRoutePlanner(final Http4FileSystemConfigBuilder builder,
        final FileSystemOptions fileSystemOptions) {
    final HttpHost proxyHost = getProxyHttpHost(builder, fileSystemOptions);

    if (proxyHost != null) {
        return new DefaultProxyRoutePlanner(proxyHost);
    }

    return new SystemDefaultRoutePlanner(ProxySelector.getDefault());
}
 

private static void init() {
    try {
        SSLContext sslContext =
                SSLContexts.custom()
                        .loadTrustMaterial(KeyStore.getInstance(KeyStore.getDefaultType()), new TrustStrategy() {
                            @Override
                            public boolean isTrusted(X509Certificate[] chain, String authType)
                                    throws CertificateException {
                                return true;
                            }
                        }).build();
        SSLConnectionSocketFactory sslSFactory =
                new SSLConnectionSocketFactory(sslContext);
        Registry<ConnectionSocketFactory> socketFactoryRegistry =
                RegistryBuilder.<ConnectionSocketFactory>create()
                        .register("http", PlainConnectionSocketFactory.INSTANCE).register("https", sslSFactory)
                        .build();

        PoolingHttpClientConnectionManager connManager =
                new PoolingHttpClientConnectionManager(socketFactoryRegistry);

        SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(CommonConstant.TIMEOUT).setTcpNoDelay(true).build();
        connManager.setDefaultSocketConfig(socketConfig);

        ConnectionConfig connectionConfig =
                ConnectionConfig.custom().setMalformedInputAction(CodingErrorAction.IGNORE)
                        .setUnmappableInputAction(CodingErrorAction.IGNORE).setCharset(Consts.UTF_8).build();
        connManager.setDefaultConnectionConfig(connectionConfig);
        connManager.setMaxTotal(500);
        connManager.setDefaultMaxPerRoute(300);
        HttpRequestRetryHandler retryHandler = new HttpRequestRetryHandler() {
            @Override
            public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
                if (executionCount > 2) {
                    return false;
                }
                if (exception instanceof InterruptedIOException) {
                    return true;
                }
                if (exception instanceof ConnectTimeoutException) {
                    return true;
                }
                if (exception instanceof UnknownHostException) {
                    return true;
                }
                if (exception instanceof SSLException) {
                    return true;
                }
                HttpRequest request = HttpClientContext.adapt(context).getRequest();
                if (!(request instanceof HttpEntityEnclosingRequest)) {
                    return true;
                }
                return false;
            }
        };
        HttpClientBuilder httpClientBuilder =
                HttpClients.custom().setConnectionManager(connManager)
                        .setRetryHandler(retryHandler)
                        .setDefaultCookieStore(new BasicCookieStore()).setUserAgent(userAgent);
        if (proxy != null) {
            httpClientBuilder.setRoutePlanner(new DefaultProxyRoutePlanner(proxy)).build();
        }
        httpClient = httpClientBuilder.build();

        requestConfig = RequestConfig.custom().setSocketTimeout(CommonConstant.TIMEOUT).
                setConnectTimeout(CommonConstant.TIMEOUT).
                setConnectionRequestTimeout(CommonConstant.TIMEOUT).
                setCookieSpec(CookieSpecs.STANDARD).
                build();
    } catch (Exception e) {
        logger.error("Exception:", e);
    }
}
 

protected void initialzeInternalClient() {

        if (!needsInternalClientInialization) {
            // internal client is already initialized
            return;
        }

        // release any resources if this client was already used
        close();

        // rebuild the client
        HttpClientBuilder httpClientBuilder = HttpClients.custom();

        // Add this interceptor to get the values of all HTTP headers in the request.
        // Some of them are provided by the user while others are generated by Apache HTTP Components.
        httpClientBuilder.addInterceptorLast(new HttpRequestInterceptor() {
            @Override
            public void process( HttpRequest request, HttpContext context ) throws HttpException,
                                                                            IOException {

                Header[] requestHeaders = request.getAllHeaders();
                actualRequestHeaders = new ArrayList<HttpHeader>();
                for (Header header : requestHeaders) {
                    addHeaderToList(actualRequestHeaders, header.getName(), header.getValue());
                }
                if (debugLevel != HttpDebugLevel.NONE) {
                    logHTTPRequest(requestHeaders, request);
                }
            }
        });

        // connect and read timeouts
        httpClientBuilder.setDefaultRequestConfig(RequestConfig.custom()
                                                               .setConnectTimeout(connectTimeoutSeconds
                                                                                  * 1000)
                                                               .setSocketTimeout(readTimeoutSeconds
                                                                                 * 1000)
                                                               .build());

        // socket buffer size
        if (this.socketBufferSize > 0) {
            httpClientBuilder.setDefaultSocketConfig(SocketConfig.custom()
                                                                 .setRcvBufSize(this.socketBufferSize)
                                                                 .setSndBufSize(this.socketBufferSize)
                                                                 .build());
        }

        // SSL
        if (isOverSsl) {
            setupSSL(httpClientBuilder);
        }

        // setup authentication
        if (!StringUtils.isNullOrEmpty(username)) {
            setupAuthentication(httpClientBuilder);
        }

        // set proxy
        if (AtsSystemProperties.SYSTEM_HTTP_PROXY_HOST != null
            && AtsSystemProperties.SYSTEM_HTTP_PROXY_PORT != null) {

            HttpHost proxy = new HttpHost(AtsSystemProperties.SYSTEM_HTTP_PROXY_HOST,
                                          Integer.parseInt(AtsSystemProperties.SYSTEM_HTTP_PROXY_PORT));
            DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
            httpClientBuilder.setRoutePlanner(routePlanner);
        }

        // now build the client after we have already set everything needed on the client builder
        httpClient = httpClientBuilder.build();

        // do not come here again until not needed
        needsInternalClientInialization = false;
    }
 

/**
 * Configure the proxy with the required credential if needed
 *
 * @param httpClientBuilder
 * @param credentialsProvider
 * @param url
 * @return {@link HttpClientBuilder}
 */
private HttpClientBuilder configureProxy(HttpClientBuilder httpClientBuilder, CredentialsProvider credentialsProvider, String url) {
	if (proxyConfig == null) {
		return httpClientBuilder;
	}

	final String protocol = getURL(url).getProtocol();
	final boolean proxyHTTPS = Protocol.isHttps(protocol) && (proxyConfig.getHttpsProperties() != null);
	final boolean proxyHTTP = Protocol.isHttp(protocol) && (proxyConfig.getHttpProperties() != null);

	ProxyProperties proxyProps = null;
	if (proxyHTTPS) {
		LOG.debug("Use proxy https parameters");
		proxyProps = proxyConfig.getHttpsProperties();
	} else if (proxyHTTP) {
		LOG.debug("Use proxy http parameters");
		proxyProps = proxyConfig.getHttpProperties();
	} else {
		return httpClientBuilder;
	}

	String proxyHost = proxyProps.getHost();
	int proxyPort = proxyProps.getPort();
	String proxyUser = proxyProps.getUser();
	String proxyPassword = proxyProps.getPassword();
	String proxyExcludedHosts = proxyProps.getExcludedHosts();

	if (Utils.isStringNotEmpty(proxyUser) && Utils.isStringNotEmpty(proxyPassword)) {
		AuthScope proxyAuth = new AuthScope(proxyHost, proxyPort);
		UsernamePasswordCredentials proxyCredentials = new UsernamePasswordCredentials(proxyUser, proxyPassword);
		credentialsProvider.setCredentials(proxyAuth, proxyCredentials);
	}

	LOG.debug("proxy host/port: {}:{}", proxyHost, proxyPort);
	// TODO SSL peer shut down incorrectly when protocol is https
	final HttpHost proxy = new HttpHost(proxyHost, proxyPort, Protocol.HTTP.getName());

	if (Utils.isStringNotEmpty(proxyExcludedHosts)) {
		final String[] hosts = proxyExcludedHosts.split("[,; ]");

		HttpRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy) {
			@Override
			public HttpRoute determineRoute(final HttpHost host, final HttpRequest request, final HttpContext context) throws HttpException {

				String hostname = (host != null ? host.getHostName() : null);

				if ((hosts != null) && (hostname != null)) {
					for (String h : hosts) {
						if (hostname.equalsIgnoreCase(h)) {
							// bypass proxy for that hostname
							return new HttpRoute(host);
						}
					}
				}
				return super.determineRoute(host, request, context);
			}
		};

		httpClientBuilder.setRoutePlanner(routePlanner);
	}

	return httpClientBuilder.setProxy(proxy);
}
 

private HttpRoutePlanner getNewProxyRoutePlanner() {
    HttpHost httpHost = new HttpHost(proxy.getHost(), proxy.getPort());
    return new DefaultProxyRoutePlanner(httpHost);
}
 

@Test
public void givenServerThatIsBehindAuthorizationProxy_whenClientSendRequest_shouldAuthorizeProperly() throws IOException {
    //given
    proxyMock.stubFor(get(urlMatching("/private"))
      .willReturn(aResponse().proxiedFrom("http://localhost:8089/")));
    serviceMock.stubFor(get(urlEqualTo("/private"))
      .willReturn(aResponse().withStatus(200)));


    HttpHost proxy = new HttpHost("localhost", 8090);
    DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);

    // Client credentials
    CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    credentialsProvider.setCredentials(new AuthScope(proxy),
      new UsernamePasswordCredentials("username_admin", "secret_password"));


    // Create AuthCache instance
    AuthCache authCache = new BasicAuthCache();

    // Generate BASIC scheme object and add it to the local auth cache
    BasicScheme basicAuth = new BasicScheme();
    authCache.put(proxy, basicAuth);
    HttpClientContext context = HttpClientContext.create();
    context.setCredentialsProvider(credentialsProvider);
    context.setAuthCache(authCache);


    HttpClient httpclient = HttpClients.custom()
      .setRoutePlanner(routePlanner)
      .setDefaultCredentialsProvider(credentialsProvider)
      .build();


    //when
    final HttpGet httpGet = new HttpGet("http://localhost:8089/private");
    HttpResponse response = httpclient.execute(httpGet, context);

    //then
    assertEquals(response.getStatusLine().getStatusCode(), 200);
    proxyMock.verify(getRequestedFor(urlEqualTo("/private")).withHeader("Authorization", containing("Basic")));
    serviceMock.verify(getRequestedFor(urlEqualTo("/private")));
}
 

@Override
public void configure(final Config conf) {

    super.configure(conf);

    // allow up to 200 connections or same as the number of threads used for
    // fetching
    int maxFetchThreads = ConfUtils.getInt(conf, "fetcher.threads.number",
            200);
    CONNECTION_MANAGER.setMaxTotal(maxFetchThreads);
    int maxPerRoute = ConfUtils
            .getInt(conf, "fetcher.threads.per.queue", 1);
    if (maxPerRoute < 20) {
        maxPerRoute = 20;
    }
    CONNECTION_MANAGER.setDefaultMaxPerRoute(maxPerRoute);

    this.maxContent = ConfUtils.getInt(conf, "http.content.limit", -1);

    String userAgent = getAgentString(
            ConfUtils.getString(conf, "http.agent.name"),
            ConfUtils.getString(conf, "http.agent.version"),
            ConfUtils.getString(conf, "http.agent.description"),
            ConfUtils.getString(conf, "http.agent.url"),
            ConfUtils.getString(conf, "http.agent.email"));

    Collection<BasicHeader> defaultHeaders = new LinkedList<>();

    String accept = ConfUtils.getString(conf, "http.accept");
    if (StringUtils.isNotBlank(accept)) {
        defaultHeaders.add(new BasicHeader("Accept", accept));
    }

    String basicAuthUser = ConfUtils.getString(conf, "http.basicauth.user",
            null);

    // use a basic auth?
    if (StringUtils.isNotBlank(basicAuthUser)) {
        String basicAuthPass = ConfUtils.getString(conf,
                "http.basicauth.password", "");
        String encoding = Base64.getEncoder().encodeToString(
                (basicAuthUser + ":" + basicAuthPass).getBytes());
        defaultHeaders.add(new BasicHeader("Authorization", "Basic "
                + encoding));
    }

    String acceptLanguage = ConfUtils.getString(conf,
            "http.accept.language");
    if (StringUtils.isNotBlank(acceptLanguage)) {
        defaultHeaders.add(new BasicHeader("Accept-Language",
                acceptLanguage));
    }

    builder = HttpClients.custom().setUserAgent(userAgent)
            .setDefaultHeaders(defaultHeaders)
            .setConnectionManager(CONNECTION_MANAGER)
            .setConnectionManagerShared(true).disableRedirectHandling()
            .disableAutomaticRetries();

    int timeout = ConfUtils.getInt(conf, "http.timeout", 10000);

    RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
            .setSocketTimeout(timeout).setConnectTimeout(timeout)
            .setConnectionRequestTimeout(timeout)
            .setCookieSpec(CookieSpecs.STANDARD);

    String proxyHost = ConfUtils.getString(conf, "http.proxy.host", null);
    int proxyPort = ConfUtils.getInt(conf, "http.proxy.port", 8080);

    boolean useProxy = proxyHost != null && proxyHost.length() > 0;

    // use a proxy?
    if (useProxy) {

        String proxyUser = ConfUtils.getString(conf, "http.proxy.user",
                null);
        String proxyPass = ConfUtils.getString(conf, "http.proxy.pass",
                null);

        if (StringUtils.isNotBlank(proxyUser)
                && StringUtils.isNotBlank(proxyPass)) {
            List<String> authSchemes = new ArrayList<>();
            // Can make configurable and add more in future
            authSchemes.add(AuthSchemes.BASIC);
            requestConfigBuilder.setProxyPreferredAuthSchemes(authSchemes);

            BasicCredentialsProvider basicAuthCreds = new BasicCredentialsProvider();
            basicAuthCreds.setCredentials(new AuthScope(proxyHost,
                    proxyPort), new UsernamePasswordCredentials(proxyUser,
                    proxyPass));
            builder.setDefaultCredentialsProvider(basicAuthCreds);
        }

        HttpHost proxy = new HttpHost(proxyHost, proxyPort);
        DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(
                proxy);
        builder.setRoutePlanner(routePlanner);
    }

    requestConfig = requestConfigBuilder.build();
}