下面列出了org.apache.http.client.methods.HttpExecutionAware#org.apache.http.client.methods.HttpRequestWrapper 实例代码,或者点击链接到github查看源代码,也可以在右侧发表评论。
private CloseableHttpResponse executeWithSignature(HttpRoute route, HttpRequestWrapper request,
HttpClientContext context, HttpExecutionAware execAware) throws IOException, HttpException {
// 上传类不需要消耗两次故不做转换
if (!(request.getOriginal() instanceof WechatPayUploadHttpPost)) {
convertToRepeatableRequestEntity(request);
}
// 添加认证信息
request.addHeader("Authorization",
credentials.getSchema() + " " + credentials.getToken(request));
// 执行
CloseableHttpResponse response = mainExec.execute(route, request, context, execAware);
// 对成功应答验签
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() >= 200 && statusLine.getStatusCode() < 300) {
convertToRepeatableResponseEntity(response);
if (!validator.validate(response)) {
throw new HttpException("应答的微信支付签名验证失败");
}
}
return response;
}
@Override
public final String getToken(HttpRequestWrapper request) throws IOException {
String nonceStr = generateNonceStr();
long timestamp = generateTimestamp();
String message = buildMessage(nonceStr, timestamp, request);
log.debug("authorization message=[{}]", message);
Signer.SignatureResult signature = signer.sign(message.getBytes(StandardCharsets.UTF_8));
String token = "mchid=\"" + getMerchantId() + "\","
+ "nonce_str=\"" + nonceStr + "\","
+ "timestamp=\"" + timestamp + "\","
+ "serial_no=\"" + signature.certificateSerialNumber + "\","
+ "signature=\"" + signature.sign + "\"";
log.debug("authorization token=[{}]", token);
return token;
}
protected final String buildMessage(String nonce, long timestamp, HttpRequestWrapper request)
throws IOException {
URI uri = request.getURI();
String canonicalUrl = uri.getRawPath();
if (uri.getQuery() != null) {
canonicalUrl += "?" + uri.getRawQuery();
}
String body = "";
// PATCH,POST,PUT
if (request.getOriginal() instanceof WechatPayUploadHttpPost) {
body = ((WechatPayUploadHttpPost) request.getOriginal()).getMeta();
} else if (request instanceof HttpEntityEnclosingRequest) {
body = EntityUtils.toString(((HttpEntityEnclosingRequest) request).getEntity());
}
return request.getRequestLine().getMethod() + "\n"
+ canonicalUrl + "\n"
+ timestamp + "\n"
+ nonce + "\n"
+ body + "\n";
}
@Advice.OnMethodEnter(suppress = Throwable.class)
private static void onBeforeExecute(@Advice.Argument(0) HttpRoute route,
@Advice.Argument(1) HttpRequestWrapper request,
@Advice.Local("span") Span span) {
if (tracer == null || tracer.getActive() == null) {
return;
}
final AbstractSpan<?> parent = tracer.getActive();
span = HttpClientHelper.startHttpClientSpan(parent, request.getMethod(), request.getURI(), route.getTargetHost().getHostName());
TextHeaderSetter<HttpRequest> headerSetter = headerSetterHelperClassManager.getForClassLoaderOfClass(HttpRequest.class);
TextHeaderGetter<HttpRequest> headerGetter = headerGetterHelperClassManager.getForClassLoaderOfClass(HttpRequest.class);
if (span != null) {
span.activate();
if (headerSetter != null) {
span.propagateTraceContext(request, headerSetter);
}
} else if (headerGetter != null && !TraceContext.containsTraceContextTextHeaders(request, headerGetter)
&& headerSetter != null && parent != null) {
// re-adds the header on redirects
parent.propagateTraceContext(request, headerSetter);
}
}
@Override
public HttpUriRequest getRedirect(HttpRequest request, HttpResponse response, HttpContext context) throws ProtocolException {
URI uri = getLocationURI(request, response, context);
String method = request.getRequestLine().getMethod();
if (HttpConstants.POST.equalsIgnoreCase(method)) {
try {
HttpRequestWrapper httpRequestWrapper = (HttpRequestWrapper) request;
httpRequestWrapper.setURI(uri);
httpRequestWrapper.removeHeaders("Content-Length");
return httpRequestWrapper;
} catch (Exception e) {
e.printStackTrace();
}
return new HttpPost(uri);
} else {
return new HttpGet(uri);
}
}
@Override
public HttpUriRequest getRedirect(HttpRequest request, HttpResponse response, HttpContext context) throws ProtocolException {
URI uri = getLocationURI(request, response, context);
String method = request.getRequestLine().getMethod();
if ("post".equalsIgnoreCase(method)) {
try {
HttpRequestWrapper httpRequestWrapper = (HttpRequestWrapper) request;
httpRequestWrapper.setURI(uri);
httpRequestWrapper.removeHeaders("Content-Length");
return httpRequestWrapper;
} catch (Exception e) {
log.error("强转为HttpRequestWrapper出错");
}
return new HttpPost(uri);
} else {
return new HttpGet(uri);
}
}
@Override
public CloseableHttpResponse execute(HttpRoute route, HttpRequestWrapper request, HttpClientContext clientContext,
HttpExecutionAware execAware) throws IOException, HttpException {
Proxy proxy = (Proxy) clientContext.getAttribute(VSCrawlerConstant.VSCRAWLER_AVPROXY_KEY);
if (proxy != null) {
proxy.recordUsage();
}
try {
return delegate.execute(route, request, clientContext, execAware);
} catch (IOException ioe) {
if (proxy != null) {
proxy.recordFailed();
}
throw ioe;
}
}
@Override
public Proxy determineProxy(HttpHost host, HttpRequest request, HttpContext context, IPPool ipPool,
CrawlerSession crawlerSession) {
HttpClientContext httpClientContext = HttpClientContext.adapt(context);
Proxy proxy = (Proxy) crawlerSession.getExtInfo(VSCRAWLER_AVPROXY_KEY);
if (proxy == null) {
String accessUrl = null;
if (request instanceof HttpRequestWrapper || request instanceof HttpGet) {
accessUrl = HttpUriRequest.class.cast(request).getURI().toString();
}
if (!PoolUtil.isDungProxyEnabled(httpClientContext)) {
log.info("{}不会被代理", accessUrl);
return null;
}
proxy = ipPool.getIP(host.getHostName(), accessUrl);
if (proxy == null) {
return null;
}
crawlerSession.setExtInfo(VSCRAWLER_AVPROXY_KEY, proxy);
}
return proxy;
}
@Override
public @Nullable String getRequestUrl(@Nullable HttpRequest request) {
if (request == null) {
return null;
}
String uri = request.getRequestLine().getUri();
if (request instanceof HttpRequestWrapper && uri.startsWith("/")) {
HttpRequestWrapper wrapper = (HttpRequestWrapper) request;
HttpHost target = wrapper.getTarget();
if (target != null) {
uri = wrapper.getTarget().toURI() + uri;
}
}
return uri;
}
@DataProvider(value = {
"http://foo.bar/some/path | /some/path",
"http://foo.bar/some/path?thing=stuff | /some/path",
"/some/path | /some/path",
"/some/path?thing=stuff | /some/path",
"http://foo.bar/ | /",
"http://foo.bar/?thing=stuff | /",
"/ | /",
"/?thing=stuff | /",
}, splitBy = "\\|")
@Test
public void getRequestPath_works_as_expected_for_a_request_that_is_an_HttpRequestWrapper(
String uriString, String expectedResult
) {
// given
HttpRequestWrapper requestWrapperMock = mock(HttpRequestWrapper.class);
doReturn(URI.create(uriString)).when(requestWrapperMock).getURI();
// when
String result = implSpy.getRequestPath(requestWrapperMock);
// then
assertThat(result).isEqualTo(expectedResult);
}
GetRequestUrlScenario(
boolean requestIsNull, boolean requestIsWrapper, Object baseUri, boolean targetIsNull,
String targetUri, String expectedResult
) {
this.expectedResult = expectedResult;
HttpRequest request = null;
if (!requestIsNull) {
request = (requestIsWrapper) ? mock(HttpRequestWrapper.class) : mock(HttpRequest.class);
RequestLine requestLine = mock(RequestLine.class);
doReturn(requestLine).when(request).getRequestLine();
doReturn(baseUri).when(requestLine).getUri();
if (!targetIsNull) {
HttpHost target = HttpHost.create(targetUri);
doReturn(target).when((HttpRequestWrapper) request).getTarget();
}
}
this.requestMock = request;
}
private String inferUri(HttpRequest request) {
String inferredUri = request.getRequestLine().getUri();
if (!isValidUrl(inferredUri)) { // Missing schema and domain name
String host = getHost(request);
String inferredScheme = "http";
if (host.endsWith(":443")) {
inferredScheme = "https";
} else if ((request instanceof RequestWrapper) || (request instanceof HttpRequestWrapper)) {
if (getOriginalRequestUri(request).startsWith("https")) {
// This is for original URL, so if during redirects we go out of HTTPs, this might be a wrong guess
inferredScheme = "https";
}
}
if ("CONNECT".equals(request.getRequestLine().getMethod())) {
inferredUri = String.format("%s://%s", inferredScheme, host);
} else {
inferredUri =
String.format("%s://%s/%s", inferredScheme, host, inferredUri)
.replaceAll("(?<!http(s)?:)//", "/");
}
}
return inferredUri;
}
@Test
public void testProcess() throws Exception {
URI testURI = new URIBuilder()
.setPath("/1/statuses/update.json")
.setParameter("include_entities", "true")
.build();
HttpPost testRequest = new HttpPost(testURI);
testRequest.setEntity(new StringEntity("status="+security.encode("Hello Ladies + Gentlemen, a signed OAuth request!")));
HttpHost host = new HttpHost("api.twitter.com", -1, "https");
HttpRequestWrapper wrapper = HttpRequestWrapper.wrap(testRequest, host);
TwitterOAuthConfiguration testOauthConfiguration = new TwitterOAuthConfiguration()
.withConsumerKey("xvz1evFS4wEEPTGEFPHBog")
.withConsumerSecret("kAcSOqF21Fu85e7zjz7ZN2U4ZRhfV3WpwPAoE3Z7kBw")
.withAccessToken("370773112-GmHxMAgYyLbNEtIKZeRNFsMKPR9EyMZeS9weJAEb")
.withAccessTokenSecret("LswwdoUaIvS8ltyTt5jkRh4J50vUPVVHtR2YPi5kE");
TwitterOAuthRequestInterceptor interceptor = Mockito.spy(new TwitterOAuthRequestInterceptor(testOauthConfiguration));
Mockito.when(interceptor.generateNonce()).thenReturn("kYjzVBB8Y0ZFabxSWbWovY3uYSQ2pTgmZeNu2VS4cg");
Mockito.when(interceptor.generateTimestamp()).thenReturn("1318622958");
interceptor.process(wrapper, new HttpCoreContext());
assertEquals(1, wrapper.getHeaders("Authorization").length);
String actual = wrapper.getFirstHeader("Authorization").getValue();
String expected = "OAuth oauth_consumer_key=\"xvz1evFS4wEEPTGEFPHBog\", oauth_nonce=\"kYjzVBB8Y0ZFabxSWbWovY3uYSQ2pTgmZeNu2VS4cg\", oauth_signature=\"tnnArxj06cWHq44gCs1OSKk%2FjLY%3D\", oauth_signature_method=\"HMAC-SHA1\", oauth_timestamp=\"1318622958\", oauth_token=\"370773112-GmHxMAgYyLbNEtIKZeRNFsMKPR9EyMZeS9weJAEb\", oauth_version=\"1.0\"";
assertEquals(expected, actual);
}
@Override
public HttpUriRequest getRedirect(HttpRequest request, HttpResponse response, HttpContext context) throws ProtocolException {
URI uri = getLocationURI(request, response, context);
String method = request.getRequestLine().getMethod();
if ("post".equalsIgnoreCase(method)) {
try {
HttpRequestWrapper httpRequestWrapper = (HttpRequestWrapper) request;
httpRequestWrapper.setURI(uri);
httpRequestWrapper.removeHeaders("Content-Length");
return httpRequestWrapper;
} catch (Exception e) {
logger.error("强转为HttpRequestWrapper出错");
}
return new HttpPost(uri);
} else {
return new HttpGet(uri);
}
}
protected void convertToRepeatableRequestEntity(HttpRequestWrapper request) throws IOException {
if (request instanceof HttpEntityEnclosingRequest) {
HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
if (entity != null) {
((HttpEntityEnclosingRequest) request).setEntity(new BufferedHttpEntity(entity));
}
}
}
@Override
public CloseableHttpResponse execute(HttpRoute route, HttpRequestWrapper request,
HttpClientContext context, HttpExecutionAware execAware) throws IOException, HttpException {
if (request.getURI().getHost().endsWith(".mch.weixin.qq.com")) {
return executeWithSignature(route, request, context, execAware);
} else {
return mainExec.execute(route, request, context, execAware);
}
}
public void appendHttpClientRequestSpanTags(HttpRequest httpRequest,
SofaTracerSpan httpClientSpan) {
if (httpClientSpan == null) {
return;
}
if (this.appName == null) {
this.appName = SofaTracerConfiguration.getProperty(
SofaTracerConfiguration.TRACER_APPNAME_KEY, StringUtils.EMPTY_STRING);
}
//lazy init
RequestLine requestLine = httpRequest.getRequestLine();
String methodName = requestLine.getMethod();
//appName
httpClientSpan.setTag(CommonSpanTags.LOCAL_APP,
this.appName == null ? StringUtils.EMPTY_STRING : this.appName);
//targetAppName
httpClientSpan.setTag(CommonSpanTags.REMOTE_APP,
this.targetAppName == null ? StringUtils.EMPTY_STRING : this.targetAppName);
if (httpRequest instanceof HttpRequestWrapper) {
HttpRequestWrapper httpRequestWrapper = (HttpRequestWrapper) httpRequest;
httpClientSpan.setTag(CommonSpanTags.REQUEST_URL, httpRequestWrapper.getOriginal()
.getRequestLine().getUri());
} else {
httpClientSpan.setTag(CommonSpanTags.REQUEST_URL, requestLine.getUri());
}
//method
httpClientSpan.setTag(CommonSpanTags.METHOD, methodName);
//length
if (httpRequest instanceof HttpEntityEnclosingRequest) {
HttpEntityEnclosingRequest httpEntityEnclosingRequest = (HttpEntityEnclosingRequest) httpRequest;
HttpEntity httpEntity = httpEntityEnclosingRequest.getEntity();
httpClientSpan.setTag(CommonSpanTags.REQ_SIZE,
httpEntity == null ? -1 : httpEntity.getContentLength());
}
//carrier
this.processHttpClientRequestCarrier(httpRequest, httpClientSpan);
}
@Override
public Proxy determineProxy(HttpHost host, HttpRequest request, HttpContext context, IPPool ipPool,
CrawlerSession crawlerSession) {
HttpClientContext httpClientContext = HttpClientContext.adapt(context);
Proxy bind = (Proxy) context.getAttribute(VSCrawlerConstant.VSCRAWLER_AVPROXY_KEY);
String accessUrl = null;
if (request instanceof HttpRequestWrapper || request instanceof HttpGet) {
accessUrl = HttpUriRequest.class.cast(request).getURI().toString();
}
if (!PoolUtil.isDungProxyEnabled(httpClientContext)) {
log.info("{}不会被代理", accessUrl);
return null;
}
if (bind == null || bind.isDisable()) {
bind = ipPool.getIP(host.getHostName(), accessUrl);
}
if (bind == null) {
return null;
}
log.info("{} 当前使用IP为:{}:{}", host.getHostName(), bind.getIp(), bind.getPort());
// 将绑定IP放置到context,用于后置拦截器统计这个IP的使用情况
return bind;
}
@Override
public Proxy determineProxy(HttpHost host, HttpRequest request, HttpContext context, IPPool ipPool,
CrawlerSession crawlerSession) {
HttpClientContext httpClientContext = HttpClientContext.adapt(context);
User user = UserUtil.getUser(crawlerSession);
if (user == null) {
log.warn(
"you config proxy strategy by user,but this session has not login with a user,proxy bind will be ignore");
return null;
}
Proxy proxy = (Proxy) user.getExtInfo().get(VSCrawlerConstant.VSCRAWLER_AVPROXY_KEY);
if (proxy == null) {
String accessUrl = null;
if (request instanceof HttpRequestWrapper || request instanceof HttpGet) {
accessUrl = HttpUriRequest.class.cast(request).getURI().toString();
}
if (!PoolUtil.isDungProxyEnabled(httpClientContext)) {
log.info("{}不会被代理", accessUrl);
return null;
}
proxy = ipPool.getIP(host.getHostName(), accessUrl);
if (proxy == null) {
return null;
}
user.getExtInfo().put(VSCrawlerConstant.VSCRAWLER_AVPROXY_KEY, proxy);
}
return proxy;
}
@Override
public CloseableHttpResponse execute(HttpUriRequest request) throws IOException, ClientProtocolException {
HttpRequestWrapper wrap = HttpRequestWrapper.wrap(request);
adviseRobotsTxt(wrap.getURI());
wrap.setURI(applyPHP(wrap.getURI()));
return client.execute(wrap);
}
@Override
public CloseableHttpResponse execute(HttpUriRequest request, HttpContext context) throws IOException, ClientProtocolException {
HttpRequestWrapper wrap = HttpRequestWrapper.wrap(request);
adviseRobotsTxt(wrap.getURI());
wrap.setURI(applyPHP(wrap.getURI()));
return client.execute(wrap, context);
}
@Override
public CloseableHttpResponse execute(HttpHost target, HttpRequest request) throws IOException, ClientProtocolException {
HttpRequestWrapper wrap = HttpRequestWrapper.wrap(request, target);
adviseRobotsTxt(wrap.getURI());
wrap.setURI(applyPHP(wrap.getURI()));
return client.execute(wrap);
}
@Override
public CloseableHttpResponse execute(HttpHost target, HttpRequest request, HttpContext context) throws IOException, ClientProtocolException {
HttpRequestWrapper wrap = HttpRequestWrapper.wrap(request, target);
adviseRobotsTxt(wrap.getURI());
wrap.setURI(applyPHP(wrap.getURI()));
return client.execute(wrap, context);
}
@Override
public <T> T execute(HttpUriRequest request, ResponseHandler<? extends T> responseHandler) throws IOException, ClientProtocolException {
HttpRequestWrapper wrap = HttpRequestWrapper.wrap(request);
adviseRobotsTxt(wrap.getURI());
wrap.setURI(applyPHP(wrap.getURI()));
return client.execute(wrap, responseHandler);
}
@Override
public <T> T execute(HttpUriRequest request, ResponseHandler<? extends T> responseHandler, HttpContext context) throws IOException, ClientProtocolException {
HttpRequestWrapper wrap = HttpRequestWrapper.wrap(request);
adviseRobotsTxt(wrap.getURI());
wrap.setURI(applyPHP(wrap.getURI()));
return client.execute(wrap, responseHandler, context);
}
@Override
public <T> T execute(HttpHost target, HttpRequest request, ResponseHandler<? extends T> responseHandler) throws IOException, ClientProtocolException {
HttpRequestWrapper wrap = HttpRequestWrapper.wrap(request,target);
adviseRobotsTxt(wrap.getURI());
wrap.setURI(applyPHP(wrap.getURI()));
return client.execute(wrap, responseHandler);
}
@Override
public <T> T execute(HttpHost target, HttpRequest request, ResponseHandler<? extends T> responseHandler, HttpContext context) throws IOException, ClientProtocolException {
HttpRequestWrapper wrap = HttpRequestWrapper.wrap(request,target);
adviseRobotsTxt(wrap.getURI());
wrap.setURI(applyPHP(wrap.getURI()));
return client.execute(wrap, responseHandler, context);
}
@Test
public void curlVerifyHost () {
this.curl ("https://put.anything.in.this.url:1337",
context ->
assertEquals (URI.create ("https://put.anything.in.this.url:1337"),
((HttpRequestBase)((HttpRequestWrapper)
context.getAttribute ("http.request")).getOriginal ()).getURI())
);
}
@Test
public void curlVerifyHeader () {
this.curl ("-H 'Titi: toto' https://put.anything.in.this.url:1337",
context ->
assertEquals ("toto",
((HttpRequestWrapper)
context.getAttribute ("http.request"))
.getLastHeader ("Titi").getValue ()));
}
@Test
public void curlVerifyUserAgent () {
this.curl ("-H 'User-Agent: toto' https://put.anything.in.this.url:1337",
context ->
assertEquals ("toto",
((HttpRequestWrapper)
context.getAttribute ("http.request"))
.getLastHeader ("User-Agent").getValue ()));
}