1 /*
2  * Copyright 2010-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License").
5  * You may not use this file except in compliance with the License.
6  * A copy of the License is located at
7  *
8  *  http://aws.amazon.com/apache2.0
9  *
10  * or in the "license" file accompanying this file. This file is distributed
11  * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12  * express or implied. See the License for the specific language governing
13  * permissions and limitations under the License.
14  */
15 package com.amazonaws.http;
16
17 import static com.amazonaws.SDKGlobalConfiguration.PROFILING_SYSTEM_PROPERTY;
18 import static com.amazonaws.event.SDKProgressPublisher.publishProgress;
19 import static com.amazonaws.event.SDKProgressPublisher.publishRequestContentLength;
20 import static com.amazonaws.event.SDKProgressPublisher.publishResponseContentLength;
21 import static com.amazonaws.util.AWSRequestMetrics.Field.HttpClientPoolAvailableCount;
22 import static com.amazonaws.util.AWSRequestMetrics.Field.HttpClientPoolLeasedCount;
23 import static com.amazonaws.util.AWSRequestMetrics.Field.HttpClientPoolPendingCount;
24 import static com.amazonaws.util.AWSRequestMetrics.Field.ThrottledRetryCount;
25 import static com.amazonaws.util.AwsClientSideMonitoringMetrics.MaxRetriesExceeded;
26 import static com.amazonaws.util.IOUtils.closeQuietly;
27
28 import com.amazonaws.AbortedException;
29 import com.amazonaws.AmazonClientException;
30 import com.amazonaws.AmazonServiceException;
31 import com.amazonaws.AmazonWebServiceRequest;
32 import com.amazonaws.AmazonWebServiceResponse;
33 import com.amazonaws.ClientConfiguration;
34 import com.amazonaws.Request;
35 import com.amazonaws.RequestClientOptions;
36 import com.amazonaws.RequestClientOptions.Marker;
37 import com.amazonaws.RequestConfig;
38 import com.amazonaws.ResetException;
39 import com.amazonaws.Response;
40 import com.amazonaws.ResponseMetadata;
41 import com.amazonaws.SDKGlobalTime;
42 import com.amazonaws.SdkBaseException;
43 import com.amazonaws.SdkClientException;
44 import com.amazonaws.annotation.SdkInternalApi;
45 import com.amazonaws.annotation.SdkTestInternalApi;
46 import com.amazonaws.annotation.ThreadSafe;
47 import com.amazonaws.auth.AWS4Signer;
48 import com.amazonaws.auth.AWSCredentials;
49 import com.amazonaws.auth.AWSCredentialsProvider;
50 import com.amazonaws.auth.CanHandleNullCredentials;
51 import com.amazonaws.auth.Signer;
52 import com.amazonaws.event.ProgressEventType;
53 import com.amazonaws.event.ProgressInputStream;
54 import com.amazonaws.event.ProgressListener;
55 import com.amazonaws.handlers.CredentialsRequestHandler;
56 import com.amazonaws.handlers.HandlerAfterAttemptContext;
57 import com.amazonaws.handlers.HandlerBeforeAttemptContext;
58 import com.amazonaws.handlers.HandlerContextKey;
59 import com.amazonaws.handlers.RequestHandler2;
60 import com.amazonaws.http.apache.client.impl.ApacheHttpClientFactory;
61 import com.amazonaws.http.apache.client.impl.ConnectionManagerAwareHttpClient;
62 import com.amazonaws.http.apache.request.impl.ApacheHttpRequestFactory;
63 import com.amazonaws.http.apache.utils.ApacheUtils;
64 import com.amazonaws.http.client.HttpClientFactory;
65 import com.amazonaws.http.exception.HttpRequestTimeoutException;
66 import com.amazonaws.http.request.HttpRequestFactory;
67 import com.amazonaws.http.response.AwsResponseHandlerAdapter;
68 import com.amazonaws.http.settings.HttpClientSettings;
69 import com.amazonaws.http.timers.client.ClientExecutionAbortTrackerTask;
70 import com.amazonaws.http.timers.client.ClientExecutionTimeoutException;
71 import com.amazonaws.http.timers.client.ClientExecutionTimer;
72 import com.amazonaws.http.timers.client.SdkInterruptedException;
73 import com.amazonaws.http.timers.request.HttpRequestAbortTaskTracker;
74 import com.amazonaws.http.timers.request.HttpRequestTimer;
75 import com.amazonaws.internal.AmazonWebServiceRequestAdapter;
76 import com.amazonaws.internal.CRC32MismatchException;
77 import com.amazonaws.internal.ReleasableInputStream;
78 import com.amazonaws.internal.ResettableInputStream;
79 import com.amazonaws.internal.SdkBufferedInputStream;
80 import com.amazonaws.internal.auth.SignerProviderContext;
81 import com.amazonaws.metrics.AwsSdkMetrics;
82 import com.amazonaws.metrics.RequestMetricCollector;
83 import com.amazonaws.monitoring.internal.ClientSideMonitoringRequestHandler;
84 import com.amazonaws.retry.ClockSkewAdjuster;
85 import com.amazonaws.retry.ClockSkewAdjuster.AdjustmentRequest;
86 import com.amazonaws.retry.ClockSkewAdjuster.ClockSkewAdjustment;
87 import com.amazonaws.retry.RetryMode;
88 import com.amazonaws.retry.RetryPolicyAdapter;
89 import com.amazonaws.retry.RetryUtils;
90 import com.amazonaws.internal.SdkRequestRetryHeaderProvider;
91 import com.amazonaws.retry.internal.AuthErrorRetryStrategy;
92 import com.amazonaws.retry.internal.AuthRetryParameters;
93 import com.amazonaws.retry.v2.RetryPolicy;
94 import com.amazonaws.retry.v2.RetryPolicyContext;
95 import com.amazonaws.util.AWSRequestMetrics;
96 import com.amazonaws.util.AWSRequestMetrics.Field;
97 import com.amazonaws.util.AwsClientSideMonitoringMetrics;
98 import com.amazonaws.util.CapacityManager;
99 import com.amazonaws.util.CollectionUtils;
100 import com.amazonaws.util.CountingInputStream;
101 import com.amazonaws.util.FakeIOException;
102 import com.amazonaws.util.ImmutableMapParameter;
103 import com.amazonaws.util.MetadataCache;
104 import com.amazonaws.util.NullResponseMetadataCache;
105 import com.amazonaws.util.ResponseMetadataCache;
106 import com.amazonaws.util.RuntimeHttpUtils;
107 import com.amazonaws.util.SdkHttpUtils;
108 import com.amazonaws.util.UnreliableFilterInputStream;
109 import java.io.BufferedInputStream;
110 import java.io.Closeable;
111 import java.io.FileInputStream;
112 import java.io.IOException;
113 import java.io.InputStream;
114 import java.net.SocketTimeoutException;
115 import java.net.URI;
116 import java.util.Arrays;
117 import java.util.Collections;
118 import java.util.HashMap;
119 import java.util.LinkedHashMap;
120 import java.util.List;
121 import java.util.Map;
122 import java.util.Map.Entry;
123 import java.util.Random;
124 import java.util.UUID;
125 import org.apache.commons.logging.Log;
126 import org.apache.commons.logging.LogFactory;
127 import org.apache.http.Header;
128 import org.apache.http.HttpEntity;
129 import org.apache.http.HttpEntityEnclosingRequest;
130 import org.apache.http.HttpStatus;
131 import org.apache.http.StatusLine;
132 import org.apache.http.client.methods.HttpRequestBase;
133 import org.apache.http.client.protocol.HttpClientContext;
134 import org.apache.http.conn.ConnectTimeoutException;
135 import org.apache.http.entity.BufferedHttpEntity;
136 import org.apache.http.impl.execchain.RequestAbortedException;
137 import org.apache.http.pool.ConnPoolControl;
138 import org.apache.http.pool.PoolStats;
139 import org.apache.http.protocol.HttpContext;
140
141 @ThreadSafe
142 public class AmazonHttpClient {
143
144     public static final String HEADER_USER_AGENT = "User-Agent";
145     public static final String HEADER_SDK_TRANSACTION_ID = "amz-sdk-invocation-id";
146     public static final String HEADER_SDK_RETRY_INFO = "amz-sdk-retry";
147
148     /**
149      * Logger for more detailed debugging information, that might not be as useful for end users
150      * (ex: HTTP client configuration, etc).
151      */
152     static final Log log = LogFactory.getLog(AmazonHttpClient.class);
153
154     /**
155      * Logger providing detailed information on requests/responses. Users can enable this logger to
156      * get access to AWS request IDs for responses, individual requests and parameters sent to AWS,
157      * etc.
158      */
159     @SdkInternalApi
160     public static final Log requestLog = LogFactory.getLog("com.amazonaws.request");
161
162     private static final HttpClientFactory<ConnectionManagerAwareHttpClient> httpClientFactory = new
163             ApacheHttpClientFactory();
164     /**
165      * Used for testing via failure injection.
166      */
167     private static UnreliableTestConfig unreliableTestConfig;
168
169     /**
170      * When throttled retries are enabled, each retry attempt will consume this much capacity.
171      * Successful retry attempts will release this capacity back to the pool while failed retries
172      * will not.  Successful initial (non-retry) requests will always release 1 capacity unit to the
173      * pool.
174      */
175     private static final int THROTTLED_RETRY_COST = 5;
176
177     /**
178      * The capacity to acquire for a connection timeout or socket timeout error.
179      */
180     private static final int TIMEOUT_RETRY_COST = 10;
181
182     static {
183         // Customers have reported XML parsing issues with the following
184         // JVM versions, which don't occur with more recent versions, so
185         // if we detect any of these, give customers a heads up.
186         // https://bugs.openjdk.java.net/browse/JDK-8028111
187         List<String> problematicJvmVersions = Arrays
188                 .asList("1.6.0_06", "1.6.0_13", "1.6.0_17", "1.6.0_65", "1.7.0_45");
189         String jvmVersion = System.getProperty("java.version");
190         if (problematicJvmVersions.contains(jvmVersion)) {
191             log.warn("Detected a possible problem with the current JVM version (" + jvmVersion +
192                      ").  " +
193                      "If you experience XML parsing problems using the SDK, try upgrading to a more recent JVM update.");
194         }
195     }
196
197     private final ClockSkewAdjuster clockSkewAdjuster = new ClockSkewAdjuster();
198
199     private final HttpRequestFactory<HttpRequestBase> httpRequestFactory =
200             new ApacheHttpRequestFactory();
201     /**
202      * Internal client for sending HTTP requests
203      */
204     private ConnectionManagerAwareHttpClient httpClient;
205     /**
206      * Client configuration options, such as proxy httpClientSettings, max retries, etc.
207      */
208     private final ClientConfiguration config;
209
210     private final RetryPolicy retryPolicy;
211
212     /**
213      * Client configuration options, such as proxy httpClientSettings, max retries, etc.
214      */
215     private final HttpClientSettings httpClientSettings;
216     /**
217      * Cache of metadata for recently executed requests for diagnostic purposes
218      */
219     private final MetadataCache responseMetadataCache;
220     /**
221      * Timer to enforce HTTP request timeouts.
222      */
223     private final HttpRequestTimer httpRequestTimer;
224
225     /**
226      * Retry capacity manager, used to manage throttled retry resource
227      */
228     private final CapacityManager retryCapacity;
229
230     /**
231      * Timer to enforce timeouts on the whole execution of the request (request handlers, retries,
232      * backoff strategy, unmarshalling, etc)
233      */
234     private final ClientExecutionTimer clientExecutionTimer;
235     /**
236      * A request metric collector used specifically for this httpClientSettings client; or null if
237      * there is none. This collector, if specified, always takes precedence over the one specified
238      * at the AWS SDK level.
239      *
240      * @see AwsSdkMetrics
241      */
242     private final RequestMetricCollector requestMetricCollector;
243
244     /**
245      * Used to generate UUID's for client transaction id. This gives a higher probability of id
246      * clashes but is more performant then using {@link UUID#randomUUID()} which uses SecureRandom
247      * internally.
248      **/
249     private final Random random = new Random();
250
251     /**
252      * The time difference in seconds between this client and AWS.
253      */
254     private volatile int timeOffset = SDKGlobalTime.getGlobalTimeOffset();
255
256     private final RetryMode retryMode;
257
258     private final SdkRequestRetryHeaderProvider sdkRequestHeaderProvider;
259
260     /**
261      * Constructs a new AWS client using the specified client configuration options (ex: max retry
262      * attempts, proxy httpClientSettings, etc).
263      *
264      * @param config Configuration options specifying how this client will communicate with AWS (ex:
265      *               proxy httpClientSettings, retry count, etc.).
266      */
267     public AmazonHttpClient(ClientConfiguration config) {
268         this(config, null);
269     }
270
271     /**
272      * Constructs a new AWS client using the specified client configuration options (ex: max retry
273      * attempts, proxy httpClientSettings, etc), and request metric collector.
274      *
275      * @param config                 Configuration options specifying how this client will
276      *                               communicate with AWS (ex: proxy httpClientSettings, retry
277      *                               count, etc.).
278      * @param requestMetricCollector client specific request metric collector, which takes
279      *                               precedence over the one at the AWS SDK level; or null if there
280      *                               is none.
281      */
282     public AmazonHttpClient(ClientConfiguration config,
283                             RequestMetricCollector requestMetricCollector) {
284         this(config, requestMetricCollector, false);
285     }
286
287     /**
288      * Constructs a new AWS client using the specified client configuration options (ex: max retry
289      * attempts, proxy httpClientSettings, etc), and request metric collector.
290      *
291      * @param config                 Configuration options specifying how this client will
292      *                               communicate with AWS (ex: proxy httpClientSettings, retry
293      *                               count, etc.).
294      * @param requestMetricCollector client specific request metric collector, which takes
295      *                               precedence over the one at the AWS SDK level; or null if there
296      *                               is none.
297      */
298     public AmazonHttpClient(ClientConfiguration config,
299                             RequestMetricCollector requestMetricCollector,
300                             boolean useBrowserCompatibleHostNameVerifier) {
301         this(config, requestMetricCollector, useBrowserCompatibleHostNameVerifier, false);
302     }
303
304     /**
305      * Constructs a new AWS client using the specified client configuration options (ex: max retry
306      * attempts, proxy httpClientSettings, etc), and request metric collector.
307      *
308      * @param config                           Configuration options specifying how this client will
309      *                                         communicate with AWS (ex: proxy httpClientSettings,
310      *                                         retry count, etc.).
311      * @param requestMetricCollector           client specific request metric collector, which takes
312      *                                         precedence over the one at the AWS SDK level; or null
313      *                                         if there is none.
314      * @param calculateCRC32FromCompressedData The flag indicating whether the CRC32 checksum is
315      *                                         calculated from compressed data or not. It is only
316      *                                         applicable when the header "x-amz-crc32" is set in
317      *                                         the response.
318      */
319     public AmazonHttpClient(ClientConfiguration config,
320                             RequestMetricCollector requestMetricCollector,
321                             boolean useBrowserCompatibleHostNameVerifier,
322                             boolean calculateCRC32FromCompressedData) {
323         this(config,
324              null,
325              requestMetricCollector,
326              useBrowserCompatibleHostNameVerifier,
327              calculateCRC32FromCompressedData);
328     }
329
330     private AmazonHttpClient(ClientConfiguration config,
331                              RetryPolicy retryPolicy,
332                              RequestMetricCollector requestMetricCollector,
333                              boolean useBrowserCompatibleHostNameVerifier,
334                              boolean calculateCRC32FromCompressedData) {
335         this(config,
336              retryPolicy,
337              requestMetricCollector,
338              HttpClientSettings.adapt(config, useBrowserCompatibleHostNameVerifier, calculateCRC32FromCompressedData));
339         this.httpClient = httpClientFactory.create(this.httpClientSettings);
340     }
341
342     /**
343      * Package-protected constructor for unit test purposes.
344      */
345     @SdkTestInternalApi
346     public AmazonHttpClient(ClientConfiguration clientConfig,
347                             ConnectionManagerAwareHttpClient httpClient,
348                             RequestMetricCollector requestMetricCollector) {
349         this(clientConfig,
350              null,
351              requestMetricCollector,
352              HttpClientSettings.adapt(clientConfig, false));
353         this.httpClient = httpClient;
354     }
355
356     private AmazonHttpClient(ClientConfiguration clientConfig,
357                              RetryPolicy retryPolicy,
358                              RequestMetricCollector requestMetricCollector,
359                              HttpClientSettings httpClientSettings) {
360         this.config = clientConfig;
361         this.retryPolicy =
362                 retryPolicy == null ? new RetryPolicyAdapter(clientConfig.getRetryPolicy(), clientConfig) : retryPolicy;
363         this.retryMode =
364             clientConfig.getRetryMode() == null ? clientConfig.getRetryPolicy().getRetryMode() : clientConfig.getRetryMode();
365         this.httpClientSettings = httpClientSettings;
366         this.requestMetricCollector = requestMetricCollector;
367         this.responseMetadataCache =
368                 clientConfig.getCacheResponseMetadata() ?
369                         new ResponseMetadataCache(clientConfig.getResponseMetadataCacheSize()) :
370                         new NullResponseMetadataCache();
371         this.httpRequestTimer = new HttpRequestTimer();
372         this.clientExecutionTimer = new ClientExecutionTimer();
373
374         // When enabled, total retry capacity is computed based on retry cost
375         // and desired number of retries.
376         int throttledRetryMaxCapacity = clientConfig.useThrottledRetries()
377                 ? THROTTLED_RETRY_COST * config.getMaxConsecutiveRetriesBeforeThrottling() : -1;
378         this.retryCapacity = new CapacityManager(throttledRetryMaxCapacity);
379         this.sdkRequestHeaderProvider = new SdkRequestRetryHeaderProvider(config, this.retryPolicy, clockSkewAdjuster);
380     }
381
382     public static Builder builder() {
383         return new Builder();
384     }
385
386     public static class Builder {
387
388         private ClientConfiguration clientConfig;
389         private RetryPolicy retryPolicy;
390         private RequestMetricCollector requestMetricCollector;
391         private boolean useBrowserCompatibleHostNameVerifier;
392         private boolean calculateCRC32FromCompressedData;
393
394         private Builder() {
395         }
396
397         public Builder clientConfiguration(ClientConfiguration clientConfig) {
398             this.clientConfig = clientConfig;
399             return this;
400         }
401
402         public Builder retryPolicy(RetryPolicy retryPolicy) {
403             this.retryPolicy = retryPolicy;
404             return this;
405         }
406
407         public Builder requestMetricCollector(RequestMetricCollector requestMetricCollector) {
408             this.requestMetricCollector = requestMetricCollector;
409             return this;
410         }
411
412         public Builder useBrowserCompatibleHostNameVerifier(boolean useBrowserCompatibleHostNameVerifier) {
413             this.useBrowserCompatibleHostNameVerifier = useBrowserCompatibleHostNameVerifier;
414             return this;
415         }
416
417         public Builder calculateCRC32FromCompressedData(boolean calculateCRC32FromCompressedData) {
418             this.calculateCRC32FromCompressedData = calculateCRC32FromCompressedData;
419             return this;
420         }
421
422         public AmazonHttpClient build() {
423             return new AmazonHttpClient(clientConfig,
424                                         retryPolicy,
425                                         requestMetricCollector,
426                                         useBrowserCompatibleHostNameVerifier,
427                                         calculateCRC32FromCompressedData);
428         }
429     }
430
431     private static boolean isTemporaryRedirect(org.apache.http.HttpResponse response) {
432         int status = response.getStatusLine().getStatusCode();
433         return status == HttpStatus.SC_TEMPORARY_REDIRECT && response.getHeaders("Location") != null
434                && response.getHeaders("Location").length > 0;
435     }
436
437     @Override
438     protected void finalize() throws Throwable {
439         this.shutdown();
440         super.finalize();
441     }
442
443     /**
444      * Shuts down this HTTP client object, releasing any resources that might be held open. This is
445      * an optional method, and callers are not expected to call it, but can if they want to
446      * explicitly release any open resources. Once a client has been shutdown, it cannot be used to
447      * make more requests.
448      */
449     public void shutdown() {
450         clientExecutionTimer.shutdown();
451         httpRequestTimer.shutdown();
452         IdleConnectionReaper.removeConnectionManager(httpClient.getHttpClientConnectionManager());
453         httpClient.getHttpClientConnectionManager().shutdown();
454     }
455
456     /**
457      * Used to configure the test conditions for injecting intermittent failures to the content
458      * input stream.
459      *
460      * @param config unreliable test configuration for failure injection; or null to disable such
461      *               test.
462      */
463     static void configUnreliableTestConditions(UnreliableTestConfig config) {
464         unreliableTestConfig = config;
465     }
466
467     /**
468      * Package protected for unit-testing
469      */
470     @SdkTestInternalApi
471     public HttpRequestTimer getHttpRequestTimer() {
472         return this.httpRequestTimer;
473     }
474
475     /**
476      * Package protected for unit-testing
477      */
478     @SdkTestInternalApi
479     public ClientExecutionTimer getClientExecutionTimer() {
480         return this.clientExecutionTimer;
481     }
482
483     /**
484      * Returns additional response metadata for an executed request. Response metadata isn't
485      * considered part of the standard results returned by an operation, so it's accessed instead
486      * through this diagnostic interface. Response metadata is typically used for troubleshooting
487      * issues with AWS support staff when services aren't acting as expected.
488      *
489      * @param request A previously executed AmazonWebServiceRequest object, whose response metadata
490      *                is desired.
491      * @return The response metadata for the specified request, otherwise null if there is no
492      * response metadata available for the request.
493      */
494     public ResponseMetadata getResponseMetadataForRequest(AmazonWebServiceRequest request) {
495         return responseMetadataCache.get(request);
496     }
497
498     /**
499      * Returns the httpClientSettings client specific request metric collector; or null if there is
500      * none.
501      */
502     public RequestMetricCollector getRequestMetricCollector() {
503         return requestMetricCollector;
504     }
505
506     /**
507      * Returns the time difference in seconds between this client and AWS.
508      */
509     public int getTimeOffset() {
510         return timeOffset;
511     }
512
513     /**
514      * Executes the request and returns the result.
515      *
516      * @param request              The AmazonWebServices request to send to the remote server
517      * @param responseHandler      A response handler to accept a successful response from the
518      *                             remote server
519      * @param errorResponseHandler A response handler to accept an unsuccessful response from the
520      *                             remote server
521      * @param executionContext     Additional information about the context of this web service
522      *                             call
523      * @deprecated Use {@link #requestExecutionBuilder()} to configure and execute a HTTP request.
524      */
525     @Deprecated
526     public <T> Response<T> execute(Request<?> request,
527                                    HttpResponseHandler<AmazonWebServiceResponse<T>> responseHandler,
528                                    HttpResponseHandler<AmazonServiceException> errorResponseHandler,
529                                    ExecutionContext executionContext) {
530         return execute(request, responseHandler, errorResponseHandler, executionContext,
531                        new AmazonWebServiceRequestAdapter(request.getOriginalRequest()));
532     }
533
534     @SdkInternalApi
535     public <T> Response<T> execute(Request<?> request,
536                                    HttpResponseHandler<AmazonWebServiceResponse<T>> responseHandler,
537                                    HttpResponseHandler<AmazonServiceException> errorResponseHandler,
538                                    ExecutionContext executionContext,
539                                    RequestConfig requestConfig) {
540         HttpResponseHandler<T> adaptedRespHandler = new AwsResponseHandlerAdapter<T>(
541             getNonNullResponseHandler(responseHandler),
542             request,
543             executionContext.getAwsRequestMetrics(),
544             responseMetadataCache);
545         return requestExecutionBuilder()
546             .request(request)
547             .requestConfig(requestConfig)
548             .errorResponseHandler(new AwsErrorResponseHandler(errorResponseHandler, executionContext.getAwsRequestMetrics(), config))
549             .executionContext(executionContext)
550             .execute(adaptedRespHandler);
551     }
552
553     /**
554      * Ensures the response handler is not null. If it is this method returns a dummy response
555      * handler.
556      *
557      * @return Either original response handler or dummy response handler.
558      */
559     private <T> HttpResponseHandler<T> getNonNullResponseHandler(
560             HttpResponseHandler<T> responseHandler) {
561         if (responseHandler != null) {
562             return responseHandler;
563         } else {
564             // Return a Dummy, No-Op handler
565             return new HttpResponseHandler<T>() {
566
567                 @Override
568                 public T handle(HttpResponse response) throws Exception {
569                     return null;
570                 }
571
572                 @Override
573                 public boolean needsConnectionLeftOpen() {
574                     return false;
575                 }
576             };
577         }
578     }
579
580     /**
581      * @return A builder used to configure and execute a HTTP request.
582      */
583     public RequestExecutionBuilder requestExecutionBuilder() {
584         return new RequestExecutionBuilderImpl();
585     }
586
587     /**
588      * Interface to configure a request execution and execute the request.
589      */
590     public interface RequestExecutionBuilder {
591
592         /**
593          * Fluent setter for {@link Request}
594          *
595          * @param request Request object
596          * @return This builder for method chaining.
597          */
598         RequestExecutionBuilder request(Request<?> request);
599
600         /**
601          * Fluent setter for the error response handler
602          *
603          * @param errorResponseHandler Error response handler
604          * @return This builder for method chaining.
605          */
606         RequestExecutionBuilder errorResponseHandler(
607                 HttpResponseHandler<? extends SdkBaseException> errorResponseHandler);
608
609         /**
610          * Fluent setter for the execution context
611          *
612          * @param executionContext Execution context
613          * @return This builder for method chaining.
614          */
615         RequestExecutionBuilder executionContext(ExecutionContext executionContext);
616
617         /**
618          * Fluent setter for {@link RequestConfig}
619          *
620          * @param requestConfig Request config object
621          * @return This builder for method chaining.
622          */
623         RequestExecutionBuilder requestConfig(RequestConfig requestConfig);
624
625         /**
626          * Executes the request with the given configuration.
627          *
628          * @param responseHandler Response handler that outputs the actual result type which is
629          *                        preferred going forward.
630          * @param <Output>        Result type
631          * @return Unmarshalled result type.
632          */
633         <Output> Response<Output> execute(HttpResponseHandler<Output> responseHandler);
634
635         /**
636          * Executes the request with the given configuration; not handling response.
637          *
638          * @return Void response
639          */
640         Response<Void> execute();
641
642     }
643
644     private class RequestExecutionBuilderImpl implements RequestExecutionBuilder {
645
646         private Request<?> request;
647         private RequestConfig requestConfig;
648         private HttpResponseHandler<? extends SdkBaseException> errorResponseHandler;
649         private ExecutionContext executionContext = new ExecutionContext();
650
651         @Override
652         public RequestExecutionBuilder request(Request<?> request) {
653             this.request = request;
654             return this;
655         }
656
657         @Override
658         public RequestExecutionBuilder errorResponseHandler(
659                 HttpResponseHandler<? extends SdkBaseException> errorResponseHandler) {
660             this.errorResponseHandler = errorResponseHandler;
661             return this;
662         }
663
664         @Override
665         public RequestExecutionBuilder executionContext(
666                 ExecutionContext executionContext) {
667             this.executionContext = executionContext;
668             return this;
669         }
670
671         @Override
672         public RequestExecutionBuilder requestConfig(RequestConfig requestConfig) {
673             this.requestConfig = requestConfig;
674             return this;
675         }
676
677         @Override
678         public <Output> Response<Output> execute(HttpResponseHandler<Output> responseHandler) {
679             RequestConfig config = requestConfig != null ? requestConfig : new AmazonWebServiceRequestAdapter(request.getOriginalRequest());
680             return new RequestExecutor<Output>(request,
681                                                config,
682                                                getNonNullResponseHandler(errorResponseHandler),
683                                                getNonNullResponseHandler(responseHandler),
684                                                executionContext,
685                                                getRequestHandlers()
686             ).execute();
687         }
688
689         @Override
690         public Response<Void> execute() {
691             return execute(null);
692         }
693
694         private List<RequestHandler2> getRequestHandlers() {
695             List<RequestHandler2> requestHandler2s = executionContext.getRequestHandler2s();
696             if (requestHandler2s == null) {
697                 return Collections.emptyList();
698             }
699             return requestHandler2s;
700         }
701
702     }
703
704     private class RequestExecutor<Output> {
705         private final Request<?> request;
706         private final RequestConfig requestConfig;
707         private final HttpResponseHandler<? extends SdkBaseException> errorResponseHandler;
708         private final HttpResponseHandler<Output> responseHandler;
709         private final ExecutionContext executionContext;
710         private final List<RequestHandler2> requestHandler2s;
711         private final AWSRequestMetrics awsRequestMetrics;
712         //TODO: Call CSMRequestHandler directly in this class since it's CSM aware now
713         private RequestHandler2 csmRequestHandler;
714
715         private RequestExecutor(Request<?> request, RequestConfig requestConfig,
716                                 HttpResponseHandler<? extends SdkBaseException> errorResponseHandler,
717                                 HttpResponseHandler<Output> responseHandler,
718                                 ExecutionContext executionContext,
719                                 List<RequestHandler2> requestHandler2s) {
720             this.request = request;
721             this.requestConfig = requestConfig;
722             this.errorResponseHandler = errorResponseHandler;
723             this.responseHandler = responseHandler;
724             this.executionContext = executionContext;
725             this.requestHandler2s = requestHandler2s;
726             this.awsRequestMetrics = executionContext.getAwsRequestMetrics();
727             for (RequestHandler2 requestHandler2 : requestHandler2s) {
728                 if (requestHandler2 instanceof ClientSideMonitoringRequestHandler) {
729                     csmRequestHandler = requestHandler2;
730                     break;
731                 }
732             }
733         }
734
735         /**
736          * Executes the request and returns the result.
737          */
738         private Response<Output> execute() {
739             if (executionContext == null) {
740                 throw new SdkClientException(
741                         "Internal SDK Error: No execution context parameter specified.");
742             }
743             try {
744                 return executeWithTimer();
745             } catch (InterruptedException ie) {
746                 throw handleInterruptedException(ie);
747             } catch (AbortedException ae) {
748                 throw handleAbortedException(ae);
749             } finally {
750                 if (executionContext.getClientExecutionTrackerTask().hasTimeoutExpired()) {
751                     // There might be a race condition that the timeout tracker executed before the call to cancel(),
752                     // which means it set this thread's interrupt flag, so just clear the interrupt flag
753                     Thread.interrupted();
754                 }
755             }
756         }
757
758         /**
759          * Start and end client execution timer around the execution of the request. It's important
760          * that the client execution task is canceled before the InterruptedExecption is handled by
761          * {@link #execute()} so * the interrupt status doesn't leak out to the callers code
762          */
763         private Response<Output> executeWithTimer() throws InterruptedException {
764             ClientExecutionAbortTrackerTask clientExecutionTrackerTask =
765                     clientExecutionTimer.startTimer(getClientExecutionTimeout(requestConfig));
766             Response<Output> outputResponse;
767
768             try {
769                 executionContext.setClientExecutionTrackerTask(clientExecutionTrackerTask);
770                 outputResponse = doExecute();
771
772             } finally {
773                 // Cancel the timeout tracker, guaranteeing that if it hasn't already executed and set this thread's
774                 // interrupt flag, it won't do so later. Every code path executed after this line *must* call
775                 // timeoutTracker.hasTimeoutExpired() and appropriately clear the interrupt flag if it returns true.
776                 executionContext.getClientExecutionTrackerTask().cancelTask();
777             }
778
779             return outputResponse;
780         }
781
782         private Response<Output> doExecute() throws InterruptedException {
783             runBeforeRequestHandlers();
784             setSdkTransactionId(request);
785             setUserAgent(request);
786
787             ProgressListener listener = requestConfig.getProgressListener();
788             // add custom headers
789             request.getHeaders().putAll(config.getHeaders());
790             request.getHeaders().putAll(requestConfig.getCustomRequestHeaders());
791             // add custom query parameters
792             mergeQueryParameters(requestConfig.getCustomQueryParameters());
793             Response<Output> response = null;
794             final InputStream origContent = request.getContent();
795             final InputStream toBeClosed = beforeRequest(); // for progress tracking
796             // make "notCloseable", so reset would work with retries
797             final InputStream notCloseable = (toBeClosed == null) ? null
798                     : ReleasableInputStream.wrap(toBeClosed).disableClose();
799             request.setContent(notCloseable);
800             try {
801                 publishProgress(listener, ProgressEventType.CLIENT_REQUEST_STARTED_EVENT);
802                 response = executeHelper();
803                 publishProgress(listener, ProgressEventType.CLIENT_REQUEST_SUCCESS_EVENT);
804                 awsRequestMetrics.endEvent(AwsClientSideMonitoringMetrics.ApiCallLatency);
805                 awsRequestMetrics.getTimingInfo().endTiming();
806                 afterResponse(response);
807                 return response;
808             } catch (AmazonClientException e) {
809                 publishProgress(listener, ProgressEventType.CLIENT_REQUEST_FAILED_EVENT);
810
811                 awsRequestMetrics.endEvent(AwsClientSideMonitoringMetrics.ApiCallLatency);
812                 // Exceptions generated here will block the rethrow of e.
813                 afterError(response, e);
814                 throw e;
815             } finally {
816                 // Always close so any progress tracking would get the final events propagated.
817                 closeQuietlyForRuntimeExceptions(toBeClosed, log);
818                 request.setContent(origContent); // restore the original content
819             }
820         }
821
822         private void closeQuietlyForRuntimeExceptions(Closeable c, Log log) {
823             try {
824                 closeQuietly(c, log);
825             } catch (RuntimeException e) {
826                 if (log.isDebugEnabled()) {
827                     log.debug("Unable to close closeable", e);
828                 }
829             }
830         }
831
832         private void runBeforeRequestHandlers() {
833             AWSCredentials credentials = getCredentialsFromContext();
834             request.addHandlerContext(HandlerContextKey.AWS_CREDENTIALS, credentials);
835             // Apply any additional service specific request handlers that need to be run
836             for (RequestHandler2 requestHandler2 : requestHandler2s) {
837                 // If the request handler is a type of CredentialsRequestHandler, then set the credentials in the request handler.
838                 if (requestHandler2 instanceof CredentialsRequestHandler) {
839                     ((CredentialsRequestHandler) requestHandler2).setCredentials(credentials);
840                 }
841                 requestHandler2.beforeRequest(request);
842             }
843         }
844
845         /**
846          * Determine if an interrupted exception is caused by the client execution timer
847          * interrupting the current thread or some other task interrupting the thread for another
848          * purpose.
849          *
850          * @return {@link ClientExecutionTimeoutException} if the {@link InterruptedException} was
851          * caused by the {@link ClientExecutionTimer}. Otherwise re-interrupts the current thread
852          * and returns a {@link SdkClientException} wrapping an {@link InterruptedException}
853          */
854         private RuntimeException handleInterruptedException(InterruptedException e) {
855             if (e instanceof SdkInterruptedException) {
856                 if (((SdkInterruptedException) e).getResponse() != null) {
857                     ((SdkInterruptedException) e).getResponse().getHttpResponse().getHttpRequest().abort();
858                 }
859             }
860             if (executionContext.getClientExecutionTrackerTask().hasTimeoutExpired()) {
861                 // Clear the interrupt status
862                 Thread.interrupted();
863                 ClientExecutionTimeoutException exception = new ClientExecutionTimeoutException();
864                 reportClientExecutionTimeout(exception);
865                 return exception;
866             } else {
867                 Thread.currentThread().interrupt();
868                 return new AbortedException(e);
869             }
870         }
871
872         /**
873          * Determine if an aborted exception is caused by the client execution timer interrupting
874          * the current thread. If so throws {@link ClientExecutionTimeoutException} else throws the
875          * original {@link AbortedException}
876          *
877          * @param ae aborted exception that occurred
878          * @return {@link ClientExecutionTimeoutException} if the {@link AbortedException} was
879          * caused by the {@link ClientExecutionTimer}. Otherwise throws the original {@link
880          * AbortedException}
881          */
882         private RuntimeException handleAbortedException(final AbortedException ae) {
883             if (executionContext.getClientExecutionTrackerTask().hasTimeoutExpired()) {
884                 // Clear the interrupt status
885                 Thread.interrupted();
886                 ClientExecutionTimeoutException exception = new ClientExecutionTimeoutException();
887                 reportClientExecutionTimeout(exception);
888                 return exception;
889             } else {
890                 Thread.currentThread().interrupt();
891                 return ae;
892             }
893         }
894
895         private void reportClientExecutionTimeout(ClientExecutionTimeoutException exception) {
896             if (csmRequestHandler != null) {
897                 csmRequestHandler.afterError(request, null, exception);
898             }
899         }
900
901         /**
902          * Check if the thread has been interrupted. If so throw an {@link InterruptedException}.
903          * Long running tasks should be periodically checked if the current thread has been
904          * interrupted and handle it appropriately
905          *
906          * @throws InterruptedException If thread has been interrupted
907          */
908         private void checkInterrupted() throws InterruptedException {
909             checkInterrupted(null);
910         }
911
912         /**
913          * Check if the thread has been interrupted. If so throw an {@link InterruptedException}.
914          * Long running tasks should be periodically checked if the current thread has been
915          * interrupted and handle it appropriately
916          *
917          * @param response Response to be closed before returning control to the caller to avoid
918          *                 leaking the connection.
919          * @throws InterruptedException If thread has been interrupted
920          */
921         private void checkInterrupted(Response<?> response) throws InterruptedException {
922             if (Thread.interrupted()) {
923                 throw new SdkInterruptedException(response);
924             }
925         }
926
927         /**
928          * Merge query parameters into the given request.
929          */
930         private void mergeQueryParameters(Map<String, List<String>> params) {
931             Map<String, List<String>> existingParams = request.getParameters();
932             for (Entry<String, List<String>> param : params.entrySet()) {
933                 String pName = param.getKey();
934                 List<String> pValues = param.getValue();
935                 existingParams.put(pName, CollectionUtils.mergeLists(existingParams.get(pName), pValues));
936             }
937         }
938
939         /**
940          * Publishes the "request content length" event, and returns an input stream, which will be
941          * made mark-and-resettable if possible, for progress tracking purposes.
942          *
943          * @return an input stream, which will be made mark-and-resettable if possible, for progress
944          * tracking purposes; or null if the request doesn't have an input stream
945          */
946         private InputStream beforeRequest() {
947             ProgressListener listener = requestConfig.getProgressListener();
948             reportContentLength(listener);
949             if (request.getContent() == null) {
950                 return null;
951             }
952             final InputStream content = monitorStreamProgress(listener,
953                                                               buffer(
954                                                                       makeResettable(
955                                                                               request.getContent())));
956             if (AmazonHttpClient.unreliableTestConfig == null) {
957                 return content;
958             }
959             return wrapWithUnreliableStream(content);
960         }
961
962         /**
963          * If content length is present on the request, report it to the progress listener.
964          *
965          * @param listener Listener to notify.
966          */
967         private void reportContentLength(ProgressListener listener) {
968             Map<String, String> headers = request.getHeaders();
969             String contentLengthStr = headers.get("Content-Length");
970             if (contentLengthStr != null) {
971                 try {
972                     long contentLength = Long.parseLong(contentLengthStr);
973                     publishRequestContentLength(listener, contentLength);
974                 } catch (NumberFormatException e) {
975                     log.warn("Cannot parse the Content-Length header of the request.");
976                 }
977             }
978         }
979
980         /**
981          * Make input stream resettable if possible.
982          *
983          * @param content Input stream to make resettable
984          * @return ResettableInputStream if possible otherwise original input stream.
985          */
986         private InputStream makeResettable(InputStream content) {
987             if (!content.markSupported()) {
988                 // try to wrap the content input stream to become
989                 // mark-and-resettable for signing and retry purposes.
990                 if (content instanceof FileInputStream) {
991                     try {
992                         // ResettableInputStream supports mark-and-reset without
993                         // memory buffering
994                         return new ResettableInputStream((FileInputStream) content);
995                     } catch (IOException e) {
996                         if (log.isDebugEnabled()) {
997                             log.debug("For the record; ignore otherwise", e);
998                         }
999                     }
1000                 }
1001             }
1002             return content;
1003         }
1004
1005         /**
1006          * Buffer input stream if possible.
1007          *
1008          * @param content Input stream to buffer
1009          * @return SdkBufferedInputStream if possible, otherwise original input stream.
1010          */
1011         private InputStream buffer(InputStream content) {
1012             if (!content.markSupported()) {
1013                 content = new SdkBufferedInputStream(content);
1014             }
1015             return content;
1016         }
1017
1018         /**
1019          * Wrap with a {@link ProgressInputStream} to report request progress to listener.
1020          *
1021          * @param listener Listener to report to
1022          * @param content  Input stream to monitor progress for
1023          * @return Wrapped input stream with progress monitoring capabilities.
1024          */
1025         private InputStream monitorStreamProgress(ProgressListener listener,
1026                                                   InputStream content) {
1027             return ProgressInputStream.inputStreamForRequest(content, listener);
1028         }
1029
1030         /**
1031          * Used only for internal testing purposes. Makes a stream unreliable in certain ways for
1032          * fault testing.
1033          *
1034          * @param content Input stream to make unreliable.
1035          * @return UnreliableFilterInputStream
1036          */
1037         private InputStream wrapWithUnreliableStream(InputStream content) {
1038             return new UnreliableFilterInputStream(content,
1039                                                    unreliableTestConfig.isFakeIOException())
1040                     .withBytesReadBeforeException(
1041                             unreliableTestConfig.getBytesReadBeforeException())
1042                     .withMaxNumErrors(unreliableTestConfig.getMaxNumErrors())
1043                     .withResetIntervalBeforeException(
1044                             unreliableTestConfig.getResetIntervalBeforeException());
1045         }
1046
1047
1048         private void afterError(Response<?> response,
1049                                 AmazonClientException e) throws InterruptedException {
1050             for (RequestHandler2 handler2 : requestHandler2s) {
1051                 handler2.afterError(request, response, e);
1052                 checkInterrupted(response);
1053             }
1054         }
1055
1056         private <T> void afterResponse(Response<T> response) throws InterruptedException {
1057             for (RequestHandler2 handler2 : requestHandler2s) {
1058                 handler2.afterResponse(request, response);
1059                 checkInterrupted(response);
1060             }
1061         }
1062
1063         private <T> void beforeAttempt(HandlerBeforeAttemptContext context) throws InterruptedException {
1064             for (RequestHandler2 handler2 : requestHandler2s) {
1065                 handler2.beforeAttempt(context);
1066                 checkInterrupted();
1067             }
1068         }
1069
1070         private <T> void afterAttempt(HandlerAfterAttemptContext context) throws InterruptedException {
1071             for (RequestHandler2 handler2 : requestHandler2s) {
1072                 handler2.afterAttempt(context);
1073                 checkInterrupted(context.getResponse());
1074             }
1075         }
1076
1077         /**
1078          * Internal method to execute the HTTP method given.
1079          */
1080         private Response<Output> executeHelper() throws InterruptedException {
1081         /*
1082          * add the service endpoint to the logs. You can infer service name from service endpoint
1083          */
1084             awsRequestMetrics
1085                     .addPropertyWith(Field.RequestType, requestConfig.getRequestType())
1086                     .addPropertyWith(Field.ServiceName, request.getServiceName())
1087                     .addPropertyWith(Field.ServiceEndpoint, request.getEndpoint());
1088             // Make a copy of the original request params and headers so that we can
1089             // permute it in this loop and start over with the original every time.
1090             final Map<String, List<String>> originalParameters = new LinkedHashMap<String, List<String>>(request.getParameters());
1091             final Map<String, String> originalHeaders = new HashMap<String, String>(request.getHeaders());
1092             // Always mark the input stream before execution.
1093             final ExecOneRequestParams execOneParams = new ExecOneRequestParams();
1094             final InputStream originalContent = request.getContent();
1095             if (originalContent != null && originalContent.markSupported()
1096                 && !(originalContent instanceof BufferedInputStream)) {
1097                 // Mark only once for non-BufferedInputStream
1098                 final int readLimit = requestConfig.getRequestClientOptions().getReadLimit();
1099                 originalContent.mark(readLimit);
1100             }
1101             awsRequestMetrics.startEvent(AwsClientSideMonitoringMetrics.ApiCallLatency);
1102             while (true) {
1103                 checkInterrupted();
1104                 if (originalContent instanceof BufferedInputStream && originalContent.markSupported()) {
1105                     // Mark everytime for BufferedInputStream, since the marker could have been invalidated
1106                     final int readLimit = requestConfig.getRequestClientOptions().getReadLimit();
1107                     originalContent.mark(readLimit);
1108                 }
1109                 execOneParams.initPerRetry();
1110                 URI redirectedURI = execOneParams.redirectedURI;
1111                 if (redirectedURI != null) {
1112                 /*
1113                  * [scheme:][//authority][path][?query][#fragment]
1114                  */
1115                     String scheme = redirectedURI.getScheme();
1116                     String beforeAuthority = scheme == null ? "" : scheme + "://";
1117                     String authority = redirectedURI.getAuthority();
1118                     String path = redirectedURI.getPath();
1119
1120                     request.setEndpoint(URI.create(beforeAuthority + authority));
1121                     request.setResourcePath(SdkHttpUtils.urlEncode(path, true));
1122                     awsRequestMetrics.addPropertyWith(Field.RedirectLocation,
1123                                                       redirectedURI.toString());
1124
1125                 }
1126                 if (execOneParams.authRetryParam != null) {
1127                     request.setEndpoint(execOneParams.authRetryParam.getEndpointForRetry());
1128                 }
1129                 awsRequestMetrics.setCounter(Field.RequestCount, execOneParams.requestCount);
1130                 if (execOneParams.isRetry()) {
1131                     request.setParameters(originalParameters);
1132                     request.setHeaders(originalHeaders);
1133                     request.setContent(originalContent);
1134                 }
1135
1136                 Response<Output> response = null;
1137                 Exception savedException = null;
1138                 boolean thrown = false;
1139                 try {
1140                     HandlerBeforeAttemptContext beforeAttemptContext = HandlerBeforeAttemptContext.builder()
1141                             .withRequest(request)
1142                             .build();
1143
1144                     beforeAttempt(beforeAttemptContext);
1145                     response = executeOneRequest(execOneParams);
1146                     savedException = execOneParams.retriedException;
1147
1148                     if (response != null) {
1149                         return response;
1150                     }
1151                 } catch (IOException ioe) {
1152                     savedException = ioe;
1153                     handleRetryableException(execOneParams, ioe);
1154                 } catch (InterruptedException ie) {
1155                     savedException = ie;
1156                     thrown = true;
1157                     throw ie;
1158                 } catch (RuntimeException e) {
1159                     savedException = e;
1160                     thrown = true;
1161                     throw lastReset(captureExceptionMetrics(e));
1162                 } catch (Error e) {
1163                     thrown = true;
1164                     throw lastReset(captureExceptionMetrics(e));
1165                 } finally {
1166                 /*
1167                  * Some response handlers need to manually manage the HTTP connection and will take
1168                  * care of releasing the connection on their own, but if this response handler
1169                  * doesn't need the connection left open, we go ahead and release the it to free up
1170                  * resources. But if we throw, then the caller doesn't get the handle on the response
1171                  * to close for themselves. In this case, we will close the connection for them as well.
1172                  */
1173                     if (!execOneParams.leaveHttpConnectionOpen || thrown) {
1174                         if (execOneParams.apacheResponse != null) {
1175                             HttpEntity entity = execOneParams.apacheResponse.getEntity();
1176                             if (entity != null) {
1177                                 try {
1178                                     closeQuietly(entity.getContent(), log);
1179                                 } catch (IOException e) {
1180                                     log.warn("Cannot close the response content.", e);
1181                                 }
1182                             }
1183                         }
1184                     }
1185
1186                     HandlerAfterAttemptContext afterAttemptContext = HandlerAfterAttemptContext.builder()
1187                             .withRequest(request)
1188                             .withResponse(response)
1189                             .withException(savedException)
1190                             .build();
1191
1192                     /*
1193                      * Exceptions generated here will replace ones rethrown in catch-blocks
1194                      * above or thrown in the original try-block.
1195                      */
1196                     afterAttempt(afterAttemptContext);
1197                 }
1198             } /* end while (true) */
1199         }
1200
1201         private void handleRetryableException(ExecOneRequestParams execOneParams, Exception e) {
1202             captureExceptionMetrics(e);
1203             awsRequestMetrics.addProperty(Field.AWSRequestID, null);
1204             SdkClientException sdkClientException;
1205             if (!(e instanceof SdkClientException)) {
1206                 sdkClientException = new SdkClientException(
1207                         "Unable to execute HTTP request: " + e.getMessage(), e);
1208             } else {
1209                 sdkClientException = (SdkClientException) e;
1210             }
1211             boolean willRetry = shouldRetry(execOneParams, sdkClientException);
1212             if (log.isTraceEnabled()) {
1213                 log.trace(sdkClientException.getMessage() + (willRetry ? " Request will be retried." : ""), e);
1214             } else if (log.isDebugEnabled()) {
1215                 log.debug(sdkClientException.getMessage() + (willRetry ? " Request will be retried." : ""));
1216             }
1217             if (!willRetry) {
1218                 throw lastReset(sdkClientException);
1219             }
1220             // Cache the retryable exception
1221             execOneParams.retriedException = sdkClientException;
1222         }
1223
1224         /**
1225          * Used to perform a last reset on the content input stream (if mark-supported); this is so
1226          * that, for backward compatibility reason, any "blind" retry (ie without calling reset) by
1227          * user of this library with the same input stream (such as ByteArrayInputStream) could
1228          * still succeed.
1229          *
1230          * @param t the failure
1231          * @return the failure as given
1232          */
1233         private <T extends Throwable> T lastReset(final T t) {
1234             try {
1235                 InputStream content = request.getContent();
1236                 if (content != null) {
1237                     if (content.markSupported()) {
1238                         content.reset();
1239                     }
1240                 }
1241             } catch (Exception ex) {
1242                 log.debug("FYI: failed to reset content inputstream before throwing up", ex);
1243             }
1244             return t;
1245         }
1246
1247         /**
1248          * Returns the credentials from the execution if exists. Else returns null.
1249          */
1250         private AWSCredentials getCredentialsFromContext() {
1251             final AWSCredentialsProvider credentialsProvider = executionContext.getCredentialsProvider();
1252
1253             AWSCredentials credentials = null;
1254             if (credentialsProvider != null) {
1255                 awsRequestMetrics.startEvent(Field.CredentialsRequestTime);
1256                 try {
1257                     credentials = credentialsProvider.getCredentials();
1258                 } finally {
1259                     awsRequestMetrics.endEvent(Field.CredentialsRequestTime);
1260                 }
1261             }
1262             return credentials;
1263         }
1264
1265         /**
1266          * Returns the response from executing one httpClientSettings request; or null for retry.
1267          */
1268         private Response<Output> executeOneRequest(ExecOneRequestParams execOneParams)
1269                 throws IOException, InterruptedException {
1270
1271             if (execOneParams.isRetry()) {
1272                 resetRequestInputStream(request, execOneParams.retriedException);
1273             }
1274             checkInterrupted();
1275             if (requestLog.isDebugEnabled()) {
1276                 requestLog.debug((execOneParams.isRetry() ? "Retrying " : "Sending ") + "Request: " + request);
1277             }
1278             final AWSCredentials credentials = getCredentialsFromContext();
1279             final ProgressListener listener = requestConfig.getProgressListener();
1280
1281             if (execOneParams.isRetry()) {
1282                 pauseBeforeRetry(execOneParams, listener);
1283             }
1284             updateRetryHeaderInfo(request, execOneParams);
1285             sdkRequestHeaderProvider.addSdkRequestRetryHeader(request, execOneParams.requestCount);
1286
1287             // Sign the request if a signer was provided
1288             execOneParams.newSigner(request, executionContext);
1289             if (execOneParams.signer != null &&
1290                 (credentials != null || execOneParams.signer instanceof CanHandleNullCredentials)) {
1291                 awsRequestMetrics.startEvent(Field.RequestSigningTime);
1292                 try {
1293                     if (timeOffset != 0) {
1294                         // Always use the client level timeOffset if it was
1295                         // non-zero; Otherwise, we respect the timeOffset in the
1296                         // request, which could have been externally configured (at
1297                         // least for the 1st non-retry request).
1298                         //
1299                         // For retry due to clock skew, the timeOffset in the
1300                         // request used for the retry is assumed to have been
1301                         // adjusted when execution reaches here.
1302                         request.setTimeOffset(timeOffset);
1303                     }
1304                     execOneParams.signer.sign(request, credentials);
1305                 } finally {
1306                     awsRequestMetrics.endEvent(Field.RequestSigningTime);
1307                 }
1308             }
1309
1310             checkInterrupted();
1311             execOneParams.newApacheRequest(httpRequestFactory, request, httpClientSettings);
1312
1313             captureConnectionPoolMetrics();
1314
1315             final HttpClientContext localRequestContext =
1316                     ApacheUtils.newClientContext(httpClientSettings, ImmutableMapParameter.of
1317                             (AWSRequestMetrics.SIMPLE_NAME, awsRequestMetrics));
1318
1319             execOneParams.resetBeforeHttpRequest();
1320             publishProgress(listener, ProgressEventType.HTTP_REQUEST_STARTED_EVENT);
1321             awsRequestMetrics.startEvent(Field.HttpRequestTime);
1322             awsRequestMetrics.setCounter(Field.RetryCapacityConsumed, retryCapacity.consumedCapacity());
1323
1324             /////////// Send HTTP request ////////////
1325             executionContext.getClientExecutionTrackerTask().setCurrentHttpRequest(execOneParams.apacheRequest);
1326             final HttpRequestAbortTaskTracker requestAbortTaskTracker = httpRequestTimer
1327                     .startTimer(execOneParams.apacheRequest, getRequestTimeout(requestConfig));
1328
1329             try {
1330                 execOneParams.apacheResponse = httpClient.execute(execOneParams.apacheRequest, localRequestContext);
1331                 if (shouldBufferHttpEntity(responseHandler.needsConnectionLeftOpen(),
1332                                            executionContext,
1333                                            execOneParams,
1334                                            requestAbortTaskTracker)) {
1335                     execOneParams.apacheResponse
1336                             .setEntity(new BufferedHttpEntity(
1337                                     execOneParams.apacheResponse.getEntity()));
1338                 }
1339             } catch (IOException ioe) {
1340                 // Client execution timeouts take precedence as it's not retryable
1341                 if (executionContext.getClientExecutionTrackerTask().hasTimeoutExpired()) {
1342                     throw new InterruptedException();
1343                 } else if (requestAbortTaskTracker.httpRequestAborted()) {
1344                     // Interrupt flag can leak from apache when aborting the request
1345                     // https://issues.apache.org/jira/browse/HTTPCLIENT-1958, TT0174038332
1346                     if (ioe instanceof RequestAbortedException) {
1347                         Thread.interrupted();
1348                     }
1349                      throw new HttpRequestTimeoutException(ioe);
1350                 } else {
1351                     throw ioe;
1352                 }
1353             } finally {
1354                 requestAbortTaskTracker.cancelTask();
1355                 awsRequestMetrics.endEvent(Field.HttpRequestTime);
1356             }
1357
1358             publishProgress(listener, ProgressEventType.HTTP_REQUEST_COMPLETED_EVENT);
1359             final StatusLine statusLine = execOneParams.apacheResponse.getStatusLine();
1360             final int statusCode = statusLine == null ? -1 : statusLine.getStatusCode();
1361
1362             // Always update estimated skew if the wire call is successful.
1363             clockSkewAdjuster.updateEstimatedSkew(new AdjustmentRequest()
1364                                                       .clientRequest(request)
1365                                                       .serviceResponse(execOneParams.apacheResponse));
1366
1367             if (ApacheUtils.isRequestSuccessful(execOneParams.apacheResponse)) {
1368                 return handleSuccessResponse(execOneParams, localRequestContext, statusCode);
1369             }
1370
1371             return handleServiceErrorResponse(execOneParams, localRequestContext, statusCode);
1372         }
1373
1374         /**
1375          * Handle service error response and check if the response is retryable or not.
1376          */
1377         private Response<Output> handleServiceErrorResponse(ExecOneRequestParams execOneParams, HttpClientContext localRequestContext, int statusCode) throws IOException, InterruptedException {
1378             if (isTemporaryRedirect(execOneParams.apacheResponse)) {
1379             /*
1380              * S3 sends 307 Temporary Redirects if you try to delete an EU bucket from the US
1381              * endpoint. If we get a 307, we'll point the HTTP method to the redirected location,
1382              * and let the next retry deliver the request to the right location.
1383              */
1384                 Header[] locationHeaders = execOneParams.apacheResponse.getHeaders("location");
1385                 String redirectedLocation = locationHeaders[0].getValue();
1386                 if (log.isDebugEnabled()) {
1387                     log.debug("Redirecting to: " + redirectedLocation);
1388                 }
1389                 execOneParams.redirectedURI = URI.create(redirectedLocation);
1390                 awsRequestMetrics.addPropertyWith(Field.StatusCode, statusCode)
1391                                  .addPropertyWith(Field.AWSRequestID, null);
1392                 return null; // => retry
1393             }
1394             execOneParams.leaveHttpConnectionOpen = errorResponseHandler.needsConnectionLeftOpen();
1395             final SdkBaseException exception = handleErrorResponse(execOneParams.apacheRequest,
1396                                                                    execOneParams.apacheResponse,
1397                                                                    localRequestContext);
1398
1399             ClockSkewAdjustment clockSkewAdjustment =
1400                     clockSkewAdjuster.getAdjustment(new AdjustmentRequest().exception(exception)
1401                                                                            .clientRequest(request)
1402                                                                            .serviceResponse(execOneParams.apacheResponse));
1403
1404             if (clockSkewAdjustment.shouldAdjustForSkew()) {
1405                 timeOffset = clockSkewAdjustment.inSeconds();
1406                 request.setTimeOffset(timeOffset); // adjust time offset for the retry
1407                 SDKGlobalTime.setGlobalTimeOffset(timeOffset);
1408             }
1409
1410
1411             // Check whether we should internally retry the auth error
1412             execOneParams.authRetryParam = null;
1413             AuthErrorRetryStrategy authRetry = executionContext.getAuthErrorRetryStrategy();
1414             if (authRetry != null && exception instanceof AmazonServiceException) {
1415                 HttpResponse httpResponse = ApacheUtils.createResponse(request, execOneParams.apacheRequest, execOneParams.apacheResponse, localRequestContext);
1416                 execOneParams.authRetryParam = authRetry
1417                         .shouldRetryWithAuthParam(request, httpResponse, (AmazonServiceException) exception);
1418             }
1419             if (execOneParams.authRetryParam == null && !shouldRetry(execOneParams, exception)) {
1420                 throw exception;
1421             }
1422             // Comment out for now. Ref: CR2662349
1423             // Preserve the cause of retry before retrying
1424             // awsRequestMetrics.addProperty(RetryCause, ase);
1425             if (RetryUtils.isThrottlingException(exception)) {
1426                 awsRequestMetrics.incrementCounterWith(Field.ThrottleException)
1427                         .addProperty(Field.ThrottleException, exception);
1428             }
1429             // Cache the retryable exception
1430             execOneParams.retriedException = exception;
1431
1432             return null; // => retry
1433         }
1434
1435         /**
1436          * Handle success response.
1437          */
1438         private Response<Output> handleSuccessResponse(ExecOneRequestParams execOneParams, HttpClientContext localRequestContext, int statusCode) throws IOException, InterruptedException {
1439             awsRequestMetrics.addProperty(Field.StatusCode, statusCode);
1440             /*
1441              * If we get back any 2xx status code, then we know we should treat the service call as
1442              * successful.
1443              */
1444             execOneParams.leaveHttpConnectionOpen = responseHandler.needsConnectionLeftOpen();
1445             HttpResponse httpResponse = ApacheUtils.createResponse(request, execOneParams.apacheRequest, execOneParams.apacheResponse, localRequestContext);
1446             Output response = handleResponse(httpResponse);
1447
1448             /*
1449              * If this was a successful retry attempt we'll release the full retry capacity that
1450              * the attempt originally consumed.  If this was a successful initial request
1451              * we return a lesser amount.
1452              */
1453             if (execOneParams.isRetry() && executionContext.retryCapacityConsumed()) {
1454                 retryCapacity.release(execOneParams.lastConsumedRetryCapacity);
1455             } else {
1456                 retryCapacity.release();
1457             }
1458             return new Response<Output>(response, httpResponse);
1459         }
1460
1461         /**
1462          * Reset the input stream of the request before a retry.
1463          *
1464          * @param request Request containing input stream to reset
1465          * @param retriedException
1466          * @throws ResetException If Input Stream can't be reset which means the request can't be
1467          *                        retried
1468          */
1469         private void resetRequestInputStream(final Request<?> request, SdkBaseException retriedException)
1470                 throws ResetException {
1471             InputStream requestInputStream = request.getContent();
1472             if (requestInputStream != null) {
1473                 if (requestInputStream.markSupported()) {
1474                     try {
1475                         requestInputStream.reset();
1476                     } catch (IOException ex) {
1477                         ResetException resetException = new ResetException(
1478                                 "The request to the service failed with a retryable reason, but resetting the request input " +
1479                                 "stream has failed. See exception.getExtraInfo or debug-level logging for the original failure " +
1480                                 "that caused this retry.",
1481                                 ex);
1482                         resetException.setExtraInfo(retriedException.getMessage());
1483                         throw resetException;
1484                     }
1485                 }
1486             }
1487         }
1488
1489         /**
1490          * @return True if the {@link HttpEntity} should be wrapped in a {@link BufferedHttpEntity}
1491          */
1492         private boolean shouldBufferHttpEntity(final boolean needsConnectionLeftOpen,
1493                                                final ExecutionContext execContext,
1494                                                ExecOneRequestParams execParams,
1495                                                final HttpRequestAbortTaskTracker requestAbortTaskTracker) {
1496             return (execContext.getClientExecutionTrackerTask().isEnabled() ||
1497                     requestAbortTaskTracker.isEnabled())
1498                    && !needsConnectionLeftOpen && execParams.apacheResponse.getEntity() != null;
1499         }
1500
1501
1502         /**
1503          * Captures the connection pool metrics.
1504          */
1505         private void captureConnectionPoolMetrics() {
1506             if (awsRequestMetrics.isEnabled() &&
1507                 httpClient.getHttpClientConnectionManager() instanceof
1508                         ConnPoolControl<?>) {
1509                 final PoolStats stats = ((ConnPoolControl<?>) httpClient
1510                         .getHttpClientConnectionManager()).getTotalStats();
1511
1512                 awsRequestMetrics
1513                         .withCounter(HttpClientPoolAvailableCount, stats.getAvailable())
1514                         .withCounter(HttpClientPoolLeasedCount, stats.getLeased())
1515                         .withCounter(HttpClientPoolPendingCount, stats.getPending());
1516             }
1517
1518         }
1519
1520         /**
1521          * Capture the metrics for the given throwable.
1522          */
1523         private <T extends Throwable> T captureExceptionMetrics(T t) {
1524             awsRequestMetrics.incrementCounterWith(Field.Exception)
1525                     .addProperty(Field.Exception, t);
1526             if (t instanceof AmazonServiceException) {
1527                 AmazonServiceException ase = (AmazonServiceException) t;
1528                 if (RetryUtils.isThrottlingException(ase)) {
1529                     awsRequestMetrics.incrementCounterWith(Field.ThrottleException)
1530                             .addProperty(Field.ThrottleException, ase);
1531                 }
1532             }
1533             return t;
1534         }
1535
1536         /**
1537          * Create a client side identifier that will be sent with the initial request and each
1538          * retry.
1539          */
1540         private void setSdkTransactionId(Request<?> request) {
1541             request.addHeader(HEADER_SDK_TRANSACTION_ID,
1542                               new UUID(random.nextLong(), random.nextLong()).toString());
1543         }
1544
1545         /**
1546          * Sets a User-Agent for the specified request, taking into account any custom data.
1547          */
1548         private void setUserAgent(Request<?> request) {
1549             RequestClientOptions opts = requestConfig.getRequestClientOptions();
1550             if (opts != null) {
1551                 request.addHeader(HEADER_USER_AGENT, RuntimeHttpUtils
1552                         .getUserAgent(config, opts.getClientMarker(Marker.USER_AGENT)));
1553             } else {
1554                 request.addHeader(HEADER_USER_AGENT, RuntimeHttpUtils.getUserAgent(config, null));
1555             }
1556         }
1557
1558         /**
1559          * Adds Retry information to the {@link #HEADER_SDK_RETRY_INFO} header. Used for analysis of
1560          * retry policy.
1561          *
1562          * @param request              Request to add header to
1563          * @param execOneRequestParams Request context containing retry information
1564          */
1565         private void updateRetryHeaderInfo(Request<?> request,
1566                                            ExecOneRequestParams execOneRequestParams) {
1567             int availableRetryCapacity = retryCapacity.availableCapacity();
1568
1569             String headerValue = String.format("%s/%s/%s",
1570                                                execOneRequestParams.requestCount - 1,
1571                                                execOneRequestParams.lastBackoffDelay,
1572                                                availableRetryCapacity >= 0 ?
1573                                                        availableRetryCapacity : "");
1574
1575             request.addHeader(HEADER_SDK_RETRY_INFO, headerValue);
1576         }
1577
1578         /**
1579          * Returns true if a failed request should be retried.
1580          *
1581          * @param params    Params for the individual request being executed.
1582          * @param exception The client/service exception from the failed request.
1583          * @return True if the failed request should be retried.
1584          */
1585         private boolean shouldRetry(ExecOneRequestParams params, SdkBaseException exception) {
1586             final int retriesAttempted = params.requestCount - 1;
1587             final HttpRequestBase method = params.apacheRequest;
1588
1589             // Never retry on requests containing non-repeatable entity
1590             if (method instanceof HttpEntityEnclosingRequest) {
1591                 HttpEntity entity = ((HttpEntityEnclosingRequest) method).getEntity();
1592                 if (entity != null && !entity.isRepeatable()) {
1593                     if (log.isDebugEnabled()) {
1594                         log.debug("Entity not repeatable");
1595                     }
1596                     return false;
1597                 }
1598             }
1599
1600             RetryPolicyContext context = RetryPolicyContext.builder()
1601                                                            .request(request)
1602                                                            .originalRequest(requestConfig.getOriginalRequest())
1603                                                            .exception(exception)
1604                                                            .retriesAttempted(retriesAttempted)
1605                                                            .httpStatusCode(params.getStatusCode())
1606                                                            .build();
1607
1608             if (!acquireRetryCapacity(context, params)) {
1609                 return false;
1610             }
1611
1612             // Finally, pass all the context information to the RetryCondition and let it
1613             // decide whether it should be retried.
1614             if (!retryPolicy.shouldRetry(context)) {
1615                 // If the retry policy fails we immediately return consumed capacity to the pool.
1616                 if (executionContext.retryCapacityConsumed()) {
1617                     retryCapacity.release(THROTTLED_RETRY_COST);
1618                 }
1619                 reportMaxRetriesExceededIfRetryable(context);
1620                 return false;
1621             }
1622
1623             return true;
1624         }
1625
1626         /**
1627          * Attempts to acquire retry capacity.
1628          *
1629          * @return true if retry capacity can be acquired, false otherwise.
1630          */
1631         private boolean acquireRetryCapacity(RetryPolicyContext context, ExecOneRequestParams params) {
1632             switch (retryMode) {
1633                 case LEGACY:
1634                     return legacyAcquireRetryCapacity(context, params);
1635                 case STANDARD:
1636                     return standardAcquireRetryCapacity(context, params);
1637                 default:
1638                     throw new IllegalStateException("Unsupported retry mode: " + retryMode);
1639             }
1640         }
1641
1642         private boolean standardAcquireRetryCapacity(RetryPolicyContext context, ExecOneRequestParams params) {
1643             SdkBaseException exception = context.exception();
1644             if (isTimeoutError(exception)) {
1645                 return doAcquireCapacity(context, TIMEOUT_RETRY_COST, params);
1646             }
1647
1648             return doAcquireCapacity(context, THROTTLED_RETRY_COST, params);
1649         }
1650
1651         private boolean isTimeoutError(SdkBaseException exception) {
1652             Throwable cause = exception.getCause();
1653             return cause instanceof ConnectTimeoutException || cause instanceof SocketTimeoutException;
1654         }
1655
1656         private boolean legacyAcquireRetryCapacity(RetryPolicyContext context, ExecOneRequestParams params) {
1657             // For legacy retry mode, we only attempt to acquire capacity for non-throttling errors
1658             if (!RetryUtils.isThrottlingException(context.exception())) {
1659                 // Do not use retry capacity for throttling exceptions if the retry mode
1660                 return doAcquireCapacity(context, THROTTLED_RETRY_COST, params);
1661             } else {
1662                 return true;
1663             }
1664         }
1665
1666         private boolean doAcquireCapacity(RetryPolicyContext context, int retryCost, ExecOneRequestParams params) {
1667             // See if we have enough available retry capacity to be able to execute
1668             // this retry attempt.
1669             if (!retryCapacity.acquire(retryCost)) {
1670                 awsRequestMetrics.incrementCounter(ThrottledRetryCount);
1671                 reportMaxRetriesExceededIfRetryable(context);
1672                 return false;
1673             }
1674             params.lastConsumedRetryCapacity = retryCost;
1675             executionContext.markRetryCapacityConsumed();
1676             return true;
1677         }
1678
1679         private void reportMaxRetriesExceededIfRetryable(RetryPolicyContext context) {
1680             if (retryPolicy instanceof RetryPolicyAdapter && ((RetryPolicyAdapter) retryPolicy).isRetryable(context)) {
1681                 awsRequestMetrics.addPropertyWith(MaxRetriesExceeded, true);
1682             }
1683         }
1684
1685         /**
1686          * Handles a successful response from a service call by unmarshalling the results using the
1687          * specified response handler.
1688          *
1689          * @return The contents of the response, unmarshalled using the specified response handler.
1690          * @throws IOException If any problems were encountered reading the response contents from
1691          *                     the HTTP method object.
1692          */
1693         @SuppressWarnings("deprecation")
1694         private Output handleResponse(HttpResponse httpResponse) throws IOException,
1695                                                                         InterruptedException {
1696             ProgressListener listener = requestConfig.getProgressListener();
1697             try {
1698             /*
1699              * Apply the byte counting stream wrapper if the legacy runtime profiling is enabled.
1700              */
1701                 CountingInputStream countingInputStream = null;
1702                 InputStream is = httpResponse.getContent();
1703                 if (is != null) {
1704                     if (System.getProperty(PROFILING_SYSTEM_PROPERTY) != null) {
1705                         is = countingInputStream = new CountingInputStream(is);
1706                         httpResponse.setContent(is);
1707                     }
1708                     httpResponse.setContent(ProgressInputStream.inputStreamForResponse(is, listener));
1709                 }
1710                 Map<String, String> headers = httpResponse.getHeaders();
1711                 String s = headers.get("Content-Length");
1712                 if (s != null) {
1713                     try {
1714                         long contentLength = Long.parseLong(s);
1715                         publishResponseContentLength(listener, contentLength);
1716                     } catch (NumberFormatException e) {
1717                         log.warn("Cannot parse the Content-Length header of the response.");
1718                     }
1719                 }
1720
1721                 Output awsResponse;
1722                 awsRequestMetrics.startEvent(Field.ResponseProcessingTime);
1723                 publishProgress(listener, ProgressEventType.HTTP_RESPONSE_STARTED_EVENT);
1724                 try {
1725                     awsResponse = responseHandler
1726                             .handle(beforeUnmarshalling(httpResponse));
1727                 } finally {
1728                     awsRequestMetrics.endEvent(Field.ResponseProcessingTime);
1729                 }
1730                 publishProgress(listener, ProgressEventType.HTTP_RESPONSE_COMPLETED_EVENT);
1731
1732                 if (countingInputStream != null) {
1733                     awsRequestMetrics
1734                             .setCounter(Field.BytesProcessed, countingInputStream.getByteCount());
1735                 }
1736                 return awsResponse;
1737             } catch (CRC32MismatchException e) {
1738                 throw e;
1739             } catch (IOException e) {
1740                 throw e;
1741             } catch (AmazonClientException e) {
1742                 throw e; // simply rethrow rather than further wrapping it
1743             } catch (InterruptedException e) {
1744                 throw e;
1745             } catch (Exception e) {
1746                 String errorMessage =
1747                         "Unable to unmarshall response (" + e.getMessage() + "). Response Code: "
1748                         + httpResponse.getStatusCode() + ", Response Text: " +
1749                         httpResponse.getStatusText();
1750                 throw new SdkClientException(errorMessage, e);
1751             }
1752         }
1753
1754         /**
1755          * Run {@link RequestHandler2#beforeUnmarshalling(Request, HttpResponse)} callback
1756          *
1757          * @param origHttpResponse Original {@link HttpResponse}
1758          * @return {@link HttpResponse} object to pass to unmarshaller. May have been modified or
1759          * replaced by the request handlers
1760          */
1761         private HttpResponse beforeUnmarshalling(HttpResponse origHttpResponse) {
1762             HttpResponse toReturn = origHttpResponse;
1763             for (RequestHandler2 requestHandler : requestHandler2s) {
1764                 toReturn = requestHandler.beforeUnmarshalling(request, toReturn);
1765             }
1766             return toReturn;
1767         }
1768
1769         /**
1770          * Responsible for handling an error response, including unmarshalling the error response
1771          * into the most specific exception type possible, and throwing the exception.
1772          *
1773          * @param method The HTTP method containing the actual response content.
1774          * @throws IOException If any problems are encountering reading the error response.
1775          */
1776         private SdkBaseException handleErrorResponse(HttpRequestBase method,
1777                                                            final org.apache.http.HttpResponse apacheHttpResponse,
1778                                                            final HttpContext context)
1779                 throws IOException, InterruptedException {
1780             final StatusLine statusLine = apacheHttpResponse.getStatusLine();
1781             final int statusCode;
1782             final String reasonPhrase;
1783             if (statusLine == null) {
1784                 statusCode = -1;
1785                 reasonPhrase = null;
1786             } else {
1787                 statusCode = statusLine.getStatusCode();
1788                 reasonPhrase = statusLine.getReasonPhrase();
1789             }
1790             HttpResponse response = ApacheUtils.createResponse(request, method, apacheHttpResponse, context);
1791             SdkBaseException exception;
1792             try {
1793                 exception = errorResponseHandler.handle(response);
1794                 if (requestLog.isDebugEnabled()) {
1795                     requestLog.debug("Received error response: " + exception);
1796                 }
1797             } catch (InterruptedException e) {
1798                 throw e;
1799             } catch (Exception e) {
1800                 if (e instanceof IOException) {
1801                     throw (IOException) e;
1802                 } else {
1803                     String errorMessage = "Unable to unmarshall error response (" + e.getMessage() +
1804                                           "). Response Code: "
1805                                           + (statusLine == null ? "None" : statusCode) +
1806                                           ", Response Text: " + reasonPhrase;
1807                     throw new SdkClientException(errorMessage, e);
1808                 }
1809             }
1810
1811             exception.fillInStackTrace();
1812             return exception;
1813         }
1814
1815         /**
1816          * Pause before the next retry and record metrics around retry behavior.
1817          */
1818         private void pauseBeforeRetry(ExecOneRequestParams execOneParams,
1819                                       final ProgressListener listener) throws InterruptedException {
1820             publishProgress(listener, ProgressEventType.CLIENT_REQUEST_RETRY_EVENT);
1821             // Notify the progress listener of the retry
1822             awsRequestMetrics.startEvent(Field.RetryPauseTime);
1823             try {
1824                 doPauseBeforeRetry(execOneParams);
1825             } finally {
1826                 awsRequestMetrics.endEvent(Field.RetryPauseTime);
1827             }
1828         }
1829
1830         /**
1831          * Sleep for a period of time on failed request to avoid flooding a service with retries.
1832          */
1833         private void doPauseBeforeRetry(ExecOneRequestParams execOneParams) throws InterruptedException {
1834             final int retriesAttempted = execOneParams.requestCount - 2;
1835             RetryPolicyContext context = RetryPolicyContext.builder()
1836                     .request(request)
1837                     .originalRequest(requestConfig.getOriginalRequest())
1838                     .retriesAttempted(retriesAttempted)
1839                     .exception(execOneParams.retriedException)
1840                     .build();
1841             // don't pause if the retry was not due to a redirection (I.E. when retried exception is null)
1842             if (context.exception() != null) {
1843                 long delay = retryPolicy.computeDelayBeforeNextRetry(context);
1844                 execOneParams.lastBackoffDelay = delay;
1845
1846                 if (log.isDebugEnabled()) {
1847                     log.debug("Retriable error detected, " + "will retry in " + delay +
1848                               "ms, attempt number: " + retriesAttempted);
1849                 }
1850                 Thread.sleep(delay);
1851             }
1852         }
1853
1854         /**
1855          * Gets the correct request timeout taking into account precedence of the configuration in
1856          * {@link AmazonWebServiceRequest} versus {@link ClientConfiguration}
1857          *
1858          * @param requestConfig Current request configuration
1859          * @return Request timeout value or 0 if none is set
1860          */
1861         private int getRequestTimeout(RequestConfig requestConfig) {
1862             if (requestConfig.getRequestTimeout() != null) {
1863                 return requestConfig.getRequestTimeout();
1864             } else {
1865                 return config.getRequestTimeout();
1866             }
1867         }
1868
1869         /**
1870          * Gets the correct client execution timeout taking into account precedence of the
1871          * configuration in {@link AmazonWebServiceRequest} versus {@link ClientConfiguration}
1872          *
1873          * @param requestConfig Current request configuration
1874          * @return Client Execution timeout value or 0 if none is set
1875          */
1876         private int getClientExecutionTimeout(RequestConfig requestConfig) {
1877             if (requestConfig.getClientExecutionTimeout() != null) {
1878                 return requestConfig.getClientExecutionTimeout();
1879             } else {
1880                 return config.getClientExecutionTimeout();
1881             }
1882         }
1883
1884         /**
1885          * Stateful parameters that are used for executing a single httpClientSettings request.
1886          */
1887         private class ExecOneRequestParams {
1888             int requestCount; // monotonic increasing
1889             /**
1890              * Last delay between retries
1891              */
1892             long lastBackoffDelay = 0;
1893             SdkBaseException retriedException; // last retryable exception
1894             HttpRequestBase apacheRequest;
1895             org.apache.http.HttpResponse apacheResponse;
1896             URI redirectedURI;
1897             AuthRetryParameters authRetryParam;
1898             int lastConsumedRetryCapacity;
1899             /*
1900              * Depending on which response handler we end up choosing to handle the HTTP response, it
1901              * might require us to leave the underlying HTTP connection open, depending on whether or
1902              * not it reads the complete HTTP response stream from the HTTP connection, or if delays
1903              * reading any of the content until after a response is returned to the caller.
1904              */
1905             boolean leaveHttpConnectionOpen;
1906             private Signer signer; // cached
1907             private URI signerURI;
1908
1909             boolean isRetry() {
1910                 return requestCount > 1 || redirectedURI != null || authRetryParam != null;
1911             }
1912
1913             void initPerRetry() {
1914                 requestCount++;
1915                 apacheRequest = null;
1916                 apacheResponse = null;
1917                 leaveHttpConnectionOpen = false;
1918             }
1919
1920             void newSigner(final Request<?> request, final ExecutionContext execContext) {
1921                 final SignerProviderContext.Builder signerProviderContext = SignerProviderContext
1922                         .builder()
1923                         .withRequest(request)
1924                         .withRequestConfig(requestConfig);
1925                 if (authRetryParam != null) {
1926                     signerURI = authRetryParam.getEndpointForRetry();
1927                     signer = authRetryParam.getSignerForRetry();
1928                     // Push the local signer override back to the execution context
1929                     execContext.setSigner(signer);
1930                 } else if (redirectedURI != null && !redirectedURI.equals(signerURI)) {
1931                     signerURI = redirectedURI;
1932                     signer = execContext.getSigner(signerProviderContext
1933                                                            .withUri(signerURI)
1934                                                            .withIsRedirect(true)
1935                                                            .build());
1936
1937                     if (signer instanceof AWS4Signer) {
1938                         String regionName = ((AWS4Signer) signer).getRegionName();
1939                         if (regionName != null) {
1940                             request.addHandlerContext(HandlerContextKey.SIGNING_REGION, regionName);
1941                         }
1942                     }
1943                 } else if (signer == null) {
1944                     signerURI = request.getEndpoint();
1945                     signer = execContext
1946                             .getSigner(signerProviderContext.withUri(signerURI).build());
1947                 }
1948             }
1949
1950             /**
1951              * @throws FakeIOException thrown only during test simulation
1952              */
1953             HttpRequestBase newApacheRequest(
1954                     final HttpRequestFactory<HttpRequestBase> httpRequestFactory,
1955                     final Request<?> request,
1956                     final HttpClientSettings options) throws IOException {
1957
1958                 apacheRequest = httpRequestFactory.create(request, options);
1959                 if (redirectedURI != null) {
1960                     apacheRequest.setURI(redirectedURI);
1961                 }
1962                 return apacheRequest;
1963             }
1964
1965             void resetBeforeHttpRequest() {
1966                 retriedException = null;
1967                 authRetryParam = null;
1968                 redirectedURI = null;
1969             }
1970
1971             private Integer getStatusCode() {
1972                 if (apacheResponse == null || apacheResponse.getStatusLine() == null) {
1973                     return null;
1974                 }
1975                 return apacheResponse.getStatusLine().getStatusCode();
1976             }
1977         }
1978     }
1979 }
1980