1 /*
2  * Copyright 2010-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License").
5  * You may not use this file except in compliance with the License.
6  * A copy of the License is located at
7  *
8  *  http://aws.amazon.com/apache2.0
9  *
10  * or in the "license" file accompanying this file. This file is distributed
11  * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12  * express or implied. See the License for the specific language governing
13  * permissions and limitations under the License.
14  */
15 package com.amazonaws.services.s3;
16
17 import static com.amazonaws.event.SDKProgressPublisher.publishProgress;
18 import static com.amazonaws.internal.ResettableInputStream.newResettableInputStream;
19 import static com.amazonaws.services.s3.model.S3DataSource.Utils.cleanupDataSource;
20 import static com.amazonaws.util.LengthCheckInputStream.EXCLUDE_SKIPPED_BYTES;
21 import static com.amazonaws.util.LengthCheckInputStream.INCLUDE_SKIPPED_BYTES;
22 import static com.amazonaws.util.Throwables.failure;
23 import static com.amazonaws.util.ValidationUtils.assertNotNull;
24 import static com.amazonaws.util.ValidationUtils.assertStringNotEmpty;
25
26 import com.amazonaws.AmazonClientException;
27 import com.amazonaws.AmazonServiceException;
28 import com.amazonaws.AmazonServiceException.ErrorType;
29 import com.amazonaws.AmazonWebServiceClient;
30 import com.amazonaws.AmazonWebServiceRequest;
31 import com.amazonaws.AmazonWebServiceResponse;
32 import com.amazonaws.ClientConfiguration;
33 import com.amazonaws.DefaultRequest;
34 import com.amazonaws.HttpMethod;
35 import com.amazonaws.Protocol;
36 import com.amazonaws.Request;
37 import com.amazonaws.RequestConfig;
38 import com.amazonaws.ResetException;
39 import com.amazonaws.Response;
40 import com.amazonaws.SDKGlobalConfiguration;
41 import com.amazonaws.SdkClientException;
42 import com.amazonaws.annotation.SdkInternalApi;
43 import com.amazonaws.annotation.SdkTestInternalApi;
44 import com.amazonaws.annotation.ThreadSafe;
45 import com.amazonaws.arn.Arn;
46 import com.amazonaws.auth.AWSCredentials;
47 import com.amazonaws.auth.AWSCredentialsProvider;
48 import com.amazonaws.auth.Presigner;
49 import com.amazonaws.auth.Signer;
50 import com.amazonaws.auth.SignerFactory;
51 import com.amazonaws.client.builder.AwsClientBuilder;
52 import com.amazonaws.event.ProgressEventType;
53 import com.amazonaws.event.ProgressInputStream;
54 import com.amazonaws.event.ProgressListener;
55 import com.amazonaws.handlers.HandlerChainFactory;
56 import com.amazonaws.handlers.HandlerContextKey;
57 import com.amazonaws.handlers.RequestHandler2;
58 import com.amazonaws.http.ExecutionContext;
59 import com.amazonaws.http.HttpMethodName;
60 import com.amazonaws.http.HttpResponseHandler;
61 import com.amazonaws.internal.AmazonWebServiceRequestAdapter;
62 import com.amazonaws.internal.DefaultServiceEndpointBuilder;
63 import com.amazonaws.internal.IdentityEndpointBuilder;
64 import com.amazonaws.internal.ReleasableInputStream;
65 import com.amazonaws.internal.ResettableInputStream;
66 import com.amazonaws.internal.SdkFilterInputStream;
67 import com.amazonaws.internal.ServiceEndpointBuilder;
68 import com.amazonaws.internal.StaticCredentialsProvider;
69 import com.amazonaws.internal.auth.NoOpSignerProvider;
70 import com.amazonaws.internal.auth.SignerProvider;
71 import com.amazonaws.metrics.AwsSdkMetrics;
72 import com.amazonaws.metrics.RequestMetricCollector;
73 import com.amazonaws.regions.RegionUtils;
74 import com.amazonaws.regions.Regions;
75 import com.amazonaws.retry.PredefinedRetryPolicies;
76 import com.amazonaws.retry.RetryPolicy;
77 import com.amazonaws.services.s3.internal.AWSS3V4Signer;
78 import com.amazonaws.services.s3.internal.BucketNameUtils;
79 import com.amazonaws.services.s3.internal.CompleteMultipartUploadRetryCondition;
80 import com.amazonaws.services.s3.internal.Constants;
81 import com.amazonaws.services.s3.internal.DeleteObjectTaggingHeaderHandler;
82 import com.amazonaws.services.s3.internal.DeleteObjectsResponse;
83 import com.amazonaws.services.s3.internal.DigestValidationInputStream;
84 import com.amazonaws.services.s3.internal.DualstackEndpointBuilder;
85 import com.amazonaws.services.s3.internal.GetObjectTaggingResponseHeaderHandler;
86 import com.amazonaws.services.s3.internal.InitiateMultipartUploadHeaderHandler;
87 import com.amazonaws.services.s3.internal.InputSubstream;
88 import com.amazonaws.services.s3.internal.ListPartsHeaderHandler;
89 import com.amazonaws.services.s3.internal.MD5DigestCalculatingInputStream;
90 import com.amazonaws.services.s3.internal.Mimetypes;
91 import com.amazonaws.services.s3.internal.MultiFileOutputStream;
92 import com.amazonaws.services.s3.internal.ObjectExpirationHeaderHandler;
93 import com.amazonaws.services.s3.internal.RegionalEndpointsOptionResolver;
94 import com.amazonaws.services.s3.internal.ResponseHeaderHandlerChain;
95 import com.amazonaws.services.s3.internal.S3AbortableInputStream;
96 import com.amazonaws.services.s3.internal.S3AccessPointBuilder;
97 import com.amazonaws.services.s3.internal.S3ErrorResponseHandler;
98 import com.amazonaws.services.s3.internal.S3MetadataResponseHandler;
99 import com.amazonaws.services.s3.internal.S3ObjectResponseHandler;
100 import com.amazonaws.services.s3.internal.S3QueryStringSigner;
101 import com.amazonaws.services.s3.internal.S3RequestEndpointResolver;
102 import com.amazonaws.services.s3.internal.S3RequesterChargedHeaderHandler;
103 import com.amazonaws.services.s3.internal.S3RestoreOutputPathHeaderHandler;
104 import com.amazonaws.services.s3.internal.S3Signer;
105 import com.amazonaws.services.s3.internal.S3StringResponseHandler;
106 import com.amazonaws.services.s3.internal.S3V4AuthErrorRetryStrategy;
107 import com.amazonaws.services.s3.internal.S3VersionHeaderHandler;
108 import com.amazonaws.services.s3.internal.S3XmlResponseHandler;
109 import com.amazonaws.services.s3.internal.ServerSideEncryptionHeaderHandler;
110 import com.amazonaws.services.s3.internal.ServiceUtils;
111 import com.amazonaws.services.s3.internal.SetObjectTaggingResponseHeaderHandler;
112 import com.amazonaws.services.s3.internal.SkipMd5CheckStrategy;
113 import com.amazonaws.services.s3.internal.UploadObjectStrategy;
114 import com.amazonaws.services.s3.internal.UseArnRegionResolver;
115 import com.amazonaws.services.s3.internal.XmlWriter;
116 import com.amazonaws.services.s3.internal.auth.S3SignerProvider;
117 import com.amazonaws.services.s3.metrics.S3ServiceMetric;
118 import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
119 import com.amazonaws.services.s3.model.AccessControlList;
120 import com.amazonaws.services.s3.model.AmazonS3Exception;
121 import com.amazonaws.services.s3.model.Bucket;
122 import com.amazonaws.services.s3.model.BucketAccelerateConfiguration;
123 import com.amazonaws.services.s3.model.BucketCrossOriginConfiguration;
124 import com.amazonaws.services.s3.model.BucketLifecycleConfiguration;
125 import com.amazonaws.services.s3.model.BucketLoggingConfiguration;
126 import com.amazonaws.services.s3.model.BucketNotificationConfiguration;
127 import com.amazonaws.services.s3.model.BucketPolicy;
128 import com.amazonaws.services.s3.model.BucketReplicationConfiguration;
129 import com.amazonaws.services.s3.model.BucketTaggingConfiguration;
130 import com.amazonaws.services.s3.model.BucketVersioningConfiguration;
131 import com.amazonaws.services.s3.model.BucketWebsiteConfiguration;
132 import com.amazonaws.services.s3.model.CannedAccessControlList;
133 import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
134 import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
135 import com.amazonaws.services.s3.model.CopyObjectRequest;
136 import com.amazonaws.services.s3.model.CopyObjectResult;
137 import com.amazonaws.services.s3.model.CopyPartRequest;
138 import com.amazonaws.services.s3.model.CopyPartResult;
139 import com.amazonaws.services.s3.model.CreateBucketRequest;
140 import com.amazonaws.services.s3.model.DeleteBucketAnalyticsConfigurationRequest;
141 import com.amazonaws.services.s3.model.DeleteBucketAnalyticsConfigurationResult;
142 import com.amazonaws.services.s3.model.DeleteBucketCrossOriginConfigurationRequest;
143 import com.amazonaws.services.s3.model.DeleteBucketEncryptionRequest;
144 import com.amazonaws.services.s3.model.DeleteBucketEncryptionResult;
145 import com.amazonaws.services.s3.model.DeleteBucketInventoryConfigurationRequest;
146 import com.amazonaws.services.s3.model.DeleteBucketInventoryConfigurationResult;
147 import com.amazonaws.services.s3.model.DeleteBucketLifecycleConfigurationRequest;
148 import com.amazonaws.services.s3.model.DeleteBucketMetricsConfigurationRequest;
149 import com.amazonaws.services.s3.model.DeleteBucketMetricsConfigurationResult;
150 import com.amazonaws.services.s3.model.DeleteBucketPolicyRequest;
151 import com.amazonaws.services.s3.model.DeleteBucketReplicationConfigurationRequest;
152 import com.amazonaws.services.s3.model.DeleteBucketRequest;
153 import com.amazonaws.services.s3.model.DeleteBucketTaggingConfigurationRequest;
154 import com.amazonaws.services.s3.model.DeleteBucketWebsiteConfigurationRequest;
155 import com.amazonaws.services.s3.model.DeleteObjectRequest;
156 import com.amazonaws.services.s3.model.DeleteObjectTaggingRequest;
157 import com.amazonaws.services.s3.model.DeleteObjectTaggingResult;
158 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
159 import com.amazonaws.services.s3.model.DeleteObjectsResult;
160 import com.amazonaws.services.s3.model.DeletePublicAccessBlockRequest;
161 import com.amazonaws.services.s3.model.DeletePublicAccessBlockResult;
162 import com.amazonaws.services.s3.model.DeleteVersionRequest;
163 import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
164 import com.amazonaws.services.s3.model.GenericBucketRequest;
165 import com.amazonaws.services.s3.model.GetBucketAccelerateConfigurationRequest;
166 import com.amazonaws.services.s3.model.GetBucketAclRequest;
167 import com.amazonaws.services.s3.model.GetBucketAnalyticsConfigurationRequest;
168 import com.amazonaws.services.s3.model.GetBucketAnalyticsConfigurationResult;
169 import com.amazonaws.services.s3.model.GetBucketCrossOriginConfigurationRequest;
170 import com.amazonaws.services.s3.model.GetBucketEncryptionRequest;
171 import com.amazonaws.services.s3.model.GetBucketEncryptionResult;
172 import com.amazonaws.services.s3.model.GetBucketInventoryConfigurationRequest;
173 import com.amazonaws.services.s3.model.GetBucketInventoryConfigurationResult;
174 import com.amazonaws.services.s3.model.GetBucketLifecycleConfigurationRequest;
175 import com.amazonaws.services.s3.model.GetBucketLocationRequest;
176 import com.amazonaws.services.s3.model.GetBucketLoggingConfigurationRequest;
177 import com.amazonaws.services.s3.model.GetBucketMetricsConfigurationRequest;
178 import com.amazonaws.services.s3.model.GetBucketMetricsConfigurationResult;
179 import com.amazonaws.services.s3.model.GetBucketNotificationConfigurationRequest;
180 import com.amazonaws.services.s3.model.GetBucketPolicyRequest;
181 import com.amazonaws.services.s3.model.GetBucketPolicyStatusRequest;
182 import com.amazonaws.services.s3.model.GetBucketPolicyStatusResult;
183 import com.amazonaws.services.s3.model.GetBucketReplicationConfigurationRequest;
184 import com.amazonaws.services.s3.model.GetBucketTaggingConfigurationRequest;
185 import com.amazonaws.services.s3.model.GetBucketVersioningConfigurationRequest;
186 import com.amazonaws.services.s3.model.GetBucketWebsiteConfigurationRequest;
187 import com.amazonaws.services.s3.model.GetObjectAclRequest;
188 import com.amazonaws.services.s3.model.GetObjectLegalHoldRequest;
189 import com.amazonaws.services.s3.model.GetObjectLegalHoldResult;
190 import com.amazonaws.services.s3.model.GetObjectLockConfigurationRequest;
191 import com.amazonaws.services.s3.model.GetObjectLockConfigurationResult;
192 import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
193 import com.amazonaws.services.s3.model.GetObjectRequest;
194 import com.amazonaws.services.s3.model.GetObjectRetentionRequest;
195 import com.amazonaws.services.s3.model.GetObjectRetentionResult;
196 import com.amazonaws.services.s3.model.GetObjectTaggingRequest;
197 import com.amazonaws.services.s3.model.GetObjectTaggingResult;
198 import com.amazonaws.services.s3.model.GetPublicAccessBlockRequest;
199 import com.amazonaws.services.s3.model.GetPublicAccessBlockResult;
200 import com.amazonaws.services.s3.model.GetRequestPaymentConfigurationRequest;
201 import com.amazonaws.services.s3.model.GetS3AccountOwnerRequest;
202 import com.amazonaws.services.s3.model.Grant;
203 import com.amazonaws.services.s3.model.Grantee;
204 import com.amazonaws.services.s3.model.GroupGrantee;
205 import com.amazonaws.services.s3.model.HeadBucketRequest;
206 import com.amazonaws.services.s3.model.HeadBucketResult;
207 import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
208 import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
209 import com.amazonaws.services.s3.model.ListBucketAnalyticsConfigurationsRequest;
210 import com.amazonaws.services.s3.model.ListBucketAnalyticsConfigurationsResult;
211 import com.amazonaws.services.s3.model.ListBucketInventoryConfigurationsRequest;
212 import com.amazonaws.services.s3.model.ListBucketInventoryConfigurationsResult;
213 import com.amazonaws.services.s3.model.ListBucketMetricsConfigurationsRequest;
214 import com.amazonaws.services.s3.model.ListBucketMetricsConfigurationsResult;
215 import com.amazonaws.services.s3.model.ListBucketsRequest;
216 import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
217 import com.amazonaws.services.s3.model.ListNextBatchOfObjectsRequest;
218 import com.amazonaws.services.s3.model.ListNextBatchOfVersionsRequest;
219 import com.amazonaws.services.s3.model.ListObjectsRequest;
220 import com.amazonaws.services.s3.model.ListObjectsV2Request;
221 import com.amazonaws.services.s3.model.ListObjectsV2Result;
222 import com.amazonaws.services.s3.model.ListPartsRequest;
223 import com.amazonaws.services.s3.model.ListVersionsRequest;
224 import com.amazonaws.services.s3.model.MultiFactorAuthentication;
225 import com.amazonaws.services.s3.model.MultiObjectDeleteException;
226 import com.amazonaws.services.s3.model.MultipartUploadListing;
227 import com.amazonaws.services.s3.model.ObjectListing;
228 import com.amazonaws.services.s3.model.ObjectMetadata;
229 import com.amazonaws.services.s3.model.ObjectTagging;
230 import com.amazonaws.services.s3.model.Owner;
231 import com.amazonaws.services.s3.model.PartETag;
232 import com.amazonaws.services.s3.model.PartListing;
233 import com.amazonaws.services.s3.model.Permission;
234 import com.amazonaws.services.s3.model.PresignedUrlDownloadRequest;
235 import com.amazonaws.services.s3.model.PresignedUrlDownloadResult;
236 import com.amazonaws.services.s3.model.PresignedUrlUploadRequest;
237 import com.amazonaws.services.s3.model.PresignedUrlUploadResult;
238 import com.amazonaws.services.s3.model.PublicAccessBlockConfiguration;
239 import com.amazonaws.services.s3.model.PutObjectRequest;
240 import com.amazonaws.services.s3.model.PutObjectResult;
241 import com.amazonaws.services.s3.model.Region;
242 import com.amazonaws.services.s3.model.RequestPaymentConfiguration;
243 import com.amazonaws.services.s3.model.RequestPaymentConfiguration.Payer;
244 import com.amazonaws.services.s3.model.ResponseHeaderOverrides;
245 import com.amazonaws.services.s3.model.RestoreObjectRequest;
246 import com.amazonaws.services.s3.model.RestoreObjectResult;
247 import com.amazonaws.services.s3.model.RestoreRequestType;
248 import com.amazonaws.services.s3.model.S3AccelerateUnsupported;
249 import com.amazonaws.services.s3.model.S3DataSource;
250 import com.amazonaws.services.s3.model.S3Object;
251 import com.amazonaws.services.s3.model.S3ObjectInputStream;
252 import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
253 import com.amazonaws.services.s3.model.SSEAwsKeyManagementParamsProvider;
254 import com.amazonaws.services.s3.model.SSECustomerKey;
255 import com.amazonaws.services.s3.model.SSECustomerKeyProvider;
256 import com.amazonaws.services.s3.model.SelectObjectContentEventStream;
257 import com.amazonaws.services.s3.model.SelectObjectContentRequest;
258 import com.amazonaws.services.s3.model.SelectObjectContentResult;
259 import com.amazonaws.services.s3.model.ServerSideEncryptionConfiguration;
260 import com.amazonaws.services.s3.model.SetBucketAccelerateConfigurationRequest;
261 import com.amazonaws.services.s3.model.SetBucketAclRequest;
262 import com.amazonaws.services.s3.model.SetBucketAnalyticsConfigurationRequest;
263 import com.amazonaws.services.s3.model.SetBucketAnalyticsConfigurationResult;
264 import com.amazonaws.services.s3.model.SetBucketCrossOriginConfigurationRequest;
265 import com.amazonaws.services.s3.model.SetBucketEncryptionRequest;
266 import com.amazonaws.services.s3.model.SetBucketEncryptionResult;
267 import com.amazonaws.services.s3.model.SetBucketInventoryConfigurationRequest;
268 import com.amazonaws.services.s3.model.SetBucketInventoryConfigurationResult;
269 import com.amazonaws.services.s3.model.SetBucketLifecycleConfigurationRequest;
270 import com.amazonaws.services.s3.model.SetBucketLoggingConfigurationRequest;
271 import com.amazonaws.services.s3.model.SetBucketMetricsConfigurationRequest;
272 import com.amazonaws.services.s3.model.SetBucketMetricsConfigurationResult;
273 import com.amazonaws.services.s3.model.SetBucketNotificationConfigurationRequest;
274 import com.amazonaws.services.s3.model.SetBucketPolicyRequest;
275 import com.amazonaws.services.s3.model.SetBucketReplicationConfigurationRequest;
276 import com.amazonaws.services.s3.model.SetBucketTaggingConfigurationRequest;
277 import com.amazonaws.services.s3.model.SetBucketVersioningConfigurationRequest;
278 import com.amazonaws.services.s3.model.SetBucketWebsiteConfigurationRequest;
279 import com.amazonaws.services.s3.model.SetObjectAclRequest;
280 import com.amazonaws.services.s3.model.SetObjectLegalHoldRequest;
281 import com.amazonaws.services.s3.model.SetObjectLegalHoldResult;
282 import com.amazonaws.services.s3.model.SetObjectLockConfigurationRequest;
283 import com.amazonaws.services.s3.model.SetObjectLockConfigurationResult;
284 import com.amazonaws.services.s3.model.SetObjectRetentionRequest;
285 import com.amazonaws.services.s3.model.SetObjectRetentionResult;
286 import com.amazonaws.services.s3.model.SetObjectTaggingRequest;
287 import com.amazonaws.services.s3.model.SetObjectTaggingResult;
288 import com.amazonaws.services.s3.model.SetPublicAccessBlockRequest;
289 import com.amazonaws.services.s3.model.SetPublicAccessBlockResult;
290 import com.amazonaws.services.s3.model.SetRequestPaymentConfigurationRequest;
291 import com.amazonaws.services.s3.model.StorageClass;
292 import com.amazonaws.services.s3.model.Tag;
293 import com.amazonaws.services.s3.model.UploadObjectRequest;
294 import com.amazonaws.services.s3.model.UploadPartRequest;
295 import com.amazonaws.services.s3.model.UploadPartResult;
296 import com.amazonaws.services.s3.model.VersionListing;
297 import com.amazonaws.services.s3.model.analytics.AnalyticsConfiguration;
298 import com.amazonaws.services.s3.model.inventory.InventoryConfiguration;
299 import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
300 import com.amazonaws.services.s3.model.transform.AclXmlFactory;
301 import com.amazonaws.services.s3.model.transform.BucketConfigurationXmlFactory;
302 import com.amazonaws.services.s3.model.transform.BucketNotificationConfigurationStaxUnmarshaller;
303 import com.amazonaws.services.s3.model.transform.GetBucketEncryptionStaxUnmarshaller;
304 import com.amazonaws.services.s3.model.transform.GetBucketPolicyStatusStaxUnmarshaller;
305 import com.amazonaws.services.s3.model.transform.GetPublicAccessBlockStaxUnmarshaller;
306 import com.amazonaws.services.s3.model.transform.HeadBucketResultHandler;
307 import com.amazonaws.services.s3.model.transform.MultiObjectDeleteXmlFactory;
308 import com.amazonaws.services.s3.model.transform.ObjectLockConfigurationXmlFactory;
309 import com.amazonaws.services.s3.model.transform.ObjectLockLegalHoldXmlFactory;
310 import com.amazonaws.services.s3.model.transform.ObjectLockRetentionXmlFactory;
311 import com.amazonaws.services.s3.model.transform.ObjectTaggingXmlFactory;
312 import com.amazonaws.services.s3.model.transform.RequestPaymentConfigurationXmlFactory;
313 import com.amazonaws.services.s3.model.transform.RequestXmlFactory;
314 import com.amazonaws.services.s3.model.transform.Unmarshallers;
315 import com.amazonaws.services.s3.model.transform.XmlResponsesSaxParser.CompleteMultipartUploadHandler;
316 import com.amazonaws.services.s3.model.transform.XmlResponsesSaxParser.CopyObjectResultHandler;
317 import com.amazonaws.services.s3.request.S3HandlerContextKeys;
318 import com.amazonaws.services.s3.waiters.AmazonS3Waiters;
319 import com.amazonaws.transform.Unmarshaller;
320 import com.amazonaws.util.AWSRequestMetrics;
321 import com.amazonaws.util.AWSRequestMetrics.Field;
322 import com.amazonaws.util.AwsHostNameUtils;
323 import com.amazonaws.util.Base16;
324 import com.amazonaws.util.Base64;
325 import com.amazonaws.util.BinaryUtils;
326 import com.amazonaws.util.CredentialUtils;
327 import com.amazonaws.util.DateUtils;
328 import com.amazonaws.util.IOUtils;
329 import com.amazonaws.util.LengthCheckInputStream;
330 import com.amazonaws.util.Md5Utils;
331 import com.amazonaws.util.RuntimeHttpUtils;
332 import com.amazonaws.util.SdkHttpUtils;
333 import com.amazonaws.util.ServiceClientHolderInputStream;
334 import com.amazonaws.util.StringUtils;
335 import com.amazonaws.util.ValidationUtils;
336 import java.io.ByteArrayInputStream;
337 import java.io.File;
338 import java.io.FileInputStream;
339 import java.io.IOException;
340 import java.io.InputStream;
341 import java.io.OutputStream;
342 import java.net.URI;
343 import java.net.URISyntaxException;
344 import java.net.URL;
345 import java.security.MessageDigest;
346 import java.security.NoSuchAlgorithmException;
347 import java.util.ArrayList;
348 import java.util.Arrays;
349 import java.util.Collection;
350 import java.util.Collections;
351 import java.util.Date;
352 import java.util.HashMap;
353 import java.util.Iterator;
354 import java.util.LinkedHashMap;
355 import java.util.LinkedList;
356 import java.util.List;
357 import java.util.Map;
358 import java.util.Map.Entry;
359 import java.util.concurrent.ExecutionException;
360 import java.util.concurrent.ExecutorService;
361 import java.util.concurrent.Executors;
362 import java.util.concurrent.Future;
363 import java.util.regex.Matcher;
364 import org.apache.commons.logging.Log;
365 import org.apache.commons.logging.LogFactory;
366 import org.apache.http.client.methods.HttpRequestBase;
367 import org.apache.http.entity.ContentType;
368
369 /**
370  * <p>
371  * Provides the client for accessing the Amazon S3 web service.
372  * </p>
373  * <p>
374  * Amazon S3 provides storage for the Internet,
375  * and is designed to make web-scale computing easier for developers.
376  * </p>
377  * <p>
378  * The Amazon S3 Java Client provides a simple interface that can be
379  * used to store and retrieve any amount of data, at any time,
380  * from anywhere on the web. It gives any developer access to the same
381  * highly scalable, reliable, secure, fast, inexpensive infrastructure
382  * that Amazon uses to run its own global network of web sites.
383  * The service aims to maximize benefits of scale and to pass those
384  * benefits on to developers.
385  * </p>
386  * <p>
387  * For more information about Amazon S3, please see
388  * <a href="http://aws.amazon.com/s3">
389  * http://aws.amazon.com/s3</a>
390  * </p>
391  */
392
393 @ThreadSafe
394 public class AmazonS3Client extends AmazonWebServiceClient implements AmazonS3 {
395
396     public static final String S3_SERVICE_NAME = "s3";
397
398     private static final String S3_SIGNER = "S3SignerType";
399     private static final String S3_V4_SIGNER = "AWSS3V4SignerType";
400     private static final String SERVICE_ID = "S3";
401     private static final String AWS_PARTITION_KEY = "aws";
402
403     protected static final AmazonS3ClientConfigurationFactory configFactory
404             = new AmazonS3ClientConfigurationFactory();
405
406     /** Shared logger for client events */
407     private static Log log = LogFactory.getLog(AmazonS3Client.class);
408
409     static {
410         // Enable S3 specific predefined request metrics.
411         AwsSdkMetrics.addAll(Arrays.asList(S3ServiceMetric.values()));
412
413         // Register S3-specific signers.
414         SignerFactory.registerSigner(S3_SIGNER, S3Signer.class);
415         SignerFactory.registerSigner(S3_V4_SIGNER, AWSS3V4Signer.class);
416     }
417
418     private volatile AmazonS3Waiters waiters;
419
420     /** Provider for AWS credentials. */
421     protected final AWSCredentialsProvider awsCredentialsProvider;
422
423     /** Responsible for handling error responses from all S3 service calls. */
424     protected final S3ErrorResponseHandler errorResponseHandler;
425
426     /** Shared response handler for operations with no response.  */
427     private final S3XmlResponseHandler<Void> voidResponseHandler = new S3XmlResponseHandler<Void>(null);
428
429     /** Shared factory for converting configuration objects to XML */
430     private static final BucketConfigurationXmlFactory bucketConfigurationXmlFactory = new BucketConfigurationXmlFactory();
431
432     /** Shared factory for converting request payment configuration objects to XML */
433     private static final RequestPaymentConfigurationXmlFactory requestPaymentConfigurationXmlFactory = new RequestPaymentConfigurationXmlFactory();
434
435     private static final UseArnRegionResolver USE_ARN_REGION_RESOLVER = new UseArnRegionResolver();
436
437     /** S3 specific client configuration options */
438     private volatile S3ClientOptions clientOptions = S3ClientOptions.builder().build();
439
440     /**
441      * The S3 client region that is set by either (a) calling
442      * setRegion/configureRegion OR (b) calling setEndpoint with a
443      * region-specific S3 endpoint. This region string will be used for signing
444      * requests sent by this client.
445      */
446     private volatile String clientRegion;
447
448     private static RegionalEndpointsOptionResolver REGIONAL_ENDPOINTS_OPTION_RESOLVER = new RegionalEndpointsOptionResolver();
449
450     private static final int BUCKET_REGION_CACHE_SIZE = 300;
451
452     private static final Map<String, String> bucketRegionCache =
453                  Collections.synchronizedMap(new LinkedHashMap<String, String>(BUCKET_REGION_CACHE_SIZE, 1.1f, true) {
454                      private static final long serialVersionUID = 23453L;
455
456                      @Override
457                      protected boolean removeEldestEntry(Map.Entry<String, String> eldest) {
458                          return size() > BUCKET_REGION_CACHE_SIZE;
459                      }
460                  });
461
462     static Map<String, String> getBucketRegionCache() {
463         return bucketRegionCache;
464     }
465
466     private final SkipMd5CheckStrategy skipMd5CheckStrategy;
467
468     private final CompleteMultipartUploadRetryCondition
469             completeMultipartUploadRetryCondition = new CompleteMultipartUploadRetryCondition();
470
471     /**
472      * Constructs a new client to invoke service methods on Amazon S3. A
473      * credentials provider chain will be used that searches for credentials in
474      * this order:
475      * <ul>
476      * <li>Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY</li>
477      * <li>Java System Properties - aws.accessKeyId and aws.secretKey</li>
478      * <li>Credential profiles file at the default location (~/.aws/credentials) shared by all AWS SDKs and the AWS CLI</li>
479      * <li>Instance Profile Credentials - delivered through the Amazon EC2
480      * metadata service</li>
481      * </ul>
482      *
483      * <p>
484      * If no credentials are found in the chain, this client will attempt to
485      * work in an anonymous mode where requests aren't signed. Only a subset of
486      * the Amazon S3 API will work with anonymous <i>(i.e. unsigned)</i> requests,
487      * but this can prove useful in some situations. For example:
488      * <ul>
489      * <li>If an Amazon S3 bucket has {@link Permission#Read} permission for the
490      * {@link GroupGrantee#AllUsers} group, anonymous clients can call
491      * {@link #listObjects(String)} to see what objects are stored in a bucket.</li>
492      * <li>If an object has {@link Permission#Read} permission for the
493      * {@link GroupGrantee#AllUsers} group, anonymous clients can call
494      * {@link #String, String)} and
495      * {@link #getObjectMetadata(String, String)} to pull object content and
496      * metadata.</li>
497      * <li>If a bucket has {@link Permission#Write} permission for the
498      * {@link GroupGrantee#AllUsers} group, anonymous clients can upload objects
499      * to the bucket.</li>
500      * </ul>
501      * </p>
502      * <p>
503      * You can force the client to operate in an anonymous mode, and skip the credentials
504      * provider chain, by passing in <code>null</code> for the credentials.
505      * </p>
506      *
507      * @see AmazonS3Client#AmazonS3Client(AWSCredentials)
508      * @see AmazonS3Client#AmazonS3Client(AWSCredentials, ClientConfiguration)
509      * @sample AmazonS3.CreateClient
510      * @deprecated use {@link AmazonS3ClientBuilder#defaultClient()}
511      */
512     @Deprecated
513     public AmazonS3Client() {
514         this(new S3CredentialsProviderChain());
515     }
516
517     /**
518      * Constructs a new Amazon S3 client using the specified AWS credentials to
519      * access Amazon S3.
520      *
521      * @param awsCredentials
522      *            The AWS credentials to use when making requests to Amazon S3
523      *            with this client.
524      *
525      * @see AmazonS3Client#AmazonS3Client()
526      * @see AmazonS3Client#AmazonS3Client(AWSCredentials, ClientConfiguration)
527      * @deprecated use {@link AmazonS3ClientBuilder#withCredentials(AWSCredentialsProvider)}
528      */
529     @Deprecated
530     public AmazonS3Client(AWSCredentials awsCredentials) {
531         this(awsCredentials, configFactory.getConfig());
532     }
533
534     /**
535      * Constructs a new Amazon S3 client using the specified AWS credentials and
536      * client configuration to access Amazon S3.
537      *
538      * @param awsCredentials
539      *            The AWS credentials to use when making requests to Amazon S3
540      *            with this client.
541      * @param clientConfiguration
542      *            The client configuration options controlling how this client
543      *            connects to Amazon S3 (e.g. proxy settings, retry counts, etc).
544      *
545      * @see AmazonS3Client#AmazonS3Client()
546      * @see AmazonS3Client#AmazonS3Client(AWSCredentials)
547      * @deprecated use {@link AmazonS3ClientBuilder#withCredentials(AWSCredentialsProvider)} and
548      *             {@link AmazonS3ClientBuilder#withClientConfiguration(ClientConfiguration)}
549      */
550     @Deprecated
551     public AmazonS3Client(AWSCredentials awsCredentials, ClientConfiguration clientConfiguration) {
552         this(new StaticCredentialsProvider(awsCredentials), clientConfiguration);
553     }
554
555     /**
556      * Constructs a new Amazon S3 client using the specified AWS credentials
557      * provider to access Amazon S3.
558      *
559      * @param credentialsProvider
560      *            The AWS credentials provider which will provide credentials
561      *            to authenticate requests with AWS services.
562      * @deprecated use {@link AmazonS3ClientBuilder#withCredentials(AWSCredentialsProvider)}
563      */
564     @Deprecated
565     public AmazonS3Client(AWSCredentialsProvider credentialsProvider) {
566         this(credentialsProvider, configFactory.getConfig());
567     }
568
569     /**
570      * Constructs a new Amazon S3 client using the specified AWS credentials and
571      * client configuration to access Amazon S3.
572      *
573      * @param credentialsProvider
574      *            The AWS credentials provider which will provide credentials
575      *            to authenticate requests with AWS services.
576      * @param clientConfiguration
577      *            The client configuration options controlling how this client
578      *            connects to Amazon S3 (e.g. proxy settings, retry counts, etc).
579      * @deprecated use {@link AmazonS3ClientBuilder#withCredentials(AWSCredentialsProvider)} and
580      *             {@link AmazonS3ClientBuilder#withClientConfiguration(ClientConfiguration)}
581      */
582     @Deprecated
583     public AmazonS3Client(AWSCredentialsProvider credentialsProvider,
584             ClientConfiguration clientConfiguration) {
585         this(credentialsProvider, clientConfiguration, null);
586     }
587
588     /**
589      * Constructs a new Amazon S3 client using the specified AWS credentials,
590      * client configuration and request metric collector to access Amazon S3.
591      *
592      * @param credentialsProvider
593      *            The AWS credentials provider which will provide credentials
594      *            to authenticate requests with AWS services.
595      * @param clientConfiguration
596      *            The client configuration options controlling how this client
597      *            connects to Amazon S3 (e.g. proxy settings, retry counts, etc).
598      * @param requestMetricCollector request metric collector
599      * @deprecated use {@link AmazonS3ClientBuilder#withCredentials(AWSCredentialsProvider)} and
600      *             {@link AmazonS3ClientBuilder#withClientConfiguration(ClientConfiguration)} and
601      *             {@link AmazonS3ClientBuilder#withMetricsCollector(RequestMetricCollector)}
602      */
603     @Deprecated
604     public AmazonS3Client(AWSCredentialsProvider credentialsProvider,
605             ClientConfiguration clientConfiguration,
606             RequestMetricCollector requestMetricCollector) {
607         this(credentialsProvider, clientConfiguration, requestMetricCollector, SkipMd5CheckStrategy.INSTANCE);
608     }
609
610     /**
611      * Constructs a new Amazon S3 client using the specified AWS credentials,
612      * client configuration and request metric collector to access Amazon S3.
613      *
614      * @param credentialsProvider
615      *            The AWS credentials provider which will provide credentials
616      *            to authenticate requests with AWS services.
617      * @param clientConfiguration
618      *            The client configuration options controlling how this client
619      *            connects to Amazon S3 (e.g. proxy settings, retry counts, etc).
620      * @param requestMetricCollector request metric collector
621      */
622     @SdkTestInternalApi
623     AmazonS3Client(AWSCredentialsProvider credentialsProvider,
624             ClientConfiguration clientConfiguration,
625             RequestMetricCollector requestMetricCollector,
626             SkipMd5CheckStrategy skipMd5CheckStrategy) {
627         super(clientConfiguration, requestMetricCollector, true);
628         this.awsCredentialsProvider = credentialsProvider;
629         this.skipMd5CheckStrategy = skipMd5CheckStrategy;
630         this.errorResponseHandler = new S3ErrorResponseHandler(clientConfiguration);
631         init();
632     }
633
634     /**
635      * Constructs a new client using the specified client configuration to
636      * access Amazon S3. A credentials provider chain will be used that searches
637      * for credentials in this order:
638      * <ul>
639      * <li>Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY</li>
640      * <li>Java System Properties - aws.accessKeyId and aws.secretKey</li>
641      * <li>Instance Profile Credentials - delivered through the Amazon EC2
642      * metadata service</li>
643      * </ul>
644      *
645      * <p>
646      * If no credentials are found in the chain, this client will attempt to
647      * work in an anonymous mode where requests aren't signed. Only a subset of
648      * the Amazon S3 API will work with anonymous <i>(i.e. unsigned)</i>
649      * requests, but this can prove useful in some situations. For example:
650      * <ul>
651      * <li>If an Amazon S3 bucket has {@link Permission#Read} permission for the
652      * {@link GroupGrantee#AllUsers} group, anonymous clients can call
653      * {@link #listObjects(String)} to see what objects are stored in a bucket.</li>
654      * <li>If an object has {@link Permission#Read} permission for the
655      * {@link GroupGrantee#AllUsers} group, anonymous clients can call
656      * {@link #getObject(String, String)} and
657      * {@link #getObjectMetadata(String, String)} to pull object content and
658      * metadata.</li>
659      * <li>If a bucket has {@link Permission#Write} permission for the
660      * {@link GroupGrantee#AllUsers} group, anonymous clients can upload objects
661      * to the bucket.</li>
662      * </ul>
663      * </p>
664      * <p>
665      * You can force the client to operate in an anonymous mode, and skip the
666      * credentials provider chain, by passing in <code>null</code> for the
667      * credentials.
668      * </p>
669      *
670      * @param clientConfiguration
671      *            The client configuration options controlling how this client
672      *            connects to Amazon S3 (e.g. proxy settings, retry counts, etc).
673      *
674      * @see AmazonS3Client#AmazonS3Client(AWSCredentials)
675      * @see AmazonS3Client#AmazonS3Client(AWSCredentials, ClientConfiguration)
676      * @deprecated use {@link AmazonS3ClientBuilder#withClientConfiguration(ClientConfiguration)}
677      */
678     @Deprecated
679     public AmazonS3Client(ClientConfiguration clientConfiguration) {
680         this(new S3CredentialsProviderChain(), clientConfiguration);
681     }
682
683     /**
684      * Constructs a new client to invoke service methods on S3 using the specified parameters. All
685      * service calls made using this new client object are blocking, and will not return until the
686      * service call completes.
687      *
688      * @param s3ClientParams Object providing S3 client parameters.
689      * @see AmazonS3ClientBuilder For a fluent way to construct a client.
690      */
691     @SdkInternalApi
692     AmazonS3Client(AmazonS3ClientParams s3ClientParams) {
693         super(s3ClientParams.getClientParams());
694         this.awsCredentialsProvider = s3ClientParams.getClientParams().getCredentialsProvider();
695         this.skipMd5CheckStrategy = SkipMd5CheckStrategy.INSTANCE;
696         setS3ClientOptions(s3ClientParams.getS3ClientOptions());
697         this.errorResponseHandler = new S3ErrorResponseHandler(
698             s3ClientParams.getClientParams().getClientConfiguration());
699         init();
700     }
701
702     public static AmazonS3ClientBuilder builder() {
703         return AmazonS3ClientBuilder.standard();
704     }
705
706     private void init() {
707
708         // calling this.setEndpoint(...) will also modify the signer accordingly
709         setEndpoint(Constants.S3_HOSTNAME);
710
711         HandlerChainFactory chainFactory = new HandlerChainFactory();
712         requestHandler2s.addAll(chainFactory.newRequestHandlerChain(
713                 "/com/amazonaws/services/s3/request.handlers"));
714         requestHandler2s.addAll(chainFactory.newRequestHandler2Chain(
715                 "/com/amazonaws/services/s3/request.handler2s"));
716         requestHandler2s.addAll(chainFactory.getGlobalHandlers());
717     }
718
719     /**
720      * @deprecated use {@link AmazonS3ClientBuilder#setEndpointConfiguration(AwsClientBuilder.EndpointConfiguration)}
721      */
722     @Override
723     @Deprecated
724     public synchronized void setEndpoint(String endpoint) {
725         if (ServiceUtils.isS3AccelerateEndpoint(endpoint)) {
726             throw new IllegalStateException("To enable accelerate mode, please use AmazonS3ClientBuilder.withAccelerateModeEnabled(true)");
727         } else {
728             super.setEndpoint(endpoint);
729             /*
730              * Extract the region string from the endpoint if it's not known to be a
731              * global S3 endpoint.
732              */
733             if (!ServiceUtils.isS3USStandardEndpoint(endpoint)) {
734                 clientRegion = AwsHostNameUtils.parseRegionName(this.endpoint.getHost(), S3_SERVICE_NAME);
735             }
736         }
737     }
738
739     /**
740      * @deprecated use {@link AmazonS3ClientBuilder#setRegion(String)}
741      */
742     @Override
743     @Deprecated
744     public synchronized void setRegion(com.amazonaws.regions.Region region) {
745         if (region.getName().equalsIgnoreCase("us-east-1")) {
746             if (clientOptions.isRegionalUsEast1EndpointEnabled() || REGIONAL_ENDPOINTS_OPTION_RESOLVER.useRegionalMode()) {
747                 region = RegionUtils.getRegion("us-east-1-regional");
748             }
749         }
750
751         super.setRegion(region);
752         /*
753          * We need to preserve the user provided region. This is because the
754          * region might be mapped to a global s3 endpoint (e.g. when the client
755          * is in accelerate mode), in which case we won't be able to extract the
756          * region back from the endpoint during request signing phase.
757          */
758         clientRegion = region.getName();
759     }
760
761     /**
762      * <p>
763      * Override the default S3 client options for this client. Also set the
764      * endpoint to s3-accelerate if such is specified in the S3 client options.
765      * </p>
766      *
767      * @param clientOptions
768      *            The S3 client options to use.
769      */
770     @Override
771     public synchronized void setS3ClientOptions(S3ClientOptions clientOptions) {
772         checkMutability();
773         this.clientOptions = new S3ClientOptions(clientOptions);
774     }
775
776     /**
777      * S3 uses wildcard certificates so we have to disable strict hostname verification when using
778      * SSL.
779      */
780     @Override
781     protected boolean useStrictHostNameVerification() {
782         return false;
783     }
784
785     @Override
786     public VersionListing listNextBatchOfVersions(VersionListing previousVersionListing)
787             throws SdkClientException, AmazonServiceException {
788         return listNextBatchOfVersions(new ListNextBatchOfVersionsRequest(previousVersionListing));
789     }
790
791     @Override
792     public VersionListing listNextBatchOfVersions(ListNextBatchOfVersionsRequest listNextBatchOfVersionsRequest) {
793         listNextBatchOfVersionsRequest = beforeClientExecution(listNextBatchOfVersionsRequest);
794         rejectNull(listNextBatchOfVersionsRequest,
795                 "The request object parameter must be specified when listing the next batch of versions in a bucket");
796         VersionListing previousVersionListing = listNextBatchOfVersionsRequest.getPreviousVersionListing();
797
798         if (!previousVersionListing.isTruncated()) {
799             VersionListing emptyListing = new VersionListing();
800             emptyListing.setBucketName(previousVersionListing.getBucketName());
801             emptyListing.setDelimiter(previousVersionListing.getDelimiter());
802             emptyListing.setKeyMarker(previousVersionListing.getNextKeyMarker());
803             emptyListing.setVersionIdMarker(previousVersionListing.getNextVersionIdMarker());
804             emptyListing.setMaxKeys(previousVersionListing.getMaxKeys());
805             emptyListing.setPrefix(previousVersionListing.getPrefix());
806             emptyListing.setEncodingType(previousVersionListing.getEncodingType());
807             emptyListing.setTruncated(false);
808
809             return emptyListing;
810         }
811
812         return listVersions(listNextBatchOfVersionsRequest.toListVersionsRequest());
813     }
814
815     @Override
816     public VersionListing listVersions(String bucketName, String prefix)
817             throws SdkClientException, AmazonServiceException {
818         return listVersions(new ListVersionsRequest(bucketName, prefix, null, null, null, null));
819     }
820
821     @Override
822     public VersionListing listVersions(String bucketName, String prefix, String keyMarker, String versionIdMarker, String delimiter, Integer maxKeys)
823             throws SdkClientException, AmazonServiceException {
824
825         ListVersionsRequest request = new ListVersionsRequest()
826             .withBucketName(bucketName)
827             .withPrefix(prefix)
828             .withDelimiter(delimiter)
829             .withKeyMarker(keyMarker)
830             .withVersionIdMarker(versionIdMarker)
831             .withMaxResults(maxKeys);
832         return listVersions(request);
833     }
834
835     @Override
836     public VersionListing listVersions(ListVersionsRequest listVersionsRequest)
837             throws SdkClientException, AmazonServiceException {
838         listVersionsRequest = beforeClientExecution(listVersionsRequest);
839         rejectNull(listVersionsRequest.getBucketName(), "The bucket name parameter must be specified when listing versions in a bucket");
840
841         /**
842          * This flag shows whether we need to url decode S3 key names. This flag is enabled
843          * only when the customers don't explicitly call {@link listVersionsRequest#setEncodingType(String)},
844          * otherwise, it will be disabled for maintaining backwards compatibility.
845          */
846         final boolean shouldSDKDecodeResponse = listVersionsRequest.getEncodingType() == null;
847
848         Request<ListVersionsRequest> request = createRequest(listVersionsRequest.getBucketName(), null, listVersionsRequest, HttpMethodName.GET);
849         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "ListObjectVersions");
850         request.addParameter("versions", null);
851
852         addParameterIfNotNull(request, "prefix", listVersionsRequest.getPrefix());
853         addParameterIfNotNull(request, "key-marker", listVersionsRequest.getKeyMarker());
854         addParameterIfNotNull(request, "version-id-marker", listVersionsRequest.getVersionIdMarker());
855         addParameterIfNotNull(request, "delimiter", listVersionsRequest.getDelimiter());
856
857         if (listVersionsRequest.getMaxResults() != null && listVersionsRequest.getMaxResults() >= 0) request.addParameter("max-keys", listVersionsRequest.getMaxResults().toString());
858         request.addParameter("encoding-type", shouldSDKDecodeResponse ? Constants.URL_ENCODING : listVersionsRequest.getEncodingType());
859
860         return invoke(request, new Unmarshallers.VersionListUnmarshaller(shouldSDKDecodeResponse), listVersionsRequest.getBucketName(), null);
861     }
862
863     @Override
864     public ObjectListing listObjects(String bucketName)
865             throws SdkClientException, AmazonServiceException {
866         return listObjects(new ListObjectsRequest(bucketName, null, null, null, null));
867     }
868
869     @Override
870     public ObjectListing listObjects(String bucketName, String prefix)
871             throws SdkClientException, AmazonServiceException {
872         return listObjects(new ListObjectsRequest(bucketName, prefix, null, null, null));
873     }
874
875     @Override
876     public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
877             throws SdkClientException, AmazonServiceException {
878         listObjectsRequest = beforeClientExecution(listObjectsRequest);
879         rejectNull(listObjectsRequest.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket");
880
881         /**
882          * This flag shows whether we need to url decode S3 key names. This flag is enabled
883          * only when the customers don't explicitly call {@link ListObjectsRequest#setEncodingType(String)},
884          * otherwise, it will be disabled for maintaining backwards compatibility.
885          */
886         final boolean shouldSDKDecodeResponse = listObjectsRequest.getEncodingType() == null;
887
888         Request<ListObjectsRequest> request = createRequest(listObjectsRequest.getBucketName(), null, listObjectsRequest, HttpMethodName.GET);
889         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "ListObjects");
890         addParameterIfNotNull(request, "prefix", listObjectsRequest.getPrefix());
891         addParameterIfNotNull(request, "marker", listObjectsRequest.getMarker());
892         addParameterIfNotNull(request, "delimiter", listObjectsRequest.getDelimiter());
893         if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) request.addParameter("max-keys", listObjectsRequest.getMaxKeys().toString());
894         request.addParameter("encoding-type", shouldSDKDecodeResponse ? Constants.URL_ENCODING : listObjectsRequest.getEncodingType());
895
896         populateRequesterPaysHeader(request, listObjectsRequest.isRequesterPays());
897
898         return invoke(request, new Unmarshallers.ListObjectsUnmarshaller(shouldSDKDecodeResponse), listObjectsRequest.getBucketName(), null);
899     }
900
901     @Override
902     public ListObjectsV2Result listObjectsV2(String bucketName)
903             throws SdkClientException, AmazonServiceException {
904         return listObjectsV2(new ListObjectsV2Request().withBucketName(bucketName));
905     }
906
907     @Override
908     public ListObjectsV2Result listObjectsV2(String bucketName, String prefix)
909             throws SdkClientException, AmazonServiceException {
910         return listObjectsV2(new ListObjectsV2Request().withBucketName(bucketName).withPrefix(prefix));
911     }
912
913     @Override
914     public ListObjectsV2Result listObjectsV2(ListObjectsV2Request listObjectsV2Request)
915             throws SdkClientException, AmazonServiceException {
916         listObjectsV2Request = beforeClientExecution(listObjectsV2Request);
917         rejectNull(listObjectsV2Request.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket");
918         Request<ListObjectsV2Request> request = createRequest(listObjectsV2Request.getBucketName(), null, listObjectsV2Request, HttpMethodName.GET);
919         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "ListObjectsV2");
920
921         /**
922          * List type '2' is required to opt-in to listObjectsV2.
923          */
924         request.addParameter("list-type", "2");
925
926         addParameterIfNotNull(request, "start-after", listObjectsV2Request.getStartAfter());
927         addParameterIfNotNull(request, "continuation-token", listObjectsV2Request.getContinuationToken());
928         addParameterIfNotNull(request, "delimiter", listObjectsV2Request.getDelimiter());
929         addParameterIfNotNull(request, "max-keys", listObjectsV2Request.getMaxKeys());
930         addParameterIfNotNull(request, "prefix", listObjectsV2Request.getPrefix());
931         addParameterIfNotNull(request, "encoding-type", listObjectsV2Request.getEncodingType());
932         request.addParameter("fetch-owner", Boolean.toString(listObjectsV2Request.isFetchOwner()));
933
934         populateRequesterPaysHeader(request, listObjectsV2Request.isRequesterPays());
935
936         /**
937          * If URL encoding has been requested from S3 we'll automatically decode the response.
938          */
939         final boolean shouldSDKDecodeResponse = Constants.URL_ENCODING.equals(listObjectsV2Request.getEncodingType());
940
941         return invoke(request, new Unmarshallers.ListObjectsV2Unmarshaller(shouldSDKDecodeResponse), listObjectsV2Request.getBucketName(), null);
942     }
943
944     @Override
945     public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing)
946             throws SdkClientException, AmazonServiceException {
947         return listNextBatchOfObjects(new ListNextBatchOfObjectsRequest(previousObjectListing));
948     }
949
950     @Override
951     public ObjectListing listNextBatchOfObjects(ListNextBatchOfObjectsRequest listNextBatchOfObjectsRequest)
952             throws SdkClientException, AmazonServiceException {
953         listNextBatchOfObjectsRequest = beforeClientExecution(listNextBatchOfObjectsRequest);
954         rejectNull(listNextBatchOfObjectsRequest,
955                 "The request object parameter must be specified when listing the next batch of objects in a bucket");
956         ObjectListing previousObjectListing = listNextBatchOfObjectsRequest.getPreviousObjectListing();
957
958         if (!previousObjectListing.isTruncated()) {
959             ObjectListing emptyListing = new ObjectListing();
960             emptyListing.setBucketName(previousObjectListing.getBucketName());
961             emptyListing.setDelimiter(previousObjectListing.getDelimiter());
962             emptyListing.setMarker(previousObjectListing.getNextMarker());
963             emptyListing.setMaxKeys(previousObjectListing.getMaxKeys());
964             emptyListing.setPrefix(previousObjectListing.getPrefix());
965             emptyListing.setEncodingType(previousObjectListing.getEncodingType());
966             emptyListing.setTruncated(false);
967
968             return emptyListing;
969         }
970         return listObjects(listNextBatchOfObjectsRequest.toListObjectsRequest());
971     }
972
973     @Override
974     public Owner getS3AccountOwner()
975             throws SdkClientException, AmazonServiceException {
976         return getS3AccountOwner(new GetS3AccountOwnerRequest());
977     }
978
979     @Override
980     public Owner getS3AccountOwner(GetS3AccountOwnerRequest getS3AccountOwnerRequest)
981             throws SdkClientException, AmazonServiceException {
982         getS3AccountOwnerRequest = beforeClientExecution(getS3AccountOwnerRequest);
983         rejectNull(getS3AccountOwnerRequest, "The request object parameter getS3AccountOwnerRequest must be specified.");
984         Request<GetS3AccountOwnerRequest> request = createRequest(null, null, getS3AccountOwnerRequest, HttpMethodName.GET);
985         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "ListBuckets");
986         return invoke(request, new Unmarshallers.ListBucketsOwnerUnmarshaller(), null, null);
987     }
988
989     @Override
990     public List<Bucket> listBuckets(ListBucketsRequest listBucketsRequest)
991             throws SdkClientException, AmazonServiceException {
992         listBucketsRequest = beforeClientExecution(listBucketsRequest);
993         rejectNull(listBucketsRequest, "The request object parameter listBucketsRequest must be specified.");
994         Request<ListBucketsRequest> request = createRequest(null, null, listBucketsRequest, HttpMethodName.GET);
995         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "ListBuckets");
996         return invoke(request, new Unmarshallers.ListBucketsUnmarshaller(), null, null);
997     }
998
999     @Override
1000     public List<Bucket> listBuckets()
1001             throws SdkClientException, AmazonServiceException {
1002         return listBuckets(new ListBucketsRequest());
1003     }
1004
1005     @Override
1006     public String getBucketLocation(GetBucketLocationRequest getBucketLocationRequest)
1007             throws SdkClientException, AmazonServiceException {
1008         getBucketLocationRequest = beforeClientExecution(getBucketLocationRequest);
1009         rejectNull(getBucketLocationRequest, "The request parameter must be specified when requesting a bucket's location");
1010         String bucketName = getBucketLocationRequest.getBucketName();
1011         rejectNull(bucketName, "The bucket name parameter must be specified when requesting a bucket's location");
1012
1013         Request<GetBucketLocationRequest> request = createRequest(bucketName, null, getBucketLocationRequest, HttpMethodName.GET);
1014         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketLocation");
1015         request.addParameter("location", null);
1016
1017         return invoke(request, new Unmarshallers.BucketLocationUnmarshaller(), bucketName, null);
1018     }
1019
1020     @Override
1021     public String getBucketLocation(String bucketName)
1022             throws SdkClientException, AmazonServiceException {
1023         return getBucketLocation(new GetBucketLocationRequest(bucketName));
1024     }
1025
1026     @Override
1027     public Bucket createBucket(String bucketName)
1028             throws SdkClientException, AmazonServiceException {
1029         return createBucket(new CreateBucketRequest(bucketName));
1030     }
1031
1032     @Override
1033     @Deprecated
1034     public Bucket createBucket(String bucketName, Region region)
1035             throws SdkClientException, AmazonServiceException {
1036         return createBucket(new CreateBucketRequest(bucketName, region));
1037     }
1038
1039     @Override
1040     @Deprecated
1041     public Bucket createBucket(String bucketName, String region)
1042             throws SdkClientException, AmazonServiceException {
1043         return createBucket(new CreateBucketRequest(bucketName, region));
1044     }
1045
1046     @Override
1047     public Bucket createBucket(CreateBucketRequest createBucketRequest)
1048             throws SdkClientException, AmazonServiceException {
1049         createBucketRequest = beforeClientExecution(createBucketRequest);
1050         rejectNull(createBucketRequest,
1051                    "The CreateBucketRequest parameter must be specified when creating a bucket");
1052
1053         String bucketName = createBucketRequest.getBucketName();
1054         rejectNull(bucketName, "The bucket name parameter must be specified when creating a bucket");
1055         bucketName = bucketName.trim();
1056
1057         String requestRegion = createBucketRequest.getRegion();
1058         URI requestEndpoint = getCreateBucketEndpoint(requestRegion);
1059
1060         BucketNameUtils.validateBucketName(bucketName);
1061
1062         Request<CreateBucketRequest> request = createRequest(bucketName, null, createBucketRequest, HttpMethodName.PUT, requestEndpoint);
1063         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "CreateBucket");
1064
1065         if (createBucketRequest.getAccessControlList() != null) {
1066             addAclHeaders(request, createBucketRequest.getAccessControlList());
1067         } else if (createBucketRequest.getCannedAcl() != null) {
1068             request.addHeader(Headers.S3_CANNED_ACL, createBucketRequest.getCannedAcl().toString());
1069         }
1070
1071         /*
1072          * If we're talking to a region-specific endpoint other than the US, we
1073          * *must* specify a location constraint. Try to derive the region from
1074          * the endpoint.
1075          */
1076         if (getSignerRegion() != null && !getSignerRegion().equals("us-east-1") && StringUtils.isNullOrEmpty(requestRegion)) {
1077             requestRegion = AwsHostNameUtils.parseRegion(requestEndpoint.getHost(), S3_SERVICE_NAME);
1078         }
1079
1080         /*
1081          * We can only send the CreateBucketConfiguration if we're *not*
1082          * creating a bucket in the US region.
1083          */
1084         if (requestRegion != null && !StringUtils.upperCase(requestRegion).equals(Region.US_Standard.toString())) {
1085             XmlWriter xml = new XmlWriter();
1086             xml.start("CreateBucketConfiguration", "xmlns", Constants.XML_NAMESPACE);
1087             xml.start("LocationConstraint").value(requestRegion).end();
1088             xml.end();
1089
1090             request.setContent(new ByteArrayInputStream(xml.getBytes()));
1091         }
1092
1093         if (createBucketRequest.getObjectLockEnabledForBucket()) {
1094             request.addHeader(Headers.OBJECT_LOCK_ENABLED_FOR_BUCKET, "true");
1095         }
1096
1097         invoke(request, voidResponseHandler, bucketName, null);
1098
1099         return new Bucket(bucketName);
1100     }
1101
1102     private URI getCreateBucketEndpoint(String requestRegion) {
1103         // Route to the default endpoint if they're not trying to specify a different one in the request.
1104         if(requestRegion == null || requestRegion.equals(clientRegion) || !clientOptions.isForceGlobalBucketAccessEnabled()) {
1105             return endpoint;
1106         }
1107
1108         // If they enabled global bucket access and they're trying to create a bucket in a region different than the default
1109         // one specified when they created the client, it will probably fail because only us-east-1 (actually the global
1110         // endpoint) is capable of creating buckets outside of its region. Override the endpoint to which the request
1111         // is routed so that it will succeed.
1112
1113         com.amazonaws.regions.Region targetRegion = com.amazonaws.regions.Region.getRegion(Regions.fromName(requestRegion));
1114         return new DefaultServiceEndpointBuilder(getEndpointPrefix(),
1115                                                  clientConfiguration.getProtocol().toString()).withRegion(targetRegion)
1116                                                                                               .getServiceEndpoint();
1117     }
1118
1119     @Override
1120     public AccessControlList getObjectAcl(String bucketName, String key)
1121             throws SdkClientException, AmazonServiceException {
1122         return getObjectAcl(new GetObjectAclRequest(bucketName, key));
1123     }
1124
1125     @Override
1126     public AccessControlList getObjectAcl(String bucketName, String key, String versionId)
1127             throws SdkClientException, AmazonServiceException {
1128         return getObjectAcl(new GetObjectAclRequest(bucketName, key, versionId));
1129     }
1130
1131     @Override
1132     public AccessControlList getObjectAcl(GetObjectAclRequest getObjectAclRequest) {
1133         getObjectAclRequest = beforeClientExecution(getObjectAclRequest);
1134         rejectNull(getObjectAclRequest, "The request parameter must be specified when requesting an object's ACL");
1135         rejectNull(getObjectAclRequest.getBucketName(), "The bucket name parameter must be specified when requesting an object's ACL");
1136         rejectNull(getObjectAclRequest.getKey(), "The key parameter must be specified when requesting an object's ACL");
1137
1138         return getAcl(getObjectAclRequest.getBucketName(), getObjectAclRequest.getKey(),
1139                 getObjectAclRequest.getVersionId(), getObjectAclRequest.isRequesterPays(),
1140                 getObjectAclRequest);
1141     }
1142
1143     @Override
1144     public void setObjectAcl(String bucketName, String key, AccessControlList acl)
1145             throws SdkClientException, AmazonServiceException {
1146         setObjectAcl(bucketName, key, null, acl);
1147     }
1148
1149     @Override
1150     public void setObjectAcl(String bucketName, String key, CannedAccessControlList acl)
1151             throws SdkClientException, AmazonServiceException {
1152         setObjectAcl(bucketName, key, null, acl);
1153     }
1154
1155     @Override
1156     public void setObjectAcl(String bucketName, String key, String versionId, AccessControlList acl)
1157             throws SdkClientException, AmazonServiceException {
1158         setObjectAcl(new SetObjectAclRequest(bucketName, key, versionId, acl));
1159     }
1160
1161     /**
1162      * Same as {@link #setObjectAcl(String, String, String, AccessControlList)}
1163      * but allows specifying a request metric collector.
1164      */
1165     public void setObjectAcl(String bucketName, String key, String versionId,
1166             AccessControlList acl, RequestMetricCollector requestMetricCollector)
1167             throws SdkClientException, AmazonServiceException {
1168         setObjectAcl(new SetObjectAclRequest(bucketName, key, versionId, acl)
1169                 .<SetObjectAclRequest> withRequestMetricCollector(requestMetricCollector));
1170     }
1171
1172     @Override
1173     public void setObjectAcl(String bucketName, String key, String versionId, CannedAccessControlList acl)
1174             throws SdkClientException, AmazonServiceException {
1175         setObjectAcl(new SetObjectAclRequest(bucketName, key, versionId, acl));
1176     }
1177
1178     /**
1179      * Same as {@link #setObjectAcl(String, String, String, CannedAccessControlList)}
1180      * but allows specifying a request metric collector.
1181      */
1182     public void setObjectAcl(String bucketName, String key, String versionId,
1183             CannedAccessControlList acl,
1184             RequestMetricCollector requestMetricCollector) {
1185         setObjectAcl(new SetObjectAclRequest(bucketName, key, versionId, acl)
1186                 .<SetObjectAclRequest> withRequestMetricCollector(requestMetricCollector));
1187     }
1188
1189     @Override
1190     public void setObjectAcl(SetObjectAclRequest setObjectAclRequest)
1191             throws SdkClientException, AmazonServiceException {
1192         setObjectAclRequest = beforeClientExecution(setObjectAclRequest);
1193         rejectNull(setObjectAclRequest,
1194                 "The request must not be null.");
1195         rejectNull(setObjectAclRequest.getBucketName(),
1196                 "The bucket name parameter must be specified when setting an object's ACL");
1197         rejectNull(setObjectAclRequest.getKey(),
1198                 "The key parameter must be specified when setting an object's ACL");
1199
1200         if (setObjectAclRequest.getAcl() != null && setObjectAclRequest.getCannedAcl() != null) {
1201             throw new IllegalArgumentException(
1202                     "Only one of the ACL and CannedACL parameters can be specified, not both.");
1203         }
1204
1205         if (setObjectAclRequest.getAcl() != null) {
1206             setAcl(setObjectAclRequest.getBucketName(),
1207                    setObjectAclRequest.getKey(),
1208                    setObjectAclRequest.getVersionId(),
1209                    setObjectAclRequest.getAcl(),
1210                    setObjectAclRequest.isRequesterPays(),
1211                    setObjectAclRequest);
1212
1213         } else if (setObjectAclRequest.getCannedAcl() != null) {
1214             setAcl(setObjectAclRequest.getBucketName(),
1215                    setObjectAclRequest.getKey(),
1216                    setObjectAclRequest.getVersionId(),
1217                    setObjectAclRequest.getCannedAcl(),
1218                    setObjectAclRequest.isRequesterPays(),
1219                    setObjectAclRequest);
1220
1221         } else {
1222             throw new IllegalArgumentException(
1223                     "At least one of the ACL and CannedACL parameters should be specified");
1224         }
1225     }
1226
1227     /**
1228      * {@inheritDoc}
1229      * @see #getBucketAcl(String)
1230      */
1231     @Override
1232     public AccessControlList getBucketAcl(String bucketName)
1233             throws SdkClientException, AmazonServiceException {
1234         return getBucketAcl(new GetBucketAclRequest(bucketName));
1235     }
1236
1237     @Override
1238     public AccessControlList getBucketAcl(GetBucketAclRequest getBucketAclRequest)
1239         throws SdkClientException, AmazonServiceException {
1240         getBucketAclRequest = beforeClientExecution(getBucketAclRequest);
1241         String bucketName = getBucketAclRequest.getBucketName();
1242         rejectNull(bucketName, "The bucket name parameter must be specified when requesting a bucket's ACL");
1243
1244         return getAcl(bucketName, null, null, false, getBucketAclRequest);
1245     }
1246
1247     @Override
1248     public void setBucketAcl(String bucketName, AccessControlList acl)
1249             throws SdkClientException, AmazonServiceException {
1250         setBucketAcl(new SetBucketAclRequest(bucketName, acl));
1251     }
1252
1253     /**
1254      * Same as {@link #setBucketAcl(String, AccessControlList)}
1255      * but allows specifying a request metric collector.
1256      */
1257     public void setBucketAcl(String bucketName, AccessControlList acl,
1258             RequestMetricCollector requestMetricCollector) {
1259         SetBucketAclRequest request = new SetBucketAclRequest(bucketName, acl)
1260             .withRequestMetricCollector(requestMetricCollector);
1261         setBucketAcl(request);
1262     }
1263
1264     @Override
1265     public void setBucketAcl(String bucketName, CannedAccessControlList cannedAcl)
1266             throws SdkClientException, AmazonServiceException {
1267         setBucketAcl(new SetBucketAclRequest(bucketName, cannedAcl));
1268     }
1269
1270     /**
1271      * Same as {@link #setBucketAcl(String, CannedAccessControlList)}
1272      * but allows specifying a request metric collector.
1273      */
1274     public void setBucketAcl(String bucketName, CannedAccessControlList cannedAcl,
1275             RequestMetricCollector requestMetricCollector) throws SdkClientException,
1276             AmazonServiceException {
1277         SetBucketAclRequest request = new SetBucketAclRequest(bucketName, cannedAcl)
1278             .withRequestMetricCollector(requestMetricCollector);
1279         setBucketAcl(request);
1280     }
1281
1282     @Override
1283     public void setBucketAcl(SetBucketAclRequest setBucketAclRequest)
1284             throws SdkClientException, AmazonServiceException {
1285         setBucketAclRequest = beforeClientExecution(setBucketAclRequest);
1286
1287         String bucketName = setBucketAclRequest.getBucketName();
1288         rejectNull(bucketName, "The bucket name parameter must be specified when setting a bucket's ACL");
1289
1290         AccessControlList acl = setBucketAclRequest.getAcl();
1291         CannedAccessControlList cannedAcl = setBucketAclRequest.getCannedAcl();
1292
1293         if (acl == null && cannedAcl == null) {
1294             throw new IllegalArgumentException(
1295                     "The ACL parameter must be specified when setting a bucket's ACL");
1296         }
1297         if (acl != null && cannedAcl != null) {
1298             throw new IllegalArgumentException(
1299                     "Only one of the acl and cannedAcl parameter can be specified, not both.");
1300         }
1301
1302         if (acl != null) {
1303             setAcl(bucketName, null, null, acl, false, setBucketAclRequest);
1304         } else {
1305             setAcl(bucketName, null, null, cannedAcl, false, setBucketAclRequest);
1306         }
1307     }
1308
1309     @Override
1310     public ObjectMetadata getObjectMetadata(String bucketName, String key)
1311             throws SdkClientException, AmazonServiceException {
1312         return getObjectMetadata(new GetObjectMetadataRequest(bucketName, key));
1313     }
1314
1315     @Override
1316     public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetadataRequest)
1317             throws SdkClientException, AmazonServiceException {
1318         getObjectMetadataRequest = beforeClientExecution(getObjectMetadataRequest);
1319         rejectNull(getObjectMetadataRequest, "The GetObjectMetadataRequest parameter must be specified when requesting an object's metadata");
1320
1321         String bucketName = getObjectMetadataRequest.getBucketName();
1322         String key = getObjectMetadataRequest.getKey();
1323         String versionId = getObjectMetadataRequest.getVersionId();
1324
1325         rejectNull(bucketName, "The bucket name parameter must be specified when requesting an object's metadata");
1326         rejectNull(key, "The key parameter must be specified when requesting an object's metadata");
1327
1328         Request<GetObjectMetadataRequest> request = createRequest(bucketName, key, getObjectMetadataRequest, HttpMethodName.HEAD);
1329         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "HeadObject");
1330
1331         if (versionId != null) request.addParameter("versionId", versionId);
1332
1333         populateRequesterPaysHeader(request, getObjectMetadataRequest.isRequesterPays());
1334         addPartNumberIfNotNull(request, getObjectMetadataRequest.getPartNumber());
1335
1336         populateSSE_C(request, getObjectMetadataRequest.getSSECustomerKey());
1337
1338         return invoke(request, new S3MetadataResponseHandler(), bucketName, key);
1339     }
1340
1341     @Override
1342     public S3Object getObject(String bucketName, String key)
1343             throws SdkClientException, AmazonServiceException {
1344         return getObject(new GetObjectRequest(bucketName, key));
1345     }
1346
1347     @Override
1348     public boolean doesBucketExist(String bucketName)
1349             throws SdkClientException, AmazonServiceException {
1350         try {
1351             ValidationUtils.assertStringNotEmpty(bucketName, "bucketName");
1352             headBucket(new HeadBucketRequest(bucketName));
1353             return true;
1354         } catch (AmazonServiceException ase) {
1355             // A redirect error or a forbidden error means the bucket exists. So
1356             // returning true.
1357             if ((ase.getStatusCode() == Constants.BUCKET_REDIRECT_STATUS_CODE)
1358                 || (ase.getStatusCode() == Constants.BUCKET_ACCESS_FORBIDDEN_STATUS_CODE)) {
1359                 return true;
1360             }
1361             if (ase.getStatusCode() == Constants.NO_SUCH_BUCKET_STATUS_CODE) {
1362                 return false;
1363             }
1364             throw ase;
1365         }
1366     }
1367
1368     @Override
1369     public boolean doesBucketExistV2(String bucketName) throws SdkClientException {
1370         try {
1371             ValidationUtils.assertStringNotEmpty(bucketName, "bucketName");
1372             getBucketAcl(bucketName);
1373             return true;
1374         } catch (AmazonServiceException ase) {
1375             // A redirect error or an AccessDenied exception means the bucket exists but it's not in this region
1376             // or we don't have permissions to it.
1377             if ((ase.getStatusCode() == Constants.BUCKET_REDIRECT_STATUS_CODE) || "AccessDenied".equals(ase.getErrorCode())) {
1378                 return true;
1379             }
1380             if (ase.getStatusCode() == Constants.NO_SUCH_BUCKET_STATUS_CODE) {
1381                 return false;
1382             }
1383             throw ase;
1384         }
1385     }
1386
1387     @Override
1388     public boolean doesObjectExist(String bucketName, String objectName)
1389             throws AmazonServiceException, SdkClientException {
1390         try {
1391             ValidationUtils.assertStringNotEmpty(bucketName, "bucketName");
1392             ValidationUtils.assertStringNotEmpty(objectName, "objectName");
1393             getObjectMetadata(bucketName, objectName);
1394             return true;
1395         } catch (AmazonS3Exception e) {
1396             if (e.getStatusCode() == 404) {
1397                 return false;
1398             }
1399             throw e;
1400         }
1401     }
1402
1403     @Override
1404     public HeadBucketResult headBucket(HeadBucketRequest headBucketRequest)
1405             throws SdkClientException, AmazonServiceException {
1406         headBucketRequest = beforeClientExecution(headBucketRequest);
1407         String bucketName = headBucketRequest.getBucketName();
1408
1409         rejectNull(bucketName,
1410                 "The bucketName parameter must be specified.");
1411
1412         Request<HeadBucketRequest> request = createRequest(bucketName, null,
1413                 headBucketRequest, HttpMethodName.HEAD);
1414         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "HeadBucket");
1415
1416         return invoke(request, new HeadBucketResultHandler(), bucketName, null);
1417     }
1418
1419     @Override
1420     public void changeObjectStorageClass(String bucketName, String key, StorageClass newStorageClass)
1421         throws SdkClientException, AmazonServiceException {
1422         rejectNull(bucketName,
1423             "The bucketName parameter must be specified when changing an object's storage class");
1424         rejectNull(key,
1425             "The key parameter must be specified when changing an object's storage class");
1426         rejectNull(newStorageClass,
1427             "The newStorageClass parameter must be specified when changing an object's storage class");
1428
1429         copyObject(new CopyObjectRequest(bucketName, key, bucketName, key)
1430             .withStorageClass(newStorageClass.toString()));
1431     }
1432
1433     @Override
1434     public void setObjectRedirectLocation(String bucketName, String key, String newRedirectLocation)
1435         throws SdkClientException, AmazonServiceException {
1436         rejectNull(bucketName,
1437             "The bucketName parameter must be specified when changing an object's storage class");
1438         rejectNull(key,
1439             "The key parameter must be specified when changing an object's storage class");
1440         rejectNull(newRedirectLocation,
1441             "The newStorageClass parameter must be specified when changing an object's storage class");
1442
1443         copyObject(new CopyObjectRequest(bucketName, key, bucketName, key)
1444             .withRedirectLocation(newRedirectLocation));
1445     }
1446
1447     @Override
1448     public S3Object getObject(GetObjectRequest getObjectRequest)
1449         throws SdkClientException, AmazonServiceException {
1450         getObjectRequest = beforeClientExecution(getObjectRequest);
1451         assertNotNull(getObjectRequest, "GetObjectRequest");
1452         assertStringNotEmpty(getObjectRequest.getBucketName(), "BucketName");
1453         assertStringNotEmpty(getObjectRequest.getKey(), "Key");
1454
1455         Request<GetObjectRequest> request = createRequest(getObjectRequest.getBucketName(), getObjectRequest.getKey(), getObjectRequest, HttpMethodName.GET);
1456         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetObject");
1457         request.addHandlerContext(HandlerContextKey.HAS_STREAMING_OUTPUT, Boolean.TRUE);
1458
1459         if (getObjectRequest.getVersionId() != null) {
1460             request.addParameter("versionId", getObjectRequest.getVersionId());
1461         }
1462
1463         addPartNumberIfNotNull(request, getObjectRequest.getPartNumber());
1464
1465         // Range
1466         long[] range = getObjectRequest.getRange();
1467         if (range != null) {
1468             request.addHeader(Headers.RANGE, "bytes=" + Long.toString(range[0]) + "-" + Long.toString(range[1]));
1469         }
1470
1471         populateRequesterPaysHeader(request, getObjectRequest.isRequesterPays());
1472
1473         addResponseHeaderParameters(request, getObjectRequest.getResponseHeaders());
1474
1475         addDateHeader(request, Headers.GET_OBJECT_IF_MODIFIED_SINCE,
1476                       getObjectRequest.getModifiedSinceConstraint());
1477         addDateHeader(request, Headers.GET_OBJECT_IF_UNMODIFIED_SINCE,
1478                       getObjectRequest.getUnmodifiedSinceConstraint());
1479         addStringListHeader(request, Headers.GET_OBJECT_IF_MATCH,
1480                             getObjectRequest.getMatchingETagConstraints());
1481         addStringListHeader(request, Headers.GET_OBJECT_IF_NONE_MATCH,
1482                             getObjectRequest.getNonmatchingETagConstraints());
1483
1484         // Populate the SSE-C parameters to the request header
1485         populateSSE_C(request, getObjectRequest.getSSECustomerKey());
1486         final ProgressListener listener = getObjectRequest.getGeneralProgressListener();
1487         publishProgress(listener, ProgressEventType.TRANSFER_STARTED_EVENT);
1488
1489         try {
1490             S3Object s3Object = invoke(request, new S3ObjectResponseHandler(),
1491                                        getObjectRequest.getBucketName(), getObjectRequest.getKey());
1492             /*
1493              * TODO: For now, it's easiest to set there here in the client, but
1494              *       we could push this back into the response handler with a
1495              *       little more work.
1496              */
1497             s3Object.setBucketName(getObjectRequest.getBucketName());
1498             s3Object.setKey(getObjectRequest.getKey());
1499
1500             boolean skipClientSideValidation = skipMd5CheckStrategy.skipClientSideValidation(getObjectRequest,
1501                                                                                              s3Object.getObjectMetadata());
1502             postProcessS3Object(s3Object, skipClientSideValidation, listener);
1503             return s3Object;
1504         } catch (AmazonS3Exception ase) {
1505             /*
1506              * If the request failed because one of the specified constraints
1507              * was not met (ex: matching ETag, modified since date, etc.), then
1508              * return null, so that users don't have to wrap their code in
1509              * try/catch blocks and check for this status code if they want to
1510              * use constraints.
1511              */
1512             if (ase.getStatusCode() == 412 || ase.getStatusCode() == 304) {
1513                 publishProgress(listener, ProgressEventType.TRANSFER_CANCELED_EVENT);
1514                 return null;
1515             }
1516             publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT);
1517             throw ase;
1518         }
1519     }
1520
1521     /**
1522      * Post processing the {@link S3Object} downloaded from S3. It includes wrapping the data with wrapper input streams,
1523      * doing client side validation if possible etc.
1524      */
1525     private void postProcessS3Object(final S3Object s3Object, final boolean skipClientSideValidation,
1526                                          final ProgressListener listener) {
1527         InputStream is = s3Object.getObjectContent();
1528         HttpRequestBase httpRequest = s3Object.getObjectContent().getHttpRequest();
1529         // Hold a reference to this client while the InputStream is still
1530         // around - otherwise a finalizer in the HttpClient may reset the
1531         // underlying TCP connection out from under us.
1532         is = new ServiceClientHolderInputStream(is, this);
1533         // used trigger a tranfer complete event when the stream is entirely consumed
1534         ProgressInputStream progressInputStream =
1535             new ProgressInputStream(is, listener) {
1536                 @Override protected void onEOF() {
1537                     publishProgress(getListener(), ProgressEventType.TRANSFER_COMPLETED_EVENT);
1538                 }
1539             };
1540         is = progressInputStream;
1541
1542         // The Etag header contains a server-side MD5 of the object. If
1543         // we're downloading the whole object, by default we wrap the
1544         // stream in a validator that calculates an MD5 of the downloaded
1545         // bytes and complains if what we received doesn't match the Etag.
1546         if (!skipClientSideValidation) {
1547             byte[] serverSideHash = BinaryUtils.fromHex(s3Object.getObjectMetadata().getETag());
1548             try {
1549                 // No content length check is performed when the
1550                 // MD5 check is enabled, since a correct MD5 check would
1551                 // imply a correct content length.
1552                 MessageDigest digest = MessageDigest.getInstance("MD5");
1553                 is = new DigestValidationInputStream(is, digest, serverSideHash);
1554             } catch (NoSuchAlgorithmException e) {
1555                 log.warn("No MD5 digest algorithm available.  Unable to calculate "
1556                          + "checksum and verify data integrity.", e);
1557             }
1558         } else {
1559             // Ensures the data received from S3 has the same length as the
1560             // expected content-length
1561             is = new LengthCheckInputStream(is,
1562                                             s3Object.getObjectMetadata().getContentLength(), // expected length
1563                                             INCLUDE_SKIPPED_BYTES); // bytes received from S3 are all included even if skipped
1564         }
1565
1566         S3AbortableInputStream abortableInputStream =
1567             new S3AbortableInputStream(is, httpRequest, s3Object.getObjectMetadata().getContentLength());
1568         s3Object.setObjectContent(new S3ObjectInputStream(abortableInputStream, httpRequest, false));
1569     }
1570
1571     @Override
1572     public ObjectMetadata getObject(final GetObjectRequest getObjectRequest, File destinationFile)
1573             throws SdkClientException, AmazonServiceException {
1574         rejectNull(destinationFile,
1575                 "The destination file parameter must be specified when downloading an object directly to a file");
1576
1577         S3Object s3Object = ServiceUtils.retryableDownloadS3ObjectToFile(destinationFile, new ServiceUtils.RetryableS3DownloadTask() {
1578
1579             @Override
1580             public S3Object getS3ObjectStream() {
1581                 return getObject(getObjectRequest);
1582             }
1583
1584             @Override
1585             public boolean needIntegrityCheck() {
1586                 return !skipMd5CheckStrategy.skipClientSideValidationPerRequest(getObjectRequest);
1587             }
1588
1589         }, ServiceUtils.OVERWRITE_MODE);
1590         // getObject can return null if constraints were specified but not met
1591         if (s3Object == null) return null;
1592
1593         return s3Object.getObjectMetadata();
1594     }
1595
1596     @Override
1597     public String getObjectAsString(String bucketName, String key)
1598             throws AmazonServiceException, SdkClientException {
1599         rejectNull(bucketName, "Bucket name must be provided");
1600         rejectNull(key, "Object key must be provided");
1601
1602         S3Object object = getObject(bucketName, key);
1603         try {
1604             return IOUtils.toString(object.getObjectContent());
1605         } catch (IOException e) {
1606             throw new SdkClientException("Error streaming content from S3 during download", e);
1607         } finally {
1608             IOUtils.closeQuietly(object, log);
1609         }
1610     }
1611
1612     @Override
1613     public GetObjectTaggingResult getObjectTagging(GetObjectTaggingRequest getObjectTaggingRequest) {
1614         getObjectTaggingRequest = beforeClientExecution(getObjectTaggingRequest);
1615         rejectNull(getObjectTaggingRequest,
1616                 "The request parameter must be specified when getting the object tags");
1617         String bucketName = assertStringNotEmpty(getObjectTaggingRequest.getBucketName(), "BucketName");
1618         String key = assertNotNull(getObjectTaggingRequest.getKey(), "Key");
1619
1620         Request<GetObjectTaggingRequest> request = createRequest(bucketName, key, getObjectTaggingRequest, HttpMethodName.GET);
1621         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetObjectTagging");
1622         request.addParameter("tagging", null);
1623         addParameterIfNotNull(request, "versionId", getObjectTaggingRequest.getVersionId());
1624
1625         ResponseHeaderHandlerChain<GetObjectTaggingResult> handlerChain = new ResponseHeaderHandlerChain<GetObjectTaggingResult>(
1626                 new Unmarshallers.GetObjectTaggingResponseUnmarshaller(),
1627                 new GetObjectTaggingResponseHeaderHandler()
1628         );
1629
1630         return invoke(request, handlerChain, bucketName, key);
1631     }
1632
1633     @Override
1634     public SetObjectTaggingResult setObjectTagging(SetObjectTaggingRequest setObjectTaggingRequest) {
1635         setObjectTaggingRequest = beforeClientExecution(setObjectTaggingRequest);
1636         rejectNull(setObjectTaggingRequest,
1637                 "The request parameter must be specified setting the object tags");
1638         String bucketName = assertStringNotEmpty(setObjectTaggingRequest.getBucketName(), "BucketName");
1639         String key = assertNotNull(setObjectTaggingRequest.getKey(), "Key");
1640         ObjectTagging tagging = assertNotNull(setObjectTaggingRequest.getTagging(), "ObjectTagging");
1641
1642         Request<SetObjectTaggingRequest> request = createRequest(bucketName, key, setObjectTaggingRequest, HttpMethodName.PUT);
1643         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutObjectTagging");
1644         request.addParameter("tagging", null);
1645         addParameterIfNotNull(request, "versionId", setObjectTaggingRequest.getVersionId());
1646         byte[] content = new ObjectTaggingXmlFactory().convertToXmlByteArray(tagging);
1647         setContent(request, content, "application/xml", true);
1648
1649         ResponseHeaderHandlerChain<SetObjectTaggingResult> handlerChain = new ResponseHeaderHandlerChain<SetObjectTaggingResult>(
1650                 new Unmarshallers.SetObjectTaggingResponseUnmarshaller(),
1651                 new SetObjectTaggingResponseHeaderHandler()
1652         );
1653
1654         return invoke(request, handlerChain, bucketName, key);
1655     }
1656
1657     @Override
1658     public DeleteObjectTaggingResult deleteObjectTagging(DeleteObjectTaggingRequest deleteObjectTaggingRequest) {
1659         deleteObjectTaggingRequest = beforeClientExecution(deleteObjectTaggingRequest);
1660         rejectNull(deleteObjectTaggingRequest, "The request parameter must be specified when delete the object tags");
1661         String bucketName = assertStringNotEmpty(deleteObjectTaggingRequest.getBucketName(), "BucketName");
1662         String key = assertStringNotEmpty(deleteObjectTaggingRequest.getKey(), "Key");
1663
1664         Request<DeleteObjectTaggingRequest> request = createRequest(bucketName, key, deleteObjectTaggingRequest, HttpMethodName.DELETE);
1665         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteObjectTagging");
1666         request.addParameter("tagging", null);
1667         addParameterIfNotNull(request, "versionId", deleteObjectTaggingRequest.getVersionId());
1668
1669         ResponseHeaderHandlerChain<DeleteObjectTaggingResult> handlerChain = new ResponseHeaderHandlerChain<DeleteObjectTaggingResult>(
1670                 new Unmarshallers.DeleteObjectTaggingResponseUnmarshaller(),
1671                 new DeleteObjectTaggingHeaderHandler()
1672         );
1673
1674         return invoke(request, handlerChain, bucketName, key);
1675     }
1676
1677     @Override
1678     public void deleteBucket(String bucketName)
1679             throws SdkClientException, AmazonServiceException {
1680         deleteBucket(new DeleteBucketRequest(bucketName));
1681     }
1682
1683     @Override
1684     public void deleteBucket(DeleteBucketRequest deleteBucketRequest)
1685             throws SdkClientException, AmazonServiceException {
1686         deleteBucketRequest = beforeClientExecution(deleteBucketRequest);
1687         rejectNull(deleteBucketRequest,
1688                 "The DeleteBucketRequest parameter must be specified when deleting a bucket");
1689
1690         String bucketName = deleteBucketRequest.getBucketName();
1691         rejectNull(bucketName,
1692                 "The bucket name parameter must be specified when deleting a bucket");
1693
1694         Request<DeleteBucketRequest> request = createRequest(bucketName, null, deleteBucketRequest, HttpMethodName.DELETE);
1695         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteBucket");
1696         invoke(request, voidResponseHandler, bucketName, null);
1697         bucketRegionCache.remove(bucketName);
1698     }
1699
1700     @Override
1701     public PutObjectResult putObject(String bucketName, String key, File file)
1702             throws SdkClientException, AmazonServiceException {
1703         return putObject(new PutObjectRequest(bucketName, key, file)
1704             .withMetadata(new ObjectMetadata()));
1705     }
1706
1707     @Override
1708     public PutObjectResult putObject(String bucketName, String key, InputStream input, ObjectMetadata metadata)
1709             throws SdkClientException, AmazonServiceException {
1710         return putObject(new PutObjectRequest(bucketName, key, input, metadata));
1711     }
1712
1713     @Override
1714     public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws SdkClientException, AmazonServiceException {
1715         putObjectRequest = beforeClientExecution(putObjectRequest);
1716         rejectNull(putObjectRequest, "The PutObjectRequest parameter must be specified when uploading an object");
1717         final File file = putObjectRequest.getFile();
1718         final InputStream isOrig = putObjectRequest.getInputStream();
1719         final String bucketName = putObjectRequest.getBucketName();
1720         final String key = putObjectRequest.getKey();
1721         final ProgressListener listener = putObjectRequest.getGeneralProgressListener();
1722         rejectNull(bucketName, "The bucket name parameter must be specified when uploading an object");
1723         rejectNull(key, "The key parameter must be specified when uploading an object");
1724
1725         ObjectMetadata metadata = putObjectRequest.getMetadata();
1726         if (metadata == null)
1727             metadata = new ObjectMetadata();
1728
1729         Request<PutObjectRequest> request = createRequest(bucketName, key, putObjectRequest, HttpMethodName.PUT);
1730         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutObject");
1731         request.addHandlerContext(HandlerContextKey.REQUIRES_LENGTH, Boolean.TRUE);
1732         request.addHandlerContext(HandlerContextKey.HAS_STREAMING_INPUT, Boolean.TRUE);
1733
1734         // Make backward compatible with buffer size via system property
1735         final Integer bufsize = Constants.getS3StreamBufferSize();
1736         if (bufsize != null) {
1737             AmazonWebServiceRequest awsreq = request.getOriginalRequest();
1738             // Note awsreq is never null at this point even if the original
1739             // request was
1740             awsreq.getRequestClientOptions()
1741                   .setReadLimit(bufsize.intValue());
1742         }
1743         if ( putObjectRequest.getAccessControlList() != null) {
1744             addAclHeaders(request, putObjectRequest.getAccessControlList());
1745         } else if ( putObjectRequest.getCannedAcl() != null ) {
1746             request.addHeader(Headers.S3_CANNED_ACL, putObjectRequest.getCannedAcl().toString());
1747         }
1748
1749         if (putObjectRequest.getStorageClass() != null) {
1750             request.addHeader(Headers.STORAGE_CLASS, putObjectRequest.getStorageClass());
1751         }
1752
1753         if (putObjectRequest.getRedirectLocation() != null) {
1754             request.addHeader(Headers.REDIRECT_LOCATION, putObjectRequest.getRedirectLocation());
1755         }
1756
1757         addHeaderIfNotNull(request, Headers.S3_TAGGING, urlEncodeTags(putObjectRequest.getTagging()));
1758
1759         populateRequesterPaysHeader(request, putObjectRequest.isRequesterPays());
1760
1761         // Populate the SSE-C parameters to the request header
1762         populateSSE_C(request, putObjectRequest.getSSECustomerKey());
1763
1764         // Populate the SSE AWS KMS parameters to the request header
1765         populateSSE_KMS(request,
1766                         putObjectRequest.getSSEAwsKeyManagementParams());
1767
1768         populateObjectLockHeaders(request, putObjectRequest.getObjectLockMode(), putObjectRequest.getObjectLockRetainUntilDate(),
1769                                   putObjectRequest.getObjectLockLegalHoldStatus());
1770
1771
1772         return uploadObject(isOrig, file, metadata, listener, request, putObjectRequest,
1773                             skipMd5CheckStrategy.skipServerSideValidation(putObjectRequest),
1774                             skipMd5CheckStrategy.skipClientSideValidationPerRequest(putObjectRequest),
1775                             new PutObjectStrategy(bucketName, key),
1776                             true);
1777     }
1778
1779     /**
1780      * Helper method used by {@link #putObject(PutObjectRequest)} and {@link #upload(PresignedUrlUploadRequest)}.
1781      */
1782     private <RequestT, ResponseT> ResponseT uploadObject(final InputStream originalStream,
1783                                                          final File file,
1784                                                          final ObjectMetadata metadata,
1785                                                          final ProgressListener listener,
1786                                                          final Request<RequestT> request,
1787                                                          final S3DataSource originalRequest,
1788                                                          final boolean skipServerSideValidation,
1789                                                          final boolean skipClientSideValidationPerRequest,
1790                                                          final UploadObjectStrategy<RequestT, ResponseT> uploadStrategy,
1791                                                          final boolean setContentTypeIfNotProvided) {
1792
1793         InputStream input = getInputStream(originalStream, file, metadata, request,
1794                                            skipServerSideValidation, setContentTypeIfNotProvided);
1795
1796         final ObjectMetadata returnedMetadata;
1797         MD5DigestCalculatingInputStream md5DigestStream = null;
1798         try {
1799             if (metadata.getContentMD5() == null && !skipClientSideValidationPerRequest) {
1800                 /*
1801                  * If the user hasn't set the content MD5, then we don't want to buffer the whole
1802                  * stream in memory just to calculate it. Instead, we can calculate it on the fly
1803                  * and validate it with the returned ETag from the object upload.
1804                  */
1805                 input = md5DigestStream = new MD5DigestCalculatingInputStream(input);
1806             }
1807
1808             populateRequestMetadata(request, metadata);
1809             request.setContent(input);
1810             publishProgress(listener, ProgressEventType.TRANSFER_STARTED_EVENT);
1811             try {
1812                 returnedMetadata = uploadStrategy.invokeServiceCall(request);
1813             } catch (Throwable t) {
1814                 publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT);
1815                 throw failure(t);
1816             }
1817         } finally {
1818             cleanupDataSource(originalRequest, file, originalStream, input, log);
1819         }
1820
1821         String contentMd5 = metadata.getContentMD5();
1822         if (md5DigestStream != null) {
1823             contentMd5 = Base64.encodeAsString(md5DigestStream.getMd5Digest());
1824         }
1825
1826         final String etag = returnedMetadata.getETag();
1827         if (contentMd5 != null && !skipMd5CheckStrategy.skipClientSideValidationPerPutResponse(returnedMetadata)) {
1828             byte[] clientSideHash = BinaryUtils.fromBase64(contentMd5);
1829             byte[] serverSideHash = BinaryUtils.fromHex(etag);
1830
1831             if (!Arrays.equals(clientSideHash, serverSideHash)) {
1832                 publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT);
1833                 throw new SdkClientException(
1834                     "Unable to verify integrity of data upload. Client calculated content hash (contentMD5: " + contentMd5
1835                     + " in base 64) didn't match hash (etag: " + etag + " in hex) calculated by Amazon S3.  "
1836                     + "You may need to delete the data stored in Amazon S3. (metadata.contentMD5: " + metadata.getContentMD5()
1837                     + ", md5DigestStream: " + md5DigestStream
1838                     + uploadStrategy.md5ValidationErrorSuffix()
1839                     + ")");
1840             }
1841         }
1842
1843         publishProgress(listener, ProgressEventType.TRANSFER_COMPLETED_EVENT);
1844
1845         return uploadStrategy.createResult(returnedMetadata, contentMd5);
1846     }
1847
1848     private InputStream getInputStream(final InputStream origStream, final File file, final ObjectMetadata metadata,
1849                                        final Request<?> request, final boolean skipServerSideValidation,
1850                                        final boolean setContentTypeIfNotProvided) {
1851         InputStream input = origStream;
1852
1853         // If a file is specified for upload, we need to pull some additional
1854         // information from it to auto-configure a few options
1855         if (file == null) {
1856             // When input is a FileInputStream, this wrapping enables
1857             // unlimited mark-and-reset
1858             if (input != null)
1859                 input = ReleasableInputStream.wrap(input);
1860         } else {
1861             // Always set the content length, even if it's already set
1862             metadata.setContentLength(file.length());
1863             final boolean calculateMD5 = metadata.getContentMD5() == null;
1864             // Only set the content type if it hasn't already been set
1865             if (metadata.getContentType() == null && setContentTypeIfNotProvided) {
1866                 metadata.setContentType(Mimetypes.getInstance().getMimetype(file));
1867             }
1868
1869             if (calculateMD5 && !skipServerSideValidation) {
1870                 try {
1871                     String contentMd5_b64 = Md5Utils.md5AsBase64(file);
1872                     metadata.setContentMD5(contentMd5_b64);
1873                 } catch (Exception e) {
1874                     throw new SdkClientException(
1875                         "Unable to calculate MD5 hash: " + e.getMessage(), e);
1876                 }
1877             }
1878             input = newResettableInputStream(file, "Unable to find file to upload");
1879         }
1880
1881         if (metadata.getContentType() == null && setContentTypeIfNotProvided) {
1882             /*
1883              * Default to the "application/octet-stream" if the user hasn't
1884              * specified a content type.
1885              */
1886             metadata.setContentType(Mimetypes.MIMETYPE_OCTET_STREAM);
1887         }
1888
1889         if (request.getHeaders().get(Headers.REDIRECT_LOCATION) != null && input == null) {
1890             input = new ByteArrayInputStream(new byte[0]);
1891         }
1892
1893         // Use internal interface to differentiate 0 from unset.
1894         final Long contentLength = (Long)metadata.getRawMetadataValue(Headers.CONTENT_LENGTH);
1895         if (contentLength == null) {
1896                 /*
1897                  * There's nothing we can do except for let the HTTP client buffer
1898                  * the input stream contents if the caller doesn't tell us how much
1899                  * data to expect in a stream since we have to explicitly tell
1900                  * Amazon S3 how much we're sending before we start sending any of
1901                  * it.
1902                  */
1903             log.warn("No content length specified for stream data.  " +
1904                      "Stream contents will be buffered in memory and could result in " +
1905                      "out of memory errors.");
1906         } else {
1907             final long expectedLength = contentLength.longValue();
1908             if (expectedLength >= 0) {
1909                 // Performs length check on the underlying data stream.
1910                 // For S3 encryption client, the underlying data stream here
1911                 // refers to the cipher-text data stream (ie not the underlying
1912                 // plain-text data stream which in turn may have been wrapped
1913                 // with it's own length check input stream.)
1914                 LengthCheckInputStream lcis = new LengthCheckInputStream(
1915                     input,
1916                     expectedLength, // expected data length to be uploaded
1917                     EXCLUDE_SKIPPED_BYTES);
1918                 input = lcis;
1919             }
1920         }
1921
1922         return input;
1923     }
1924
1925     private static PutObjectResult createPutObjectResult(ObjectMetadata metadata) {
1926         final PutObjectResult result = new PutObjectResult();
1927         result.setVersionId(metadata.getVersionId());
1928         result.setSSEAlgorithm(metadata.getSSEAlgorithm());
1929         result.setSSECustomerAlgorithm(metadata.getSSECustomerAlgorithm());
1930         result.setSSECustomerKeyMd5(metadata.getSSECustomerKeyMd5());
1931         result.setExpirationTime(metadata.getExpirationTime());
1932         result.setExpirationTimeRuleId(metadata.getExpirationTimeRuleId());
1933         result.setETag(metadata.getETag());
1934         result.setMetadata(metadata);
1935         result.setRequesterCharged(metadata.isRequesterCharged());
1936         return result;
1937     }
1938
1939     /**
1940      * Sets the access control headers for the request given.
1941      */
1942     private static void addAclHeaders(Request<? extends AmazonWebServiceRequest> request, AccessControlList acl) {
1943         List<Grant> grants = acl.getGrantsAsList();
1944         Map<Permission, Collection<Grantee>> grantsByPermission = new HashMap<Permission, Collection<Grantee>>();
1945         for ( Grant grant : grants ) {
1946             if ( !grantsByPermission.containsKey(grant.getPermission()) ) {
1947                 grantsByPermission.put(grant.getPermission(), new LinkedList<Grantee>());
1948             }
1949             grantsByPermission.get(grant.getPermission()).add(grant.getGrantee());
1950         }
1951         for ( Permission permission : Permission.values() ) {
1952             if ( grantsByPermission.containsKey(permission) ) {
1953                 Collection<Grantee> grantees = grantsByPermission.get(permission);
1954                 boolean seenOne = false;
1955                 StringBuilder granteeString = new StringBuilder();
1956                 for ( Grantee grantee : grantees ) {
1957                     if ( !seenOne )
1958                         seenOne = true;
1959                     else
1960                         granteeString.append(", ");
1961                     granteeString.append(grantee.getTypeIdentifier()).append("=").append("\"")
1962                             .append(grantee.getIdentifier()).append("\"");
1963                 }
1964                 request.addHeader(permission.getHeaderName(), granteeString.toString());
1965             }
1966         }
1967     }
1968
1969     @Override
1970     public CopyObjectResult copyObject(String sourceBucketName, String sourceKey,
1971                                        String destinationBucketName, String destinationKey)
1972             throws SdkClientException, AmazonServiceException {
1973         return copyObject(new CopyObjectRequest(sourceBucketName, sourceKey,
1974                                                 destinationBucketName, destinationKey));
1975     }
1976
1977     @Override
1978     public CopyObjectResult copyObject(CopyObjectRequest copyObjectRequest)
1979             throws SdkClientException, AmazonServiceException {
1980         copyObjectRequest = beforeClientExecution(copyObjectRequest);
1981         rejectNull(copyObjectRequest.getSourceBucketName(),
1982                 "The source bucket name must be specified when copying an object");
1983         rejectNull(copyObjectRequest.getSourceKey(),
1984                 "The source object key must be specified when copying an object");
1985         rejectNull(copyObjectRequest.getDestinationBucketName(),
1986                 "The destination bucket name must be specified when copying an object");
1987         rejectNull(copyObjectRequest.getDestinationKey(),
1988                 "The destination object key must be specified when copying an object");
1989
1990         String destinationKey = copyObjectRequest.getDestinationKey();
1991         String destinationBucketName = copyObjectRequest.getDestinationBucketName();
1992
1993         Request<CopyObjectRequest> request = createRequest(destinationBucketName, destinationKey, copyObjectRequest, HttpMethodName.PUT);
1994         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "CopyObject");
1995
1996         populateRequestWithCopyObjectParameters(request, copyObjectRequest);
1997
1998         // Populate the SSE AWS KMS parameters to the request header
1999         populateSSE_KMS(request,
2000                 copyObjectRequest.getSSEAwsKeyManagementParams());
2001
2002         populateObjectLockHeaders(request, copyObjectRequest.getObjectLockMode(), copyObjectRequest.getObjectLockRetainUntilDate(),
2003                 copyObjectRequest.getObjectLockLegalHoldStatus());
2004
2005         /*
2006          * We can't send a non-zero length Content-Length header if the user
2007          * specified it, otherwise it messes up the HTTP connection when the
2008          * remote server thinks there's more data to pull.
2009          */
2010         setZeroContentLength(request);
2011         CopyObjectResultHandler copyObjectResultHandler = null;
2012         try {
2013             @SuppressWarnings("unchecked")
2014             ResponseHeaderHandlerChain<CopyObjectResultHandler> handler = new ResponseHeaderHandlerChain<CopyObjectResultHandler>(
2015                     // xml payload unmarshaller
2016                     new Unmarshallers.CopyObjectUnmarshaller(),
2017                     // header handlers
2018                     new ServerSideEncryptionHeaderHandler<CopyObjectResultHandler>(),
2019                     new S3VersionHeaderHandler<CopyObjectResultHandler>(),
2020                     new ObjectExpirationHeaderHandler<CopyObjectResultHandler>(),
2021                     new S3RequesterChargedHeaderHandler<CopyObjectResultHandler>());
2022             copyObjectResultHandler = invoke(request, handler, destinationBucketName, destinationKey);
2023         } catch (AmazonS3Exception ase) {
2024             /*
2025              * If the request failed because one of the specified constraints
2026              * was not met (ex: matching ETag, modified since date, etc.), then
2027              * return null, so that users don't have to wrap their code in
2028              * try/catch blocks and check for this status code if they want to
2029              * use constraints.
2030              */
2031             if (ase.getStatusCode() == Constants.FAILED_PRECONDITION_STATUS_CODE) {
2032                return null;
2033             }
2034
2035             throw ase;
2036         }
2037
2038         /*
2039          * CopyObject has two failure modes:
2040          *  1 - An HTTP error code is returned and the error is processed like any
2041          *      other error response.
2042          *  2 - An HTTP 200 OK code is returned, but the response content contains
2043          *      an XML error response.
2044          *
2045          * This makes it very difficult for the client runtime to cleanly detect
2046          * this case and handle it like any other error response.  We could
2047          * extend the runtime to have a more flexible/customizable definition of
2048          * success/error (per request), but it's probably overkill for this
2049          * one special case.
2050          */
2051         if (copyObjectResultHandler.getErrorCode() != null) {
2052             String errorCode = copyObjectResultHandler.getErrorCode();
2053             String errorMessage = copyObjectResultHandler.getErrorMessage();
2054             String requestId = copyObjectResultHandler.getErrorRequestId();
2055             String hostId = copyObjectResultHandler.getErrorHostId();
2056
2057             AmazonS3Exception ase = new AmazonS3Exception(errorMessage);
2058             ase.setErrorCode(errorCode);
2059             ase.setErrorType(ErrorType.Service);
2060             ase.setRequestId(requestId);
2061             ase.setExtendedRequestId(hostId);
2062             ase.setServiceName(request.getServiceName());
2063             ase.setStatusCode(200);
2064             ase.setProxyHost(clientConfiguration.getProxyHost());
2065
2066             throw ase;
2067         }
2068
2069         // TODO: Might be nice to create this in our custom S3VersionHeaderHandler
2070         CopyObjectResult copyObjectResult = new CopyObjectResult();
2071         copyObjectResult.setETag(copyObjectResultHandler.getETag());
2072         copyObjectResult.setLastModifiedDate(copyObjectResultHandler.getLastModified());
2073         copyObjectResult.setVersionId(copyObjectResultHandler.getVersionId());
2074         copyObjectResult.setSSEAlgorithm(copyObjectResultHandler.getSSEAlgorithm());
2075         copyObjectResult.setSSECustomerAlgorithm(copyObjectResultHandler.getSSECustomerAlgorithm());
2076         copyObjectResult.setSSECustomerKeyMd5(copyObjectResultHandler.getSSECustomerKeyMd5());
2077         copyObjectResult.setExpirationTime(copyObjectResultHandler.getExpirationTime());
2078         copyObjectResult.setExpirationTimeRuleId(copyObjectResultHandler.getExpirationTimeRuleId());
2079         copyObjectResult.setRequesterCharged(copyObjectResultHandler.isRequesterCharged());
2080
2081         return copyObjectResult;
2082     }
2083
2084     /**
2085      * Copies a source object to a part of a multipart upload.
2086      *
2087      * To copy an object, the caller's account must have read access to the source object and
2088      * write access to the destination bucket.
2089      * </p>
2090      * <p>
2091      * If constraints are specified in the <code>CopyPartRequest</code>
2092      * (e.g.
2093      * {@link CopyPartRequest#setMatchingETagConstraints(List)})
2094      * and are not satisfied when Amazon S3 receives the
2095      * request, this method returns <code>null</code>.
2096      * This method returns a non-null result under all other
2097      * circumstances.
2098      * </p>
2099      *
2100      * @param copyPartRequest
2101      *            The request object containing all the options for copying an
2102      *            Amazon S3 object.
2103      *
2104      * @return A {@link CopyPartResult} object containing the information
2105      *         returned by Amazon S3 about the newly created object, or <code>null</code> if
2106      *         constraints were specified that weren't met when Amazon S3 attempted
2107      *         to copy the object.
2108      *
2109      * @throws SdkClientException
2110      *             If any errors are encountered in the client while making the
2111      *             request or handling the response.
2112      * @throws AmazonServiceException
2113      *             If any errors occurred in Amazon S3 while processing the
2114      *             request.
2115      *
2116      * @see AmazonS3#copyObject(CopyObjectRequest)
2117      * @see AmazonS3#initiateMultipartUpload(InitiateMultipartUploadRequest)
2118      */
2119     @Override
2120     public CopyPartResult copyPart(CopyPartRequest copyPartRequest) {
2121         copyPartRequest = beforeClientExecution(copyPartRequest);
2122         rejectNull(copyPartRequest.getSourceBucketName(),
2123                 "The source bucket name must be specified when copying a part");
2124         rejectNull(copyPartRequest.getSourceKey(),
2125                 "The source object key must be specified when copying a part");
2126         rejectNull(copyPartRequest.getDestinationBucketName(),
2127                 "The destination bucket name must be specified when copying a part");
2128         rejectNull(copyPartRequest.getUploadId(),
2129                 "The upload id must be specified when copying a part");
2130         rejectNull(copyPartRequest.getDestinationKey(),
2131                 "The destination object key must be specified when copying a part");
2132         rejectNull(copyPartRequest.getPartNumber(),
2133                 "The part number must be specified when copying a part");
2134
2135         String destinationKey = copyPartRequest.getDestinationKey();
2136         String destinationBucketName = copyPartRequest.getDestinationBucketName();
2137
2138         Request<CopyPartRequest> request = createRequest(destinationBucketName, destinationKey, copyPartRequest,
2139                 HttpMethodName.PUT);
2140         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "UploadPartCopy");
2141
2142         populateRequestWithCopyPartParameters(request, copyPartRequest);
2143
2144         request.addParameter("uploadId", copyPartRequest.getUploadId());
2145         request.addParameter("partNumber", Integer.toString(copyPartRequest.getPartNumber()));
2146
2147         populateRequesterPaysHeader(request, copyPartRequest.isRequesterPays());
2148
2149         /*
2150          * We can't send a non-zero length Content-Length header if the user
2151          * specified it, otherwise it messes up the HTTP connection when the
2152          * remote server thinks there's more data to pull.
2153          */
2154         setZeroContentLength(request);
2155         CopyObjectResultHandler copyObjectResultHandler = null;
2156         try {
2157             @SuppressWarnings("unchecked")
2158             ResponseHeaderHandlerChain<CopyObjectResultHandler> handler = new ResponseHeaderHandlerChain<CopyObjectResultHandler>(
2159                     // xml payload unmarshaller
2160                     new Unmarshallers.CopyObjectUnmarshaller(),
2161                     // header handlers
2162                     new ServerSideEncryptionHeaderHandler<CopyObjectResultHandler>(),
2163                     new S3VersionHeaderHandler<CopyObjectResultHandler>());
2164             copyObjectResultHandler = invoke(request, handler, destinationBucketName, destinationKey);
2165         } catch ( AmazonS3Exception ase ) {
2166             /*
2167              * If the request failed because one of the specified constraints
2168              * was not met (ex: matching ETag, modified since date, etc.), then
2169              * return null, so that users don't have to wrap their code in
2170              * try/catch blocks and check for this status code if they want to
2171              * use constraints.
2172              */
2173             if ( ase.getStatusCode() == Constants.FAILED_PRECONDITION_STATUS_CODE ) {
2174                 return null;
2175             }
2176
2177             throw ase;
2178         }
2179
2180         /*
2181          * CopyPart has two failure modes: 1 - An HTTP error code is returned
2182          * and the error is processed like any other error response. 2 - An HTTP
2183          * 200 OK code is returned, but the response content contains an XML
2184          * error response.
2185          *
2186          * This makes it very difficult for the client runtime to cleanly detect
2187          * this case and handle it like any other error response. We could
2188          * extend the runtime to have a more flexible/customizable definition of
2189          * success/error (per request), but it's probably overkill for this one
2190          * special case.
2191          */
2192         if ( copyObjectResultHandler.getErrorCode() != null ) {
2193             String errorCode = copyObjectResultHandler.getErrorCode();
2194             String errorMessage = copyObjectResultHandler.getErrorMessage();
2195             String requestId = copyObjectResultHandler.getErrorRequestId();
2196             String hostId = copyObjectResultHandler.getErrorHostId();
2197
2198             AmazonS3Exception ase = new AmazonS3Exception(errorMessage);
2199             ase.setErrorCode(errorCode);
2200             ase.setErrorType(ErrorType.Service);
2201             ase.setRequestId(requestId);
2202             ase.setExtendedRequestId(hostId);
2203             ase.setServiceName(request.getServiceName());
2204             ase.setStatusCode(200);
2205             ase.setProxyHost(clientConfiguration.getProxyHost());
2206
2207             throw ase;
2208         }
2209
2210         CopyPartResult copyPartResult = new CopyPartResult();
2211         copyPartResult.setETag(copyObjectResultHandler.getETag());
2212         copyPartResult.setPartNumber(copyPartRequest.getPartNumber());
2213         copyPartResult.setLastModifiedDate(copyObjectResultHandler.getLastModified());
2214         copyPartResult.setVersionId(copyObjectResultHandler.getVersionId());
2215         copyPartResult.setSSEAlgorithm(copyObjectResultHandler.getSSEAlgorithm());
2216         copyPartResult.setSSECustomerAlgorithm(copyObjectResultHandler.getSSECustomerAlgorithm());
2217         copyPartResult.setSSECustomerKeyMd5(copyObjectResultHandler.getSSECustomerKeyMd5());
2218
2219         return copyPartResult;
2220     }
2221
2222     @Override
2223     public void deleteObject(String bucketName, String key)
2224             throws SdkClientException, AmazonServiceException {
2225         deleteObject(new DeleteObjectRequest(bucketName, key));
2226     }
2227
2228     @Override
2229     public void deleteObject(DeleteObjectRequest deleteObjectRequest)
2230             throws SdkClientException, AmazonServiceException {
2231         deleteObjectRequest = beforeClientExecution(deleteObjectRequest);
2232         rejectNull(deleteObjectRequest,
2233             "The delete object request must be specified when deleting an object");
2234
2235         rejectNull(deleteObjectRequest.getBucketName(), "The bucket name must be specified when deleting an object");
2236         rejectNull(deleteObjectRequest.getKey(), "The key must be specified when deleting an object");
2237
2238         Request<DeleteObjectRequest> request = createRequest(deleteObjectRequest.getBucketName(), deleteObjectRequest.getKey(), deleteObjectRequest, HttpMethodName.DELETE);
2239         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteObject");
2240
2241         invoke(request, voidResponseHandler, deleteObjectRequest.getBucketName(), deleteObjectRequest.getKey());
2242     }
2243
2244     @Override
2245     public DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsRequest) {
2246         deleteObjectsRequest = beforeClientExecution(deleteObjectsRequest);
2247         Request<DeleteObjectsRequest> request = createRequest(deleteObjectsRequest.getBucketName(), null, deleteObjectsRequest, HttpMethodName.POST);
2248         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteObjects");
2249         request.addParameter("delete", null);
2250         if (deleteObjectsRequest.getBypassGovernanceRetention()) {
2251             request.addHeader(Headers.BYPASS_GOVERNANCE_RETENTION, "true");
2252         }
2253
2254         if ( deleteObjectsRequest.getMfa() != null ) {
2255             populateRequestWithMfaDetails(request, deleteObjectsRequest.getMfa());
2256         }
2257
2258         populateRequesterPaysHeader(request, deleteObjectsRequest.isRequesterPays());
2259
2260         byte[] content = new MultiObjectDeleteXmlFactory().convertToXmlByteArray(deleteObjectsRequest);
2261         request.addHeader("Content-Length", String.valueOf(content.length));
2262         request.addHeader("Content-Type", "application/xml");
2263         request.setContent(new ByteArrayInputStream(content));
2264         try {
2265             byte[] md5 = Md5Utils.computeMD5Hash(content);
2266             String md5Base64 = BinaryUtils.toBase64(md5);
2267             request.addHeader("Content-MD5", md5Base64);
2268         } catch ( Exception e ) {
2269             throw new SdkClientException("Couldn't compute md5 sum", e);
2270         }
2271
2272         @SuppressWarnings("unchecked")
2273         ResponseHeaderHandlerChain<DeleteObjectsResponse> responseHandler = new ResponseHeaderHandlerChain<DeleteObjectsResponse>(
2274                 new Unmarshallers.DeleteObjectsResultUnmarshaller(),
2275                 new S3RequesterChargedHeaderHandler<DeleteObjectsResponse>());
2276
2277         DeleteObjectsResponse response = invoke(request, responseHandler, deleteObjectsRequest.getBucketName(), null);
2278
2279         /*
2280          * If the result was only partially successful, throw an exception
2281          */
2282         if ( !response.getErrors().isEmpty() ) {
2283             Map<String, String> headers = responseHandler.getResponseHeaders();
2284
2285             MultiObjectDeleteException ex = new MultiObjectDeleteException(
2286                     response.getErrors(),
2287                     response.getDeletedObjects());
2288
2289             ex.setStatusCode(200);
2290             ex.setRequestId(headers.get(Headers.REQUEST_ID));
2291             ex.setExtendedRequestId(headers.get(Headers.EXTENDED_REQUEST_ID));
2292             ex.setCloudFrontId(headers.get(Headers.CLOUD_FRONT_ID));
2293             ex.setProxyHost(clientConfiguration.getProxyHost());
2294
2295             throw ex;
2296         }
2297         DeleteObjectsResult result = new DeleteObjectsResult(response.getDeletedObjects(), response.isRequesterCharged());
2298
2299         return result;
2300     }
2301
2302     @Override
2303     public void deleteVersion(String bucketName, String key, String versionId)
2304             throws SdkClientException, AmazonServiceException {
2305         deleteVersion(new DeleteVersionRequest(bucketName, key, versionId));
2306     }
2307
2308     @Override
2309     public void deleteVersion(DeleteVersionRequest deleteVersionRequest)
2310             throws SdkClientException, AmazonServiceException {
2311         deleteVersionRequest = beforeClientExecution(deleteVersionRequest);
2312         rejectNull(deleteVersionRequest,
2313             "The delete version request object must be specified when deleting a version");
2314
2315         String bucketName = deleteVersionRequest.getBucketName();
2316         String key = deleteVersionRequest.getKey();
2317         String versionId = deleteVersionRequest.getVersionId();
2318
2319         rejectNull(bucketName, "The bucket name must be specified when deleting a version");
2320         rejectNull(key, "The key must be specified when deleting a version");
2321         rejectNull(versionId, "The version ID must be specified when deleting a version");
2322
2323         Request<DeleteVersionRequest> request = createRequest(bucketName, key, deleteVersionRequest, HttpMethodName.DELETE);
2324         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteObject");
2325         if (versionId != null) request.addParameter("versionId", versionId);
2326
2327         if (deleteVersionRequest.getMfa() != null) {
2328             populateRequestWithMfaDetails(request, deleteVersionRequest.getMfa());
2329         }
2330         if (deleteVersionRequest.getBypassGovernanceRetention()) {
2331             request.addHeader(Headers.BYPASS_GOVERNANCE_RETENTION, "true");
2332         }
2333
2334         invoke(request, voidResponseHandler, bucketName, key);
2335     }
2336
2337     @Override
2338     public void setBucketVersioningConfiguration(SetBucketVersioningConfigurationRequest setBucketVersioningConfigurationRequest)
2339         throws SdkClientException, AmazonServiceException {
2340         setBucketVersioningConfigurationRequest = beforeClientExecution(setBucketVersioningConfigurationRequest);
2341         rejectNull(setBucketVersioningConfigurationRequest,
2342             "The SetBucketVersioningConfigurationRequest object must be specified when setting versioning configuration");
2343
2344         String bucketName = setBucketVersioningConfigurationRequest.getBucketName();
2345         BucketVersioningConfiguration versioningConfiguration = setBucketVersioningConfigurationRequest.getVersioningConfiguration();
2346
2347         rejectNull(bucketName,
2348             "The bucket name parameter must be specified when setting versioning configuration");
2349         rejectNull(versioningConfiguration,
2350             "The bucket versioning parameter must be specified when setting versioning configuration");
2351         if (versioningConfiguration.isMfaDeleteEnabled() != null) {
2352             rejectNull(setBucketVersioningConfigurationRequest.getMfa(),
2353                 "The MFA parameter must be specified when changing MFA Delete status in the versioning configuration");
2354         }
2355
2356         Request<SetBucketVersioningConfigurationRequest> request = createRequest(bucketName, null, setBucketVersioningConfigurationRequest, HttpMethodName.PUT);
2357         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketVersioning");
2358         request.addParameter("versioning", null);
2359
2360         if (versioningConfiguration.isMfaDeleteEnabled() != null) {
2361             if (setBucketVersioningConfigurationRequest.getMfa() != null) {
2362                 populateRequestWithMfaDetails(request, setBucketVersioningConfigurationRequest.getMfa());
2363             }
2364         }
2365
2366         byte[] bytes = bucketConfigurationXmlFactory.convertToXmlByteArray(versioningConfiguration);
2367         request.setContent(new ByteArrayInputStream(bytes));
2368
2369         invoke(request, voidResponseHandler, bucketName, null);
2370     }
2371
2372     @Override
2373     public BucketVersioningConfiguration getBucketVersioningConfiguration(String bucketName)
2374             throws SdkClientException, AmazonServiceException {
2375         return getBucketVersioningConfiguration(new GetBucketVersioningConfigurationRequest(bucketName));
2376     }
2377
2378     @Override
2379     public BucketVersioningConfiguration getBucketVersioningConfiguration(GetBucketVersioningConfigurationRequest getBucketVersioningConfigurationRequest)
2380             throws SdkClientException, AmazonServiceException {
2381         getBucketVersioningConfigurationRequest = beforeClientExecution(getBucketVersioningConfigurationRequest);
2382         rejectNull(getBucketVersioningConfigurationRequest, "The request object parameter getBucketVersioningConfigurationRequest must be specified.");
2383         String bucketName = getBucketVersioningConfigurationRequest.getBucketName();
2384         rejectNull(bucketName,
2385                 "The bucket name parameter must be specified when querying versioning configuration");
2386
2387         Request<GetBucketVersioningConfigurationRequest> request = createRequest(bucketName, null, getBucketVersioningConfigurationRequest, HttpMethodName.GET);
2388         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketVersioning");
2389         request.addParameter("versioning", null);
2390
2391         return invoke(request, new Unmarshallers.BucketVersioningConfigurationUnmarshaller(), bucketName, null);
2392     }
2393
2394     @Override
2395     public BucketWebsiteConfiguration getBucketWebsiteConfiguration(String bucketName)
2396             throws SdkClientException, AmazonServiceException {
2397         return getBucketWebsiteConfiguration(new GetBucketWebsiteConfigurationRequest(bucketName));
2398     }
2399
2400     @Override
2401     public BucketWebsiteConfiguration getBucketWebsiteConfiguration(GetBucketWebsiteConfigurationRequest getBucketWebsiteConfigurationRequest)
2402             throws SdkClientException, AmazonServiceException {
2403         getBucketWebsiteConfigurationRequest = beforeClientExecution(getBucketWebsiteConfigurationRequest);
2404         rejectNull(getBucketWebsiteConfigurationRequest, "The request object parameter getBucketWebsiteConfigurationRequest must be specified.");
2405         String bucketName = getBucketWebsiteConfigurationRequest.getBucketName();
2406         rejectNull(bucketName,
2407             "The bucket name parameter must be specified when requesting a bucket's website configuration");
2408
2409         Request<GetBucketWebsiteConfigurationRequest> request = createRequest(bucketName, null, getBucketWebsiteConfigurationRequest, HttpMethodName.GET);
2410         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketWebsite");
2411         request.addParameter("website", null);
2412         request.addHeader("Content-Type", "application/xml");
2413
2414         try {
2415             return invoke(request, new Unmarshallers.BucketWebsiteConfigurationUnmarshaller(), bucketName, null);
2416         } catch (AmazonServiceException ase) {
2417             if (ase.getStatusCode() == 404) return null;
2418             throw ase;
2419         }
2420     }
2421
2422     @Override
2423     public BucketLifecycleConfiguration getBucketLifecycleConfiguration(String bucketName) {
2424         return getBucketLifecycleConfiguration(new GetBucketLifecycleConfigurationRequest(bucketName));
2425     }
2426
2427     @Override
2428     public BucketLifecycleConfiguration getBucketLifecycleConfiguration(GetBucketLifecycleConfigurationRequest getBucketLifecycleConfigurationRequest) {
2429         getBucketLifecycleConfigurationRequest = beforeClientExecution(getBucketLifecycleConfigurationRequest);
2430         rejectNull(getBucketLifecycleConfigurationRequest, "The request object pamameter getBucketLifecycleConfigurationRequest must be specified.");
2431         String bucketName = getBucketLifecycleConfigurationRequest.getBucketName();
2432         rejectNull(bucketName, "The bucket name must be specifed when retrieving the bucket lifecycle configuration.");
2433
2434         Request<GetBucketLifecycleConfigurationRequest> request = createRequest(bucketName, null, getBucketLifecycleConfigurationRequest, HttpMethodName.GET);
2435         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketLifecycleConfiguration");
2436         request.addParameter("lifecycle", null);
2437
2438         try {
2439             return invoke(request, new Unmarshallers.BucketLifecycleConfigurationUnmarshaller(), bucketName, null);
2440         } catch (AmazonServiceException ase) {
2441             switch (ase.getStatusCode()) {
2442             case 404:
2443                 return null;
2444             default:
2445                 throw ase;
2446             }
2447         }
2448     }
2449
2450     @Override
2451     public void setBucketLifecycleConfiguration(String bucketName, BucketLifecycleConfiguration bucketLifecycleConfiguration) {
2452         setBucketLifecycleConfiguration(new SetBucketLifecycleConfigurationRequest(bucketName, bucketLifecycleConfiguration));
2453     }
2454
2455     @Override
2456     public void setBucketLifecycleConfiguration(
2457             SetBucketLifecycleConfigurationRequest setBucketLifecycleConfigurationRequest) {
2458         setBucketLifecycleConfigurationRequest = beforeClientExecution(setBucketLifecycleConfigurationRequest);
2459         rejectNull(setBucketLifecycleConfigurationRequest,
2460                 "The set bucket lifecycle configuration request object must be specified.");
2461
2462         String bucketName = setBucketLifecycleConfigurationRequest.getBucketName();
2463         BucketLifecycleConfiguration bucketLifecycleConfiguration = setBucketLifecycleConfigurationRequest.getLifecycleConfiguration();
2464
2465         rejectNull(bucketName,
2466                 "The bucket name parameter must be specified when setting bucket lifecycle configuration.");
2467         rejectNull(bucketLifecycleConfiguration,
2468                 "The lifecycle configuration parameter must be specified when setting bucket lifecycle configuration.");
2469
2470         Request<SetBucketLifecycleConfigurationRequest> request = createRequest(bucketName, null, setBucketLifecycleConfigurationRequest, HttpMethodName.PUT);
2471         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketLifecycleConfiguration");
2472         request.addParameter("lifecycle", null);
2473
2474         byte[] content = new BucketConfigurationXmlFactory().convertToXmlByteArray(bucketLifecycleConfiguration);
2475         request.addHeader("Content-Length", String.valueOf(content.length));
2476         request.addHeader("Content-Type", "application/xml");
2477         request.setContent(new ByteArrayInputStream(content));
2478         try {
2479             byte[] md5 = Md5Utils.computeMD5Hash(content);
2480             String md5Base64 = BinaryUtils.toBase64(md5);
2481             request.addHeader("Content-MD5", md5Base64);
2482         } catch ( Exception e ) {
2483             throw new SdkClientException("Couldn't compute md5 sum", e);
2484         }
2485
2486         invoke(request, voidResponseHandler, bucketName, null);
2487     }
2488
2489     @Override
2490     public void deleteBucketLifecycleConfiguration(String bucketName) {
2491         deleteBucketLifecycleConfiguration(new DeleteBucketLifecycleConfigurationRequest(bucketName));
2492     }
2493
2494     @Override
2495     public void deleteBucketLifecycleConfiguration(
2496             DeleteBucketLifecycleConfigurationRequest deleteBucketLifecycleConfigurationRequest) {
2497         deleteBucketLifecycleConfigurationRequest = beforeClientExecution(deleteBucketLifecycleConfigurationRequest);
2498         rejectNull(deleteBucketLifecycleConfigurationRequest,
2499                 "The delete bucket lifecycle configuration request object must be specified.");
2500
2501         String bucketName = deleteBucketLifecycleConfigurationRequest.getBucketName();
2502         rejectNull(bucketName,
2503                 "The bucket name parameter must be specified when deleting bucket lifecycle configuration.");
2504
2505         Request<DeleteBucketLifecycleConfigurationRequest> request = createRequest(bucketName, null, deleteBucketLifecycleConfigurationRequest, HttpMethodName.DELETE);
2506         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteBucketLifecycle");
2507         request.addParameter("lifecycle", null);
2508
2509         invoke(request, voidResponseHandler, bucketName, null);
2510     }
2511
2512     @Override
2513     public BucketCrossOriginConfiguration getBucketCrossOriginConfiguration(String bucketName) {
2514         return getBucketCrossOriginConfiguration(new GetBucketCrossOriginConfigurationRequest(bucketName));
2515     }
2516
2517     @Override
2518     public BucketCrossOriginConfiguration getBucketCrossOriginConfiguration(GetBucketCrossOriginConfigurationRequest getBucketCrossOriginConfigurationRequest) {
2519         getBucketCrossOriginConfigurationRequest = beforeClientExecution(getBucketCrossOriginConfigurationRequest);
2520         rejectNull(getBucketCrossOriginConfigurationRequest, "The request object parameter getBucketCrossOriginConfigurationRequest must be specified.");
2521         String bucketName = getBucketCrossOriginConfigurationRequest.getBucketName();
2522         rejectNull(bucketName, "The bucket name must be specified when retrieving the bucket cross origin configuration.");
2523
2524         Request<GetBucketCrossOriginConfigurationRequest> request = createRequest(bucketName, null, getBucketCrossOriginConfigurationRequest, HttpMethodName.GET);
2525         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketCors");
2526         request.addParameter("cors", null);
2527
2528         try {
2529             return invoke(request, new Unmarshallers.BucketCrossOriginConfigurationUnmarshaller(), bucketName, null);
2530         } catch (AmazonServiceException ase) {
2531             switch (ase.getStatusCode()) {
2532             case 404:
2533                 return null;
2534             default:
2535                 throw ase;
2536             }
2537         }
2538     }
2539
2540     @Override
2541     public void setBucketCrossOriginConfiguration(String bucketName, BucketCrossOriginConfiguration bucketCrossOriginConfiguration) {
2542         setBucketCrossOriginConfiguration(new SetBucketCrossOriginConfigurationRequest(bucketName, bucketCrossOriginConfiguration));
2543     }
2544
2545     @Override
2546     public void setBucketCrossOriginConfiguration(
2547             SetBucketCrossOriginConfigurationRequest setBucketCrossOriginConfigurationRequest) {
2548         setBucketCrossOriginConfigurationRequest = beforeClientExecution(setBucketCrossOriginConfigurationRequest);
2549         rejectNull(setBucketCrossOriginConfigurationRequest,
2550                 "The set bucket cross origin configuration request object must be specified.");
2551
2552         String bucketName = setBucketCrossOriginConfigurationRequest.getBucketName();
2553         BucketCrossOriginConfiguration bucketCrossOriginConfiguration = setBucketCrossOriginConfigurationRequest.getCrossOriginConfiguration();
2554
2555         rejectNull(bucketName,
2556                 "The bucket name parameter must be specified when setting bucket cross origin configuration.");
2557         rejectNull(bucketCrossOriginConfiguration,
2558                 "The cross origin configuration parameter must be specified when setting bucket cross origin configuration.");
2559
2560         Request<SetBucketCrossOriginConfigurationRequest> request = createRequest(bucketName, null, setBucketCrossOriginConfigurationRequest, HttpMethodName.PUT);
2561         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketCors");
2562         request.addParameter("cors", null);
2563
2564         byte[] content = new BucketConfigurationXmlFactory().convertToXmlByteArray(bucketCrossOriginConfiguration);
2565         request.addHeader("Content-Length", String.valueOf(content.length));
2566         request.addHeader("Content-Type", "application/xml");
2567         request.setContent(new ByteArrayInputStream(content));
2568         try {
2569             byte[] md5 = Md5Utils.computeMD5Hash(content);
2570             String md5Base64 = BinaryUtils.toBase64(md5);
2571             request.addHeader("Content-MD5", md5Base64);
2572         } catch ( Exception e ) {
2573             throw new SdkClientException("Couldn't compute md5 sum", e);
2574         }
2575
2576         invoke(request, voidResponseHandler, bucketName, null);
2577     }
2578
2579     @Override
2580     public void deleteBucketCrossOriginConfiguration(String bucketName) {
2581         deleteBucketCrossOriginConfiguration(new DeleteBucketCrossOriginConfigurationRequest(bucketName));
2582       }
2583
2584     @Override
2585     public void deleteBucketCrossOriginConfiguration(
2586             DeleteBucketCrossOriginConfigurationRequest deleteBucketCrossOriginConfigurationRequest) {
2587         deleteBucketCrossOriginConfigurationRequest = beforeClientExecution(deleteBucketCrossOriginConfigurationRequest);
2588         rejectNull(deleteBucketCrossOriginConfigurationRequest,
2589                 "The delete bucket cross origin configuration request object must be specified.");
2590
2591         String bucketName = deleteBucketCrossOriginConfigurationRequest.getBucketName();
2592         rejectNull(bucketName,
2593                 "The bucket name parameter must be specified when deleting bucket cross origin configuration.");
2594
2595         Request<DeleteBucketCrossOriginConfigurationRequest> request = createRequest(bucketName, null, deleteBucketCrossOriginConfigurationRequest, HttpMethodName.DELETE);
2596         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteBucketCors");
2597         request.addParameter("cors", null);
2598         invoke(request, voidResponseHandler, bucketName, null);
2599     }
2600
2601     @Override
2602     public BucketTaggingConfiguration getBucketTaggingConfiguration(String bucketName) {
2603         return getBucketTaggingConfiguration(new GetBucketTaggingConfigurationRequest(bucketName));
2604     }
2605
2606     @Override
2607     public BucketTaggingConfiguration getBucketTaggingConfiguration(GetBucketTaggingConfigurationRequest getBucketTaggingConfigurationRequest) {
2608         getBucketTaggingConfigurationRequest = beforeClientExecution(getBucketTaggingConfigurationRequest);
2609         rejectNull(getBucketTaggingConfigurationRequest, "The request object parameter getBucketTaggingConfigurationRequest must be specifed.");
2610         String bucketName = getBucketTaggingConfigurationRequest.getBucketName();
2611         rejectNull(bucketName, "The bucket name must be specified when retrieving the bucket tagging configuration.");
2612
2613         Request<GetBucketTaggingConfigurationRequest> request = createRequest(bucketName, null, getBucketTaggingConfigurationRequest, HttpMethodName.GET);
2614         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketTagging");
2615         request.addParameter("tagging", null);
2616
2617         try {
2618             return invoke(request, new Unmarshallers.BucketTaggingConfigurationUnmarshaller(), bucketName, null);
2619         } catch (AmazonServiceException ase) {
2620             switch (ase.getStatusCode()) {
2621             case 404:
2622                 return null;
2623             default:
2624                 throw ase;
2625             }
2626         }
2627     }
2628
2629     @Override
2630     public void setBucketTaggingConfiguration(String bucketName, BucketTaggingConfiguration bucketTaggingConfiguration) {
2631         setBucketTaggingConfiguration(new SetBucketTaggingConfigurationRequest(bucketName, bucketTaggingConfiguration));
2632     }
2633
2634     @Override
2635     public void setBucketTaggingConfiguration(
2636             SetBucketTaggingConfigurationRequest setBucketTaggingConfigurationRequest) {
2637         setBucketTaggingConfigurationRequest = beforeClientExecution(setBucketTaggingConfigurationRequest);
2638         rejectNull(setBucketTaggingConfigurationRequest,
2639                 "The set bucket tagging configuration request object must be specified.");
2640
2641         String bucketName = setBucketTaggingConfigurationRequest.getBucketName();
2642         BucketTaggingConfiguration bucketTaggingConfiguration = setBucketTaggingConfigurationRequest.getTaggingConfiguration();
2643
2644         rejectNull(bucketName,
2645                 "The bucket name parameter must be specified when setting bucket tagging configuration.");
2646         rejectNull(bucketTaggingConfiguration,
2647                 "The tagging configuration parameter must be specified when setting bucket tagging configuration.");
2648
2649         Request<SetBucketTaggingConfigurationRequest> request = createRequest(bucketName, null, setBucketTaggingConfigurationRequest, HttpMethodName.PUT);
2650         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketTagging");
2651         request.addParameter("tagging", null);
2652
2653         byte[] content = new BucketConfigurationXmlFactory().convertToXmlByteArray(bucketTaggingConfiguration);
2654         request.addHeader("Content-Length", String.valueOf(content.length));
2655         request.addHeader("Content-Type", "application/xml");
2656         request.setContent(new ByteArrayInputStream(content));
2657         try {
2658             byte[] md5 = Md5Utils.computeMD5Hash(content);
2659             String md5Base64 = BinaryUtils.toBase64(md5);
2660             request.addHeader("Content-MD5", md5Base64);
2661         } catch ( Exception e ) {
2662             throw new SdkClientException("Couldn't compute md5 sum", e);
2663         }
2664
2665         invoke(request, voidResponseHandler, bucketName, null);
2666     }
2667
2668     @Override
2669     public void deleteBucketTaggingConfiguration(String bucketName) {
2670         deleteBucketTaggingConfiguration(new DeleteBucketTaggingConfigurationRequest(bucketName));
2671     }
2672
2673     @Override
2674     public void deleteBucketTaggingConfiguration(
2675             DeleteBucketTaggingConfigurationRequest deleteBucketTaggingConfigurationRequest) {
2676         deleteBucketTaggingConfigurationRequest = beforeClientExecution(deleteBucketTaggingConfigurationRequest);
2677         rejectNull(deleteBucketTaggingConfigurationRequest,
2678                 "The delete bucket tagging configuration request object must be specified.");
2679
2680         String bucketName = deleteBucketTaggingConfigurationRequest.getBucketName();
2681         rejectNull(bucketName,
2682                 "The bucket name parameter must be specified when deleting bucket tagging configuration.");
2683
2684         Request<DeleteBucketTaggingConfigurationRequest> request = createRequest(bucketName, null, deleteBucketTaggingConfigurationRequest, HttpMethodName.DELETE);
2685         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteBucketTagging");
2686         request.addParameter("tagging", null);
2687
2688         invoke(request, voidResponseHandler, bucketName, null);
2689     }
2690
2691     @Override
2692     public void setBucketWebsiteConfiguration(String bucketName, BucketWebsiteConfiguration configuration)
2693             throws SdkClientException, AmazonServiceException {
2694         setBucketWebsiteConfiguration(new SetBucketWebsiteConfigurationRequest(bucketName, configuration));
2695     }
2696
2697     @Override
2698     public void setBucketWebsiteConfiguration(SetBucketWebsiteConfigurationRequest setBucketWebsiteConfigurationRequest)
2699            throws SdkClientException, AmazonServiceException {
2700         setBucketWebsiteConfigurationRequest = beforeClientExecution(setBucketWebsiteConfigurationRequest);
2701         String bucketName = setBucketWebsiteConfigurationRequest.getBucketName();
2702         BucketWebsiteConfiguration configuration = setBucketWebsiteConfigurationRequest.getConfiguration();
2703
2704         rejectNull(bucketName,
2705                 "The bucket name parameter must be specified when setting a bucket's website configuration");
2706         rejectNull(configuration,
2707                 "The bucket website configuration parameter must be specified when setting a bucket's website configuration");
2708         if (configuration.getRedirectAllRequestsTo() == null) {
2709         rejectNull(configuration.getIndexDocumentSuffix(),
2710                 "The bucket website configuration parameter must specify the index document suffix when setting a bucket's website configuration");
2711         }
2712
2713         Request<SetBucketWebsiteConfigurationRequest> request = createRequest(bucketName, null, setBucketWebsiteConfigurationRequest, HttpMethodName.PUT);
2714         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketWebsite");
2715         request.addParameter("website", null);
2716         request.addHeader("Content-Type", "application/xml");
2717
2718         byte[] bytes = bucketConfigurationXmlFactory.convertToXmlByteArray(configuration);
2719         request.setContent(new ByteArrayInputStream(bytes));
2720
2721         invoke(request, voidResponseHandler, bucketName, null);
2722     }
2723
2724     @Override
2725     public void deleteBucketWebsiteConfiguration(String bucketName)
2726             throws SdkClientException, AmazonServiceException {
2727         deleteBucketWebsiteConfiguration(new DeleteBucketWebsiteConfigurationRequest(bucketName));
2728     }
2729
2730     @Override
2731     public void deleteBucketWebsiteConfiguration(DeleteBucketWebsiteConfigurationRequest deleteBucketWebsiteConfigurationRequest)
2732         throws SdkClientException, AmazonServiceException {
2733         deleteBucketWebsiteConfigurationRequest = beforeClientExecution(deleteBucketWebsiteConfigurationRequest);
2734         String bucketName = deleteBucketWebsiteConfigurationRequest.getBucketName();
2735
2736         rejectNull(bucketName,
2737             "The bucket name parameter must be specified when deleting a bucket's website configuration");
2738
2739         Request<DeleteBucketWebsiteConfigurationRequest> request = createRequest(bucketName, null, deleteBucketWebsiteConfigurationRequest, HttpMethodName.DELETE);
2740         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteBucketWebsite");
2741         request.addParameter("website", null);
2742         request.addHeader("Content-Type", "application/xml");
2743
2744         invoke(request, voidResponseHandler, bucketName, null);
2745     }
2746
2747     @Override
2748     public void setBucketNotificationConfiguration(String bucketName, BucketNotificationConfiguration bucketNotificationConfiguration)
2749         throws SdkClientException, AmazonServiceException {
2750         setBucketNotificationConfiguration(new SetBucketNotificationConfigurationRequest(bucketName, bucketNotificationConfiguration));
2751     }
2752
2753     @Override
2754     public void setBucketNotificationConfiguration(
2755             SetBucketNotificationConfigurationRequest setBucketNotificationConfigurationRequest)
2756             throws SdkClientException, AmazonServiceException {
2757         setBucketNotificationConfigurationRequest = beforeClientExecution(setBucketNotificationConfigurationRequest);
2758         rejectNull(setBucketNotificationConfigurationRequest,
2759                 "The set bucket notification configuration request object must be specified.");
2760
2761         String bucketName = setBucketNotificationConfigurationRequest.getBucketName();
2762         BucketNotificationConfiguration bucketNotificationConfiguration = setBucketNotificationConfigurationRequest.getNotificationConfiguration();
2763
2764         rejectNull(bucketName,
2765                 "The bucket name parameter must be specified when setting bucket notification configuration.");
2766         rejectNull(bucketNotificationConfiguration,
2767                 "The notification configuration parameter must be specified when setting bucket notification configuration.");
2768
2769         Request<SetBucketNotificationConfigurationRequest> request = createRequest(bucketName, null, setBucketNotificationConfigurationRequest, HttpMethodName.PUT);
2770         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketNotificationConfiguration");
2771         request.addParameter("notification", null);
2772
2773         byte[] bytes = bucketConfigurationXmlFactory.convertToXmlByteArray(bucketNotificationConfiguration);
2774         request.setContent(new ByteArrayInputStream(bytes));
2775
2776         invoke(request, voidResponseHandler, bucketName, null);
2777     }
2778
2779     @Override
2780     public BucketNotificationConfiguration getBucketNotificationConfiguration(String bucketName)
2781             throws SdkClientException, AmazonServiceException {
2782         return getBucketNotificationConfiguration(new GetBucketNotificationConfigurationRequest(bucketName));
2783     }
2784
2785     @Override
2786     public BucketNotificationConfiguration getBucketNotificationConfiguration(GetBucketNotificationConfigurationRequest getBucketNotificationConfigurationRequest)
2787             throws SdkClientException, AmazonServiceException {
2788         getBucketNotificationConfigurationRequest = beforeClientExecution(getBucketNotificationConfigurationRequest);
2789         rejectNull(getBucketNotificationConfigurationRequest,
2790                 "The bucket request parameter must be specified when querying notification configuration");
2791         String bucketName = getBucketNotificationConfigurationRequest.getBucketName();
2792         rejectNull(bucketName,
2793                 "The bucket request must specify a bucket name when querying notification configuration");
2794
2795         Request<GetBucketNotificationConfigurationRequest> request = createRequest(bucketName, null, getBucketNotificationConfigurationRequest, HttpMethodName.GET);
2796         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketNotificationConfiguration");
2797         request.addParameter("notification", null);
2798
2799         return invoke(request, BucketNotificationConfigurationStaxUnmarshaller.getInstance(), bucketName, null);
2800     }
2801
2802     @Override
2803     public BucketLoggingConfiguration getBucketLoggingConfiguration(String bucketName)
2804             throws SdkClientException, AmazonServiceException {
2805         return getBucketLoggingConfiguration(new GetBucketLoggingConfigurationRequest(bucketName));
2806     }
2807
2808     @Override
2809     public BucketLoggingConfiguration getBucketLoggingConfiguration(GetBucketLoggingConfigurationRequest getBucketLoggingConfigurationRequest)
2810             throws SdkClientException, AmazonServiceException {
2811         getBucketLoggingConfigurationRequest = beforeClientExecution(getBucketLoggingConfigurationRequest);
2812         rejectNull(getBucketLoggingConfigurationRequest, "The request object parameter getBucketLoggingConfigurationRequest must be specifed.");
2813         String bucketName = getBucketLoggingConfigurationRequest.getBucketName();
2814         rejectNull(bucketName,
2815                 "The bucket name parameter must be specified when requesting a bucket's logging status");
2816
2817         Request<GetBucketLoggingConfigurationRequest> request = createRequest(bucketName, null, getBucketLoggingConfigurationRequest, HttpMethodName.GET);
2818         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketLogging");
2819         request.addParameter("logging", null);
2820
2821         return invoke(request, new Unmarshallers.BucketLoggingConfigurationnmarshaller(), bucketName, null);
2822     }
2823
2824     @Override
2825     public void setBucketLoggingConfiguration(SetBucketLoggingConfigurationRequest setBucketLoggingConfigurationRequest)
2826             throws SdkClientException, AmazonServiceException {
2827         setBucketLoggingConfigurationRequest = beforeClientExecution(setBucketLoggingConfigurationRequest);
2828         rejectNull(setBucketLoggingConfigurationRequest,
2829             "The set bucket logging configuration request object must be specified when enabling server access logging");
2830
2831         String bucketName = setBucketLoggingConfigurationRequest.getBucketName();
2832         BucketLoggingConfiguration loggingConfiguration = setBucketLoggingConfigurationRequest.getLoggingConfiguration();
2833
2834         rejectNull(bucketName,
2835             "The bucket name parameter must be specified when enabling server access logging");
2836         rejectNull(loggingConfiguration,
2837             "The logging configuration parameter must be specified when enabling server access logging");
2838
2839         Request<SetBucketLoggingConfigurationRequest> request = createRequest(bucketName, null, setBucketLoggingConfigurationRequest, HttpMethodName.PUT);
2840         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketLogging");
2841         request.addParameter("logging", null);
2842
2843         byte[] bytes = bucketConfigurationXmlFactory.convertToXmlByteArray(loggingConfiguration);
2844         request.setContent(new ByteArrayInputStream(bytes));
2845
2846         invoke(request, voidResponseHandler, bucketName, null);
2847     }
2848
2849     @Override
2850     public BucketAccelerateConfiguration getBucketAccelerateConfiguration(
2851             String bucketName) throws AmazonServiceException,
2852             SdkClientException {
2853         return getBucketAccelerateConfiguration(new GetBucketAccelerateConfigurationRequest(
2854                 bucketName));
2855     }
2856
2857     @Override
2858     public BucketAccelerateConfiguration getBucketAccelerateConfiguration(
2859             GetBucketAccelerateConfigurationRequest getBucketAccelerateConfigurationRequest)
2860             throws AmazonServiceException, SdkClientException {
2861         getBucketAccelerateConfigurationRequest = beforeClientExecution(getBucketAccelerateConfigurationRequest);
2862         rejectNull(getBucketAccelerateConfigurationRequest, "getBucketAccelerateConfigurationRequest must be specified.");
2863         String bucketName = getBucketAccelerateConfigurationRequest.getBucketName();
2864         rejectNull(bucketName,
2865                 "The bucket name parameter must be specified when querying accelerate configuration");
2866
2867         Request<GetBucketAccelerateConfigurationRequest> request = createRequest(bucketName, null, getBucketAccelerateConfigurationRequest, HttpMethodName.GET);
2868         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketAccelerateConfiguration");
2869         request.addParameter("accelerate", null);
2870
2871         return invoke(request, new Unmarshallers.BucketAccelerateConfigurationUnmarshaller(), bucketName, null);
2872     }
2873
2874     @Override
2875     public void setBucketAccelerateConfiguration(String bucketName,
2876             BucketAccelerateConfiguration accelerateConfiguration)
2877             throws AmazonServiceException, SdkClientException {
2878         setBucketAccelerateConfiguration(new SetBucketAccelerateConfigurationRequest(
2879                 bucketName, accelerateConfiguration));
2880     }
2881
2882     @Override
2883     public void setBucketAccelerateConfiguration(
2884             SetBucketAccelerateConfigurationRequest setBucketAccelerateConfigurationRequest)
2885             throws AmazonServiceException, SdkClientException {
2886         setBucketAccelerateConfigurationRequest = beforeClientExecution(setBucketAccelerateConfigurationRequest);
2887
2888         rejectNull(setBucketAccelerateConfigurationRequest,
2889                 "setBucketAccelerateConfigurationRequest must be specified");
2890
2891         String bucketName = setBucketAccelerateConfigurationRequest.getBucketName();
2892         BucketAccelerateConfiguration accelerateConfiguration = setBucketAccelerateConfigurationRequest.getAccelerateConfiguration();
2893
2894         rejectNull(bucketName,
2895             "The bucket name parameter must be specified when setting accelerate configuration.");
2896         rejectNull(accelerateConfiguration,
2897             "The bucket accelerate configuration parameter must be specified.");
2898         rejectNull(accelerateConfiguration.getStatus(),
2899             "The status parameter must be specified when updating bucket accelerate configuration.");
2900
2901         Request<SetBucketAccelerateConfigurationRequest> request = createRequest(
2902                 bucketName, null, setBucketAccelerateConfigurationRequest,
2903                 HttpMethodName.PUT);
2904         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketAccelerateConfiguration");
2905         request.addParameter("accelerate", null);
2906
2907         byte[] bytes = bucketConfigurationXmlFactory.convertToXmlByteArray(accelerateConfiguration);
2908         request.setContent(new ByteArrayInputStream(bytes));
2909
2910         invoke(request, voidResponseHandler, bucketName, null);
2911     }
2912
2913     @Override
2914     public BucketPolicy getBucketPolicy(String bucketName)
2915             throws SdkClientException, AmazonServiceException {
2916         return getBucketPolicy(new GetBucketPolicyRequest(bucketName));
2917     }
2918
2919     @Override
2920     public void deleteBucketPolicy(String bucketName)
2921             throws SdkClientException, AmazonServiceException {
2922         deleteBucketPolicy(new DeleteBucketPolicyRequest(bucketName));
2923     }
2924
2925     @Override
2926     public BucketPolicy getBucketPolicy(
2927             GetBucketPolicyRequest getBucketPolicyRequest)
2928             throws SdkClientException, AmazonServiceException {
2929         getBucketPolicyRequest = beforeClientExecution(getBucketPolicyRequest);
2930         rejectNull(getBucketPolicyRequest,
2931             "The request object must be specified when getting a bucket policy");
2932
2933         String bucketName = getBucketPolicyRequest.getBucketName();
2934         rejectNull(bucketName,
2935             "The bucket name must be specified when getting a bucket policy");
2936
2937         Request<GetBucketPolicyRequest> request = createRequest(bucketName, null, getBucketPolicyRequest, HttpMethodName.GET);
2938         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketPolicy");
2939         request.addParameter("policy", null);
2940
2941         BucketPolicy result = new BucketPolicy();
2942         try {
2943             String policyText = invoke(request, new S3StringResponseHandler(), bucketName, null);
2944             result.setPolicyText(policyText);
2945             return result;
2946         } catch (AmazonServiceException ase) {
2947             /*
2948              * If we receive an error response telling us that no policy has
2949              * been set for this bucket, then instead of forcing the user to
2950              * deal with the exception, we'll just return an empty result. Any
2951              * other exceptions will be rethrown for the user to handle.
2952              */
2953             if (ase.getErrorCode().equals("NoSuchBucketPolicy")) return result;
2954             throw ase;
2955         }
2956     }
2957
2958
2959     @Override
2960     public void setBucketPolicy(String bucketName, String policyText)
2961             throws SdkClientException, AmazonServiceException {
2962         setBucketPolicy(new SetBucketPolicyRequest(bucketName, policyText));
2963     }
2964
2965     @Override
2966     public void setBucketPolicy(SetBucketPolicyRequest setBucketPolicyRequest)
2967             throws SdkClientException, AmazonServiceException {
2968         setBucketPolicyRequest = beforeClientExecution(setBucketPolicyRequest);
2969         rejectNull(setBucketPolicyRequest,
2970             "The request object must be specified when setting a bucket policy");
2971
2972         String bucketName = setBucketPolicyRequest.getBucketName();
2973         String policyText = setBucketPolicyRequest.getPolicyText();
2974
2975         rejectNull(bucketName,
2976             "The bucket name must be specified when setting a bucket policy");
2977         rejectNull(policyText,
2978             "The policy text must be specified when setting a bucket policy");
2979
2980         Request<SetBucketPolicyRequest> request = createRequest(bucketName, null, setBucketPolicyRequest, HttpMethodName.PUT);
2981         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketPolicy");
2982         request.addParameter("policy", null);
2983         request.setContent(new ByteArrayInputStream(ServiceUtils.toByteArray(policyText)));
2984
2985         if (setBucketPolicyRequest.getConfirmRemoveSelfBucketAccess() != null &&
2986             setBucketPolicyRequest.getConfirmRemoveSelfBucketAccess()) {
2987             request.addHeader(Headers.REMOVE_SELF_BUCKET_ACCESS, "true");
2988         }
2989
2990         invoke(request, voidResponseHandler, bucketName, null);
2991     }
2992
2993     @Override
2994     public void deleteBucketPolicy(
2995             DeleteBucketPolicyRequest deleteBucketPolicyRequest)
2996             throws SdkClientException, AmazonServiceException {
2997         deleteBucketPolicyRequest = beforeClientExecution(deleteBucketPolicyRequest);
2998         rejectNull(deleteBucketPolicyRequest,
2999             "The request object must be specified when deleting a bucket policy");
3000
3001         String bucketName = deleteBucketPolicyRequest.getBucketName();
3002         rejectNull(bucketName,
3003             "The bucket name must be specified when deleting a bucket policy");
3004
3005         Request<DeleteBucketPolicyRequest> request = createRequest(bucketName, null, deleteBucketPolicyRequest, HttpMethodName.DELETE);
3006         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteBucketPolicy");
3007         request.addParameter("policy", null);
3008
3009         invoke(request, voidResponseHandler, bucketName, null);
3010     }
3011
3012     @Override
3013     public DeleteBucketEncryptionResult deleteBucketEncryption(String bucketName) throws SdkClientException {
3014         return deleteBucketEncryption(new DeleteBucketEncryptionRequest().withBucketName(bucketName));
3015     }
3016
3017     @Override
3018     public DeleteBucketEncryptionResult deleteBucketEncryption(DeleteBucketEncryptionRequest deleteBucketEncryptionRequest)
3019         throws SdkClientException {
3020         deleteBucketEncryptionRequest = beforeClientExecution(deleteBucketEncryptionRequest);
3021         rejectNull(deleteBucketEncryptionRequest,
3022                    "The request object must be specified when deleting a bucket encryption configuration");
3023
3024         String bucketName = deleteBucketEncryptionRequest.getBucketName();
3025         rejectNull(bucketName,
3026                    "The bucket name must be specified when deleting a bucket encryption configuration");
3027
3028         Request<DeleteBucketEncryptionRequest> request =
3029             createRequest(bucketName, null, deleteBucketEncryptionRequest, HttpMethodName.DELETE);
3030         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteBucketEncryption");
3031         request.addParameter("encryption", null);
3032
3033         return invoke(request, new Unmarshallers.DeleteBucketEncryptionUnmarshaller(), bucketName, null);
3034     }
3035
3036     @Override
3037     public GetBucketEncryptionResult getBucketEncryption(String bucketName) throws SdkClientException {
3038         return getBucketEncryption(new GetBucketEncryptionRequest().withBucketName(bucketName));
3039     }
3040
3041     @Override
3042     public GetBucketEncryptionResult getBucketEncryption(GetBucketEncryptionRequest getBucketEncryptionRequest) throws SdkClientException {
3043         getBucketEncryptionRequest = beforeClientExecution(getBucketEncryptionRequest);
3044         rejectNull(getBucketEncryptionRequest,
3045                    "The bucket request parameter must be specified when querying encryption configuration");
3046         String bucketName = getBucketEncryptionRequest.getBucketName();
3047         rejectNull(bucketName,
3048                    "The bucket request must specify a bucket name when querying encryption configuration");
3049
3050         Request<GetBucketEncryptionRequest> request = createRequest(bucketName, null, getBucketEncryptionRequest, HttpMethodName.GET);
3051         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketEncryption");
3052         request.addParameter("encryption", null);
3053
3054         return invoke(request, GetBucketEncryptionStaxUnmarshaller.getInstance(), bucketName, null);
3055     }
3056
3057     @Override
3058     public SetBucketEncryptionResult setBucketEncryption(SetBucketEncryptionRequest setBucketEncryptionRequest)
3059         throws AmazonServiceException, SdkClientException {
3060         setBucketEncryptionRequest = beforeClientExecution(setBucketEncryptionRequest);
3061         rejectNull(setBucketEncryptionRequest,
3062                    "The request object must be specified.");
3063
3064         String bucketName = setBucketEncryptionRequest.getBucketName();
3065         ServerSideEncryptionConfiguration sseConfig = setBucketEncryptionRequest.getServerSideEncryptionConfiguration();
3066         rejectNull(bucketName,
3067                    "The bucket name parameter must be specified when setting bucket encryption configuration.");
3068         rejectNull(sseConfig,
3069                    "The SSE configuration parameter must be specified when setting bucket encryption configuration.");
3070
3071
3072         Request<SetBucketEncryptionRequest> request = createRequest(bucketName, null, setBucketEncryptionRequest, HttpMethodName.PUT);
3073         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketEncryption");
3074         request.addParameter("encryption", null);
3075
3076         byte[] bytes = bucketConfigurationXmlFactory.convertToXmlByteArray(sseConfig);
3077         request.setContent(new ByteArrayInputStream(bytes));
3078
3079         return invoke(request, new Unmarshallers.SetBucketEncryptionUnmarshaller(), bucketName, null);
3080     }
3081
3082     @Override
3083     public SetPublicAccessBlockResult setPublicAccessBlock(SetPublicAccessBlockRequest setPublicAccessBlockRequest) {
3084         setPublicAccessBlockRequest = beforeClientExecution(setPublicAccessBlockRequest);
3085         rejectNull(setPublicAccessBlockRequest, "The request object must be specified.");
3086
3087         String bucketName = setPublicAccessBlockRequest.getBucketName();
3088         PublicAccessBlockConfiguration config = setPublicAccessBlockRequest.getPublicAccessBlockConfiguration();
3089         rejectNull(bucketName,
3090                    "The bucket name parameter must be specified when setting public block configuration.");
3091         rejectNull(config,
3092                    "The PublicAccessBlockConfiguration parameter must be specified when setting public block");
3093
3094
3095         Request<SetPublicAccessBlockRequest> request = createRequest(bucketName, null, setPublicAccessBlockRequest, HttpMethodName.PUT);
3096         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutPublicAccessBlock");
3097         request.addParameter("publicAccessBlock", null);
3098
3099         byte[] bytes = bucketConfigurationXmlFactory.convertToXmlByteArray(config);
3100         request.setContent(new ByteArrayInputStream(bytes));
3101
3102         return invoke(request, new Unmarshallers.SetPublicAccessBlockUnmarshaller(), bucketName, null);
3103     }
3104
3105     @Override
3106     public GetPublicAccessBlockResult getPublicAccessBlock(GetPublicAccessBlockRequest getPublicAccessBlockRequest) {
3107         getPublicAccessBlockRequest = beforeClientExecution(getPublicAccessBlockRequest);
3108         rejectNull(getPublicAccessBlockRequest, "The request object must be specified.");
3109
3110         String bucketName = getPublicAccessBlockRequest.getBucketName();
3111         rejectNull(bucketName,
3112                    "The bucket name parameter must be specified when getting public block configuration.");
3113
3114
3115         Request<GetPublicAccessBlockRequest> request = createRequest(bucketName, null, getPublicAccessBlockRequest, HttpMethodName.GET);
3116         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetPublicAccessBlock");
3117         request.addParameter("publicAccessBlock", null);
3118
3119
3120         return invoke(request, GetPublicAccessBlockStaxUnmarshaller.getInstance(), bucketName, null);
3121     }
3122
3123     @Override
3124     public DeletePublicAccessBlockResult deletePublicAccessBlock(DeletePublicAccessBlockRequest deletePublicAccessBlockRequest) {
3125         deletePublicAccessBlockRequest = beforeClientExecution(deletePublicAccessBlockRequest);
3126         rejectNull(deletePublicAccessBlockRequest, "The request object must be specified.");
3127
3128         String bucketName = deletePublicAccessBlockRequest.getBucketName();
3129         rejectNull(bucketName,
3130                    "The bucket name parameter must be specified when deleting public block configuration.");
3131
3132
3133         Request<DeletePublicAccessBlockRequest> request = createRequest(bucketName, null, deletePublicAccessBlockRequest, HttpMethodName.DELETE);
3134         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeletePublicAccessBlock");
3135         request.addParameter("publicAccessBlock", null);
3136
3137
3138         return invoke(request, new Unmarshallers.DeletePublicAccessBlockUnmarshaller(), bucketName, null);
3139     }
3140
3141     @Override
3142     public GetBucketPolicyStatusResult getBucketPolicyStatus(GetBucketPolicyStatusRequest getBucketPolicyStatusRequest) {
3143         getBucketPolicyStatusRequest = beforeClientExecution(getBucketPolicyStatusRequest);
3144         rejectNull(getBucketPolicyStatusRequest, "The request object must be specified.");
3145
3146         String bucketName = getBucketPolicyStatusRequest.getBucketName();
3147         rejectNull(bucketName,
3148                    "The bucket name parameter must be specified when getting bucket policy status");
3149
3150
3151         Request<GetBucketPolicyStatusRequest> request = createRequest(bucketName, null, getBucketPolicyStatusRequest, HttpMethodName.GET);
3152         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketPolicyStatus");
3153         request.addParameter("policyStatus", null);
3154
3155
3156         return invoke(request, GetBucketPolicyStatusStaxUnmarshaller.getInstance(), bucketName, null);
3157     }
3158
3159     @Override
3160     public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest selectRequest) throws AmazonServiceException, SdkClientException {
3161         selectRequest = beforeClientExecution(selectRequest);
3162         rejectNull(selectRequest, "The request parameter must be specified");
3163
3164         rejectNull(selectRequest.getBucketName(), "The bucket name parameter must be specified when selecting object content.");
3165         rejectNull(selectRequest.getKey(), "The key parameter must be specified when selecting object content.");
3166
3167         Request<SelectObjectContentRequest> request = createRequest(selectRequest.getBucketName(), selectRequest.getKey(), selectRequest, HttpMethodName.POST);
3168         request.addParameter("select", null);
3169         request.addParameter("select-type", "2");
3170
3171         populateSSE_C(request, selectRequest.getSSECustomerKey());
3172
3173         setContent(request, RequestXmlFactory.convertToXmlByteArray(selectRequest), ContentType.APPLICATION_XML.toString(), true);
3174
3175         S3Object result = invoke(request, new S3ObjectResponseHandler(), selectRequest.getBucketName(), selectRequest.getKey());
3176
3177         // Hold a reference to this client while the InputStream is still
3178         // around - otherwise a finalizer in the HttpClient may reset the
3179         // underlying TCP connection out from under us.
3180         SdkFilterInputStream resultStream = new ServiceClientHolderInputStream(result.getObjectContent(), this);
3181
3182         return new SelectObjectContentResult().withPayload(new SelectObjectContentEventStream(resultStream));
3183     }
3184
3185     @Override
3186     public SetObjectLegalHoldResult setObjectLegalHold(SetObjectLegalHoldRequest setObjectLegalHoldRequest) {
3187         setObjectLegalHoldRequest = beforeClientExecution(setObjectLegalHoldRequest);
3188         rejectNull(setObjectLegalHoldRequest, "The request parameter must be specified");
3189
3190         String bucketName = setObjectLegalHoldRequest.getBucketName();
3191         String key = setObjectLegalHoldRequest.getKey();
3192
3193         rejectNull(bucketName, "The bucket name parameter must be specified when setting the object legal hold.");
3194         rejectNull(key, "The key parameter must be specified when setting the object legal hold.");
3195
3196         Request<SetObjectLegalHoldRequest> request = createRequest(bucketName, key, setObjectLegalHoldRequest, HttpMethodName.PUT);
3197         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutObjectLegalHold");
3198         setContent(request, new ObjectLockLegalHoldXmlFactory().convertToXmlByteArray(setObjectLegalHoldRequest.getLegalHold()),
3199                 ContentType.APPLICATION_XML.toString(), true);
3200         request.addParameter("legal-hold", null);
3201
3202         addParameterIfNotNull(request, "versionId", setObjectLegalHoldRequest.getVersionId());
3203         populateRequesterPaysHeader(request, setObjectLegalHoldRequest.isRequesterPays());
3204
3205         ResponseHeaderHandlerChain<SetObjectLegalHoldResult> responseHandler = new ResponseHeaderHandlerChain<SetObjectLegalHoldResult>(
3206                 new Unmarshallers.SetObjectLegalHoldResultUnmarshaller(),
3207                 new S3RequesterChargedHeaderHandler<SetObjectLegalHoldResult>());
3208
3209         return invoke(request, responseHandler, bucketName, key);
3210     }
3211
3212     @Override
3213     public GetObjectLegalHoldResult getObjectLegalHold(GetObjectLegalHoldRequest getObjectLegalHoldRequest) {
3214         getObjectLegalHoldRequest = beforeClientExecution(getObjectLegalHoldRequest);
3215         rejectNull(getObjectLegalHoldRequest, "The request parameter must be specified");
3216
3217         String bucketName = getObjectLegalHoldRequest.getBucketName();
3218         String key = getObjectLegalHoldRequest.getKey();
3219         rejectNull(bucketName, "The bucket name parameter must be specified when getting the object legal hold.");
3220         rejectNull(key, "The key parameter must be specified when getting the object legal hold.");
3221
3222         Request<GetObjectLegalHoldRequest> request = createRequest(bucketName, key, getObjectLegalHoldRequest, HttpMethodName.GET);
3223         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetObjectLegalHold");
3224         request.addParameter("legal-hold", null);
3225         addParameterIfNotNull(request, "versionId", getObjectLegalHoldRequest.getVersionId());
3226         populateRequesterPaysHeader(request, getObjectLegalHoldRequest.isRequesterPays());
3227
3228         return invoke(request, new Unmarshallers.GetObjectLegalHoldResultUnmarshaller(), bucketName, key);
3229     }
3230
3231     @Override
3232     public SetObjectLockConfigurationResult setObjectLockConfiguration(SetObjectLockConfigurationRequest setObjectLockConfigurationRequest) {
3233         setObjectLockConfigurationRequest = beforeClientExecution(setObjectLockConfigurationRequest);
3234         rejectNull(setObjectLockConfigurationRequest, "The request parameter must be specified");
3235
3236         String bucketName = setObjectLockConfigurationRequest.getBucketName();
3237         rejectNull(bucketName, "The bucket name parameter must be specified when setting the object lock configuration");
3238
3239         Request<SetObjectLockConfigurationRequest> request = createRequest(bucketName, null, setObjectLockConfigurationRequest, HttpMethodName.PUT);
3240         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutObjectLockConfiguration");
3241         request.addParameter("object-lock", null);
3242
3243         addHeaderIfNotNull(request, Headers.OBJECT_LOCK_TOKEN, setObjectLockConfigurationRequest.getToken());
3244         populateRequesterPaysHeader(request, setObjectLockConfigurationRequest.isRequesterPays());
3245
3246         setContent(request, new ObjectLockConfigurationXmlFactory().convertToXmlByteArray(setObjectLockConfigurationRequest.getObjectLockConfiguration()),
3247                 ContentType.APPLICATION_XML.toString(), true);
3248
3249         ResponseHeaderHandlerChain<SetObjectLockConfigurationResult> responseHandler = new ResponseHeaderHandlerChain<SetObjectLockConfigurationResult>(
3250                 new Unmarshallers.SetObjectLockConfigurationResultUnmarshaller(),
3251                 new S3RequesterChargedHeaderHandler<SetObjectLockConfigurationResult>());
3252
3253         return invoke(request, responseHandler, bucketName, null);
3254     }
3255
3256     @Override
3257     public GetObjectLockConfigurationResult getObjectLockConfiguration(GetObjectLockConfigurationRequest getObjectLockConfigurationRequest) {
3258         getObjectLockConfigurationRequest = beforeClientExecution(getObjectLockConfigurationRequest);
3259         rejectNull(getObjectLockConfigurationRequest, "The request parameter must be specified");
3260
3261         String bucketName = getObjectLockConfigurationRequest.getBucketName();
3262         rejectNull(bucketName, "The bucket name parameter must be specified when getting the object lock configuration");
3263
3264         Request<GetObjectLockConfigurationRequest> request = createRequest(bucketName, null, getObjectLockConfigurationRequest, HttpMethodName.GET);
3265         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetObjectLockConfiguration");
3266         request.addParameter("object-lock", null);
3267
3268         return invoke(request, new Unmarshallers.GetObjectLockConfigurationResultUnmarshaller(), bucketName, null);
3269     }
3270
3271     @Override
3272     public SetObjectRetentionResult setObjectRetention(SetObjectRetentionRequest setObjectRetentionRequest) {
3273         setObjectRetentionRequest = beforeClientExecution(setObjectRetentionRequest);
3274         rejectNull(setObjectRetentionRequest, "The request parameter must be specified");
3275
3276         String bucketName = setObjectRetentionRequest.getBucketName();
3277         String key = setObjectRetentionRequest.getKey();
3278
3279         rejectNull(bucketName, "The bucket name parameter must be specified when setting the object retention.");
3280         rejectNull(key, "The key parameter must be specified when setting the object retention.");
3281
3282         Request<SetObjectRetentionRequest> request = createRequest(bucketName, key, setObjectRetentionRequest, HttpMethodName.PUT);
3283         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutObjectRetention");
3284         request.addParameter("retention", null);
3285         if (setObjectRetentionRequest.getBypassGovernanceRetention()) {
3286             request.addHeader(Headers.BYPASS_GOVERNANCE_RETENTION, "true");
3287         }
3288         addParameterIfNotNull(request, "versionId", setObjectRetentionRequest.getVersionId());
3289         populateRequesterPaysHeader(request, setObjectRetentionRequest.isRequesterPays());
3290
3291         setContent(request, new ObjectLockRetentionXmlFactory().convertToXmlByteArray(setObjectRetentionRequest.getRetention()),
3292                 ContentType.APPLICATION_XML.toString(), true);
3293
3294         ResponseHeaderHandlerChain<SetObjectRetentionResult> responseHandler = new ResponseHeaderHandlerChain<SetObjectRetentionResult>(
3295                 new Unmarshallers.SetObjectRetentionResultUnmarshaller(),
3296                 new S3RequesterChargedHeaderHandler<SetObjectRetentionResult>());
3297
3298         return invoke(request, responseHandler, bucketName, key);
3299     }
3300
3301     @Override
3302     public GetObjectRetentionResult getObjectRetention(GetObjectRetentionRequest getObjectRetentionRequest) {
3303         getObjectRetentionRequest = beforeClientExecution(getObjectRetentionRequest);
3304         rejectNull(getObjectRetentionRequest, "The request parameter must be specified");
3305
3306         String bucketName = getObjectRetentionRequest.getBucketName();
3307         String key = getObjectRetentionRequest.getKey();
3308
3309         rejectNull(bucketName, "The bucket name parameter must be specified when getting the object retention.");
3310         rejectNull(key, "The key parameter must be specified when getting the object retention.");
3311
3312         Request<GetObjectRetentionRequest> request = createRequest(bucketName, key, getObjectRetentionRequest, HttpMethodName.GET);
3313         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetObjectRetention");
3314         request.addParameter("retention", null);
3315
3316         addParameterIfNotNull(request, "versionId", getObjectRetentionRequest.getVersionId());
3317         // Note: the model has a requester pays member on the request but no requester charged on the result...
3318         populateRequesterPaysHeader(request, getObjectRetentionRequest.isRequesterPays());
3319
3320         return invoke(request, new Unmarshallers.GetObjectRetentionResultUnmarshaller(), bucketName, key);
3321     }
3322
3323     @Override
3324     public URL generatePresignedUrl(String bucketName, String key, Date expiration)
3325             throws SdkClientException {
3326         return generatePresignedUrl(bucketName, key, expiration, HttpMethod.GET);
3327     }
3328
3329     @Override
3330     public URL generatePresignedUrl(String bucketName, String key, Date expiration, HttpMethod method)
3331             throws SdkClientException {
3332         GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, key, method);
3333         request.setExpiration(expiration);
3334
3335         return generatePresignedUrl(request);
3336     }
3337
3338     @Override
3339     public URL generatePresignedUrl(GeneratePresignedUrlRequest req) {
3340         rejectNull(req,
3341             "The request parameter must be specified when generating a pre-signed URL");
3342         req.rejectIllegalArguments();
3343
3344         final String bucketName = req.getBucketName();
3345         final String key = req.getKey();
3346
3347         if (req.getExpiration() == null) {
3348             req.setExpiration(
3349                     new Date(System.currentTimeMillis() + 1000 * 60 * 15));
3350         }
3351
3352         HttpMethodName httpMethod = HttpMethodName.valueOf(req.getMethod().toString());
3353
3354         // If the key starts with a slash character itself, the following method
3355         // will actually add another slash before the resource path to prevent
3356         // the HttpClient mistakenly treating the slash as a path delimiter.
3357         // For presigned request, we need to remember to remove this extra slash
3358         // before generating the URL.
3359         Request<GeneratePresignedUrlRequest> request = createRequest(
3360                 bucketName, key, req, httpMethod);
3361         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GeneratePresignedUrl");
3362
3363         addParameterIfNotNull(request, "versionId", req.getVersionId());
3364
3365         if (req.isZeroByteContent())
3366             request.setContent(new ByteArrayInputStream(new byte[0]));
3367
3368         for (Entry<String, String> entry : req.getRequestParameters().entrySet()) {
3369             request.addParameter(entry.getKey(), entry.getValue());
3370         }
3371
3372         addHeaderIfNotNull(request, Headers.CONTENT_TYPE, req.getContentType());
3373         addHeaderIfNotNull(request, Headers.CONTENT_MD5, req.getContentMd5());
3374
3375         // SSE-C
3376         populateSSE_C(request, req.getSSECustomerKey());
3377         // SSE
3378         addHeaderIfNotNull(request, Headers.SERVER_SIDE_ENCRYPTION,
3379                 req.getSSEAlgorithm());
3380         // SSE-KMS
3381         addHeaderIfNotNull(request,
3382                 Headers.SERVER_SIDE_ENCRYPTION_AWS_KMS_KEYID, req.getKmsCmkId());
3383
3384         // Custom headers that open up the possibility of supporting unexpected
3385         // cases.
3386         Map<String, String> customHeaders = req.getCustomRequestHeaders();
3387         if (customHeaders != null) {
3388             for (Map.Entry<String, String> e: customHeaders.entrySet()) {
3389                 request.addHeader(e.getKey(), e.getValue());
3390             }
3391         }
3392
3393         addResponseHeaderParameters(request, req.getResponseHeaders());
3394
3395         Signer signer = createSigner(request, bucketName, key);
3396
3397         if (signer instanceof Presigner) {
3398             // If we have a signer which knows how to presign requests,
3399             // delegate directly to it.
3400             ((Presigner) signer).presignRequest(
3401                     request,
3402                     CredentialUtils.getCredentialsProvider(request.getOriginalRequest(), awsCredentialsProvider).getCredentials(),
3403                     req.getExpiration()
3404             );
3405         } else {
3406             // Otherwise use the default presigning method, which is hardcoded
3407             // to use QueryStringSigner.
3408             presignRequest(
3409                 request,
3410                 req.getMethod(),
3411                 bucketName,
3412                 key,
3413                 req.getExpiration(),
3414                 null
3415             );
3416         }
3417
3418         // Remove the leading slash (if any) in the resource-path
3419         return ServiceUtils.convertRequestToUrl(request, true, false);
3420     }
3421
3422     @Override
3423     public void abortMultipartUpload(AbortMultipartUploadRequest abortMultipartUploadRequest)
3424             throws SdkClientException, AmazonServiceException {
3425         abortMultipartUploadRequest = beforeClientExecution(abortMultipartUploadRequest);
3426         rejectNull(abortMultipartUploadRequest,
3427             "The request parameter must be specified when aborting a multipart upload");
3428         rejectNull(abortMultipartUploadRequest.getBucketName(),
3429             "The bucket name parameter must be specified when aborting a multipart upload");
3430         rejectNull(abortMultipartUploadRequest.getKey(),
3431             "The key parameter must be specified when aborting a multipart upload");
3432         rejectNull(abortMultipartUploadRequest.getUploadId(),
3433             "The upload ID parameter must be specified when aborting a multipart upload");
3434
3435         String bucketName = abortMultipartUploadRequest.getBucketName();
3436         String key = abortMultipartUploadRequest.getKey();
3437
3438         Request<AbortMultipartUploadRequest> request = createRequest(bucketName, key, abortMultipartUploadRequest, HttpMethodName.DELETE);
3439         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "AbortMultipartUpload");
3440         request.addParameter("uploadId", abortMultipartUploadRequest.getUploadId());
3441         populateRequesterPaysHeader(request, abortMultipartUploadRequest.isRequesterPays());
3442
3443         invoke(request, voidResponseHandler, bucketName, key);
3444     }
3445
3446     @Override
3447     public CompleteMultipartUploadResult completeMultipartUpload(
3448             CompleteMultipartUploadRequest completeMultipartUploadRequest)
3449             throws SdkClientException, AmazonServiceException {
3450         completeMultipartUploadRequest = beforeClientExecution(completeMultipartUploadRequest);
3451         rejectNull(completeMultipartUploadRequest,
3452             "The request parameter must be specified when completing a multipart upload");
3453
3454         String bucketName = completeMultipartUploadRequest.getBucketName();
3455         String key = completeMultipartUploadRequest.getKey();
3456         String uploadId = completeMultipartUploadRequest.getUploadId();
3457         rejectNull(bucketName,
3458             "The bucket name parameter must be specified when completing a multipart upload");
3459         rejectNull(key,
3460             "The key parameter must be specified when completing a multipart upload");
3461         rejectNull(uploadId,
3462             "The upload ID parameter must be specified when completing a multipart upload");
3463         rejectNull(completeMultipartUploadRequest.getPartETags(),
3464             "The part ETags parameter must be specified when completing a multipart upload");
3465
3466         int retries = 0;
3467         CompleteMultipartUploadHandler handler;
3468         do {
3469             Request<CompleteMultipartUploadRequest> request = createRequest(bucketName, key, completeMultipartUploadRequest, HttpMethodName.POST);
3470             request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "CompleteMultipartUpload");
3471             request.addParameter("uploadId", uploadId);
3472
3473             populateRequesterPaysHeader(request, completeMultipartUploadRequest.isRequesterPays());
3474
3475             byte[] xml = RequestXmlFactory.convertToXmlByteArray(completeMultipartUploadRequest.getPartETags());
3476             request.addHeader("Content-Type", "application/xml");
3477             request.addHeader("Content-Length", String.valueOf(xml.length));
3478
3479             request.setContent(new ByteArrayInputStream(xml));
3480
3481             @SuppressWarnings("unchecked")
3482             ResponseHeaderHandlerChain<CompleteMultipartUploadHandler> responseHandler = new ResponseHeaderHandlerChain<CompleteMultipartUploadHandler>(
3483                     // xml payload unmarshaller
3484                     new Unmarshallers.CompleteMultipartUploadResultUnmarshaller(),
3485                     // header handlers
3486                     new ServerSideEncryptionHeaderHandler<CompleteMultipartUploadHandler>(),
3487                     new ObjectExpirationHeaderHandler<CompleteMultipartUploadHandler>(),
3488                     new S3VersionHeaderHandler<CompleteMultipartUploadHandler>(),
3489                     new S3RequesterChargedHeaderHandler<CompleteMultipartUploadHandler>());
3490             handler = invoke(request, responseHandler, bucketName, key);
3491             if (handler.getCompleteMultipartUploadResult() != null) {
3492                 return handler.getCompleteMultipartUploadResult();
3493             }
3494         } while (shouldRetryCompleteMultipartUpload(completeMultipartUploadRequest,
3495                 handler.getAmazonS3Exception(), retries++));
3496
3497         throw handler.getAmazonS3Exception();
3498     }
3499
3500     private boolean shouldRetryCompleteMultipartUpload(AmazonWebServiceRequest originalRequest,
3501                                                        AmazonS3Exception exception,
3502                                                        int retriesAttempted) {
3503
3504         final RetryPolicy retryPolicy = clientConfiguration.getRetryPolicy();
3505
3506         if (retryPolicy == null || retryPolicy.getRetryCondition() == null) {
3507             return false;
3508         }
3509
3510         if (retryPolicy == PredefinedRetryPolicies.NO_RETRY_POLICY) {
3511             return false;
3512         }
3513
3514         return completeMultipartUploadRetryCondition.shouldRetry
3515                 (originalRequest, exception, retriesAttempted);
3516     }
3517
3518     @Override
3519     public InitiateMultipartUploadResult initiateMultipartUpload(
3520             InitiateMultipartUploadRequest initiateMultipartUploadRequest)
3521             throws SdkClientException, AmazonServiceException {
3522         initiateMultipartUploadRequest = beforeClientExecution(initiateMultipartUploadRequest);
3523         rejectNull(initiateMultipartUploadRequest,
3524             "The request parameter must be specified when initiating a multipart upload");
3525
3526         rejectNull(initiateMultipartUploadRequest.getBucketName(),
3527             "The bucket name parameter must be specified when initiating a multipart upload");
3528         rejectNull(initiateMultipartUploadRequest.getKey(),
3529             "The key parameter must be specified when initiating a multipart upload");
3530
3531         Request<InitiateMultipartUploadRequest> request = createRequest(initiateMultipartUploadRequest.getBucketName(), initiateMultipartUploadRequest.getKey(), initiateMultipartUploadRequest, HttpMethodName.POST);
3532         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "CreateMultipartUpload");
3533         request.addParameter("uploads", null);
3534
3535         if (initiateMultipartUploadRequest.getStorageClass() != null)
3536             request.addHeader(Headers.STORAGE_CLASS, initiateMultipartUploadRequest.getStorageClass().toString());
3537
3538         if (initiateMultipartUploadRequest.getRedirectLocation() != null) {
3539             request.addHeader(Headers.REDIRECT_LOCATION, initiateMultipartUploadRequest.getRedirectLocation());
3540         }
3541
3542         if ( initiateMultipartUploadRequest.getAccessControlList() != null ) {
3543             addAclHeaders(request, initiateMultipartUploadRequest.getAccessControlList());
3544         } else if ( initiateMultipartUploadRequest.getCannedACL() != null ) {
3545             request.addHeader(Headers.S3_CANNED_ACL, initiateMultipartUploadRequest.getCannedACL().toString());
3546         }
3547
3548         if (initiateMultipartUploadRequest.objectMetadata != null) {
3549             populateRequestMetadata(request, initiateMultipartUploadRequest.objectMetadata);
3550         }
3551
3552         populateRequesterPaysHeader(request, initiateMultipartUploadRequest.isRequesterPays());
3553
3554         // Populate the SSE-C parameters to the request header
3555         populateSSE_C(request, initiateMultipartUploadRequest.getSSECustomerKey());
3556
3557         // Populate the SSE AWS KMS parameters to the request header
3558         populateSSE_KMS(request,
3559                 initiateMultipartUploadRequest.getSSEAwsKeyManagementParams());
3560
3561         addHeaderIfNotNull(request, Headers.S3_TAGGING, urlEncodeTags(initiateMultipartUploadRequest.getTagging()));
3562
3563         populateObjectLockHeaders(request, initiateMultipartUploadRequest.getObjectLockMode(), initiateMultipartUploadRequest.getObjectLockRetainUntilDate(),
3564                 initiateMultipartUploadRequest.getObjectLockLegalHoldStatus());
3565
3566         // Be careful that we don't send the object's total size as the content
3567         // length for the InitiateMultipartUpload request.
3568         setZeroContentLength(request);
3569         // Set the request content to be empty (but not null) to force the runtime to pass
3570         // any query params in the query string and not the request body, to keep S3 happy.
3571         request.setContent(new ByteArrayInputStream(new byte[0]));
3572
3573         @SuppressWarnings("unchecked")
3574         ResponseHeaderHandlerChain<InitiateMultipartUploadResult> responseHandler = new ResponseHeaderHandlerChain<InitiateMultipartUploadResult>(
3575                 // xml payload unmarshaller
3576                 new Unmarshallers.InitiateMultipartUploadResultUnmarshaller(),
3577                 // header handlers
3578                 new ServerSideEncryptionHeaderHandler<InitiateMultipartUploadResult>(),
3579                 new S3RequesterChargedHeaderHandler<InitiateMultipartUploadResult>(),
3580                 new InitiateMultipartUploadHeaderHandler());
3581         return invoke(request, responseHandler,
3582                 initiateMultipartUploadRequest.getBucketName(), initiateMultipartUploadRequest.getKey());
3583     }
3584
3585     @Override
3586     public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest listMultipartUploadsRequest)
3587             throws SdkClientException, AmazonServiceException {
3588         listMultipartUploadsRequest = beforeClientExecution(listMultipartUploadsRequest);
3589         rejectNull(listMultipartUploadsRequest,
3590             "The request parameter must be specified when listing multipart uploads");
3591
3592         rejectNull(listMultipartUploadsRequest.getBucketName(),
3593             "The bucket name parameter must be specified when listing multipart uploads");
3594
3595         Request<ListMultipartUploadsRequest> request = createRequest(listMultipartUploadsRequest.getBucketName(), null, listMultipartUploadsRequest, HttpMethodName.GET);
3596         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "ListMultipartUploads");
3597         request.addParameter("uploads", null);
3598
3599         if (listMultipartUploadsRequest.getKeyMarker() != null) request.addParameter("key-marker", listMultipartUploadsRequest.getKeyMarker());
3600         if (listMultipartUploadsRequest.getMaxUploads() != null) request.addParameter("max-uploads", listMultipartUploadsRequest.getMaxUploads().toString());
3601         if (listMultipartUploadsRequest.getUploadIdMarker() != null) request.addParameter("upload-id-marker", listMultipartUploadsRequest.getUploadIdMarker());
3602         if (listMultipartUploadsRequest.getDelimiter() != null) request.addParameter("delimiter", listMultipartUploadsRequest.getDelimiter());
3603         if (listMultipartUploadsRequest.getPrefix() != null) request.addParameter("prefix", listMultipartUploadsRequest.getPrefix());
3604         if (listMultipartUploadsRequest.getEncodingType() != null) request.addParameter("encoding-type", listMultipartUploadsRequest.getEncodingType());
3605
3606         return invoke(request, new Unmarshallers.ListMultipartUploadsResultUnmarshaller(), listMultipartUploadsRequest.getBucketName(), null);
3607     }
3608
3609     @Override
3610     public PartListing listParts(ListPartsRequest listPartsRequest)
3611             throws SdkClientException, AmazonServiceException {
3612         listPartsRequest = beforeClientExecution(listPartsRequest);
3613         rejectNull(listPartsRequest,
3614             "The request parameter must be specified when listing parts");
3615
3616         rejectNull(listPartsRequest.getBucketName(),
3617             "The bucket name parameter must be specified when listing parts");
3618         rejectNull(listPartsRequest.getKey(),
3619             "The key parameter must be specified when listing parts");
3620         rejectNull(listPartsRequest.getUploadId(),
3621             "The upload ID parameter must be specified when listing parts");
3622
3623         Request<ListPartsRequest> request = createRequest(listPartsRequest.getBucketName(), listPartsRequest.getKey(), listPartsRequest, HttpMethodName.GET);
3624         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "ListParts");
3625         request.addParameter("uploadId", listPartsRequest.getUploadId());
3626
3627         if (listPartsRequest.getMaxParts() != null) request.addParameter("max-parts", listPartsRequest.getMaxParts().toString());
3628         if (listPartsRequest.getPartNumberMarker() != null) request.addParameter("part-number-marker", listPartsRequest.getPartNumberMarker().toString());
3629         if (listPartsRequest.getEncodingType() != null) request.addParameter("encoding-type", listPartsRequest.getEncodingType());
3630
3631         populateRequesterPaysHeader(request, listPartsRequest.isRequesterPays());
3632
3633         @SuppressWarnings("unchecked")
3634         ResponseHeaderHandlerChain<PartListing> responseHandler = new ResponseHeaderHandlerChain<PartListing>(
3635                 // xml payload unmarshaller
3636                 new Unmarshallers.ListPartsResultUnmarshaller(),
3637                 // header handler
3638                 new S3RequesterChargedHeaderHandler<PartListing>(),
3639                 new ListPartsHeaderHandler());
3640         return invoke(request, responseHandler, listPartsRequest.getBucketName(), listPartsRequest.getKey());
3641     }
3642
3643     @Override
3644     public UploadPartResult uploadPart(UploadPartRequest uploadPartRequest)
3645             throws SdkClientException, AmazonServiceException {
3646         uploadPartRequest = beforeClientExecution(uploadPartRequest);
3647         rejectNull(uploadPartRequest,
3648             "The request parameter must be specified when uploading a part");
3649         final File fileOrig = uploadPartRequest.getFile();
3650         final InputStream isOrig = uploadPartRequest.getInputStream();
3651         final String bucketName = uploadPartRequest.getBucketName();
3652         final String key        = uploadPartRequest.getKey();
3653         final String uploadId   = uploadPartRequest.getUploadId();
3654         final int partNumber    = uploadPartRequest.getPartNumber();
3655         final long partSize     = uploadPartRequest.getPartSize();
3656         rejectNull(bucketName,
3657             "The bucket name parameter must be specified when uploading a part");
3658         rejectNull(key,
3659             "The key parameter must be specified when uploading a part");
3660         rejectNull(uploadId,
3661             "The upload ID parameter must be specified when uploading a part");
3662         Request<UploadPartRequest> request = createRequest(bucketName, key, uploadPartRequest, HttpMethodName.PUT);
3663         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "UploadPart");
3664         request.addHandlerContext(HandlerContextKey.REQUIRES_LENGTH, Boolean.TRUE);
3665         request.addHandlerContext(HandlerContextKey.HAS_STREAMING_INPUT, Boolean.TRUE);
3666
3667         request.addParameter("uploadId", uploadId);
3668         request.addParameter("partNumber", Integer.toString(partNumber));
3669
3670         final ObjectMetadata objectMetadata = uploadPartRequest.getObjectMetadata();
3671         if (objectMetadata != null)
3672             populateRequestMetadata(request, objectMetadata);
3673
3674         addHeaderIfNotNull(request, Headers.CONTENT_MD5, uploadPartRequest.getMd5Digest());
3675         request.addHeader(Headers.CONTENT_LENGTH, Long.toString(partSize));
3676
3677         populateRequesterPaysHeader(request, uploadPartRequest.isRequesterPays());
3678
3679         // Populate the SSE-C parameters to the request header
3680         populateSSE_C(request, uploadPartRequest.getSSECustomerKey());
3681         InputStream isCurr = isOrig;
3682         try {
3683             if (fileOrig == null) {
3684                 if (isOrig == null) {
3685                     throw new IllegalArgumentException(
3686                         "A File or InputStream must be specified when uploading part");
3687                 } else {
3688                     // When isCurr is a FileInputStream, this wrapping enables
3689                     // unlimited mark-and-reset
3690                     isCurr = ReleasableInputStream.wrap(isCurr);
3691                 }
3692                 // Make backward compatible with buffer size via system property
3693                 final Integer bufsize = Constants.getS3StreamBufferSize();
3694                 if (bufsize != null) {
3695                     AmazonWebServiceRequest awsreq = request.getOriginalRequest();
3696                     // Note awsreq is never null at this point even if the original
3697                     // request was
3698                     awsreq.getRequestClientOptions()
3699                         .setReadLimit(bufsize.intValue());
3700                 }
3701             } else {
3702                 try {
3703                     isCurr = new ResettableInputStream(fileOrig);
3704                 } catch(IOException e) {
3705                     throw new IllegalArgumentException("Failed to open file "
3706                             + fileOrig, e);
3707                 }
3708             }
3709             isCurr = new InputSubstream(
3710                     isCurr,
3711                     uploadPartRequest.getFileOffset(),
3712                     partSize,
3713                     uploadPartRequest.isLastPart());
3714             MD5DigestCalculatingInputStream md5DigestStream = null;
3715             if (uploadPartRequest.getMd5Digest() == null
3716                     && !skipMd5CheckStrategy.skipClientSideValidationPerRequest(uploadPartRequest)) {
3717                 /*
3718                  * If the user hasn't set the content MD5, then we don't want to buffer the whole
3719                  * stream in memory just to calculate it. Instead, we can calculate it on the fly
3720                  * and validate it with the returned ETag from the object upload.
3721                  */
3722                 isCurr = md5DigestStream = new MD5DigestCalculatingInputStream(isCurr);
3723             }
3724             final ProgressListener listener = uploadPartRequest.getGeneralProgressListener();
3725             publishProgress(listener, ProgressEventType.TRANSFER_PART_STARTED_EVENT);
3726             return doUploadPart(bucketName, key, uploadId, partNumber,
3727                     partSize, request, isCurr, md5DigestStream, listener);
3728         } finally {
3729             cleanupDataSource(uploadPartRequest, fileOrig, isOrig, isCurr, log);
3730         }
3731     }
3732
3733     private UploadPartResult doUploadPart(final String bucketName,
3734             final String key, final String uploadId, final int partNumber,
3735             final long partSize, Request<UploadPartRequest> request,
3736             InputStream inputStream,
3737             MD5DigestCalculatingInputStream md5DigestStream,
3738             final ProgressListener listener) {
3739         try {
3740             request.setContent(inputStream);
3741             ObjectMetadata metadata = invoke(request, new S3MetadataResponseHandler(), bucketName, key);
3742             final String etag = metadata.getETag();
3743
3744             if (md5DigestStream != null
3745                     && !skipMd5CheckStrategy.skipClientSideValidationPerUploadPartResponse(metadata)) {
3746                 byte[] clientSideHash = md5DigestStream.getMd5Digest();
3747                 byte[] serverSideHash = BinaryUtils.fromHex(etag);
3748
3749                 if (!Arrays.equals(clientSideHash, serverSideHash)) {
3750                     final String info = "bucketName: " + bucketName + ", key: "
3751                             + key + ", uploadId: " + uploadId
3752                             + ", partNumber: " + partNumber + ", partSize: "
3753                             + partSize;
3754                     throw new SdkClientException(
3755                          "Unable to verify integrity of data upload.  "
3756                         + "Client calculated content hash (contentMD5: "
3757                         + Base16.encodeAsString(clientSideHash)
3758                         + " in hex) didn't match hash (etag: "
3759                         + etag
3760                         + " in hex) calculated by Amazon S3.  "
3761                         + "You may need to delete the data stored in Amazon S3. "
3762                         + "(" + info + ")");
3763                 }
3764             }
3765             publishProgress(listener, ProgressEventType.TRANSFER_PART_COMPLETED_EVENT);
3766             UploadPartResult result = new UploadPartResult();
3767             result.setETag(etag);
3768             result.setPartNumber(partNumber);
3769             result.setSSEAlgorithm(metadata.getSSEAlgorithm());
3770             result.setSSECustomerAlgorithm(metadata.getSSECustomerAlgorithm());
3771             result.setSSECustomerKeyMd5(metadata.getSSECustomerKeyMd5());
3772             result.setRequesterCharged(metadata.isRequesterCharged());
3773             return result;
3774         } catch (Throwable t) {
3775             publishProgress(listener, ProgressEventType.TRANSFER_PART_FAILED_EVENT);
3776             // Leaving this here in case anyone is depending on it, but it's
3777             // inconsistent with other methods which only generate one of
3778             // COMPLETED_EVENT_CODE or FAILED_EVENT_CODE.
3779             publishProgress(listener, ProgressEventType.TRANSFER_PART_COMPLETED_EVENT);
3780             throw failure(t);
3781         }
3782     }
3783
3784     @Override
3785     public S3ResponseMetadata getCachedResponseMetadata(AmazonWebServiceRequest request) {
3786         return (S3ResponseMetadata)client.getResponseMetadataForRequest(request);
3787     }
3788
3789     @Override
3790     public void restoreObject(RestoreObjectRequest restoreObjectRequest)
3791         throws AmazonServiceException {
3792         restoreObjectV2(restoreObjectRequest);
3793     }
3794
3795     @Override
3796     public RestoreObjectResult restoreObjectV2(RestoreObjectRequest restoreObjectRequest)
3797         throws AmazonServiceException {
3798
3799         restoreObjectRequest = beforeClientExecution(restoreObjectRequest);
3800         String bucketName = restoreObjectRequest.getBucketName();
3801         String key = restoreObjectRequest.getKey();
3802         int expirationInDays = restoreObjectRequest.getExpirationInDays();
3803
3804         rejectNull(bucketName, "The bucket name parameter must be specified when restoring a glacier object");
3805         rejectNull(key, "The key parameter must be specified when restoring a glacier object");
3806
3807         if (restoreObjectRequest.getOutputLocation() != null) {
3808             rejectNull(restoreObjectRequest.getType(), "The restore request type must be specified with restores that specify OutputLocation");
3809
3810             if (RestoreRequestType.SELECT.toString().equals(restoreObjectRequest.getType())) {
3811                 rejectNull(restoreObjectRequest.getSelectParameters(),
3812                            "The select parameters must be specified when restoring a glacier object with SELECT restore request type");
3813             }
3814         } else if (expirationInDays == -1) {
3815             throw new IllegalArgumentException("The expiration in days parameter must be specified when restoring a glacier object without OutputLocation");
3816         }
3817
3818         Request<RestoreObjectRequest> request =
3819             createRestoreObjectRequest(restoreObjectRequest);
3820
3821         @SuppressWarnings("unchecked")
3822         ResponseHeaderHandlerChain<RestoreObjectResult> responseHandler = new ResponseHeaderHandlerChain<RestoreObjectResult>(
3823             new Unmarshallers.RestoreObjectResultUnmarshaller(),
3824             new S3RequesterChargedHeaderHandler<RestoreObjectResult>(),
3825             new S3RestoreOutputPathHeaderHandler<RestoreObjectResult>());
3826
3827         return invoke(request, responseHandler, bucketName, key);
3828     }
3829
3830     @Override
3831     public void restoreObject(String bucketName, String key, int expirationInDays)
3832             throws AmazonServiceException {
3833         restoreObject(new RestoreObjectRequest(bucketName, key, expirationInDays));
3834     }
3835
3836     @Override
3837     public PutObjectResult putObject(String bucketName, String key, String content)
3838             throws AmazonServiceException, SdkClientException {
3839
3840         rejectNull(bucketName, "Bucket name must be provided");
3841         rejectNull(key, "Object key must be provided");
3842         rejectNull(content, "String content must be provided");
3843
3844         byte[] contentBytes = content.getBytes(StringUtils.UTF8);
3845
3846         InputStream is = new ByteArrayInputStream(contentBytes);
3847         ObjectMetadata metadata = new ObjectMetadata();
3848         metadata.setContentType("text/plain");
3849         metadata.setContentLength(contentBytes.length);
3850
3851         return putObject(new PutObjectRequest(bucketName, key, is, metadata));
3852     }
3853
3854     /*
3855      * Private Interface
3856      */
3857
3858     /**
3859      * <p>
3860      * Asserts that the specified parameter value is not <code>null</code> and if it is,
3861      * throws an <code>IllegalArgumentException</code> with the specified error message.
3862      * </p>
3863      *
3864      * @param parameterValue
3865      *            The parameter value being checked.
3866      * @param errorMessage
3867      *            The error message to include in the IllegalArgumentException
3868      *            if the specified parameter is null.
3869      */
3870     private void rejectNull(Object parameterValue, String errorMessage) {
3871         if (parameterValue == null) throw new IllegalArgumentException(errorMessage);
3872     }
3873
3874     /**
3875      * <p>
3876      * Gets the Amazon S3 {@link AccessControlList} (ACL) for the specified resource.
3877      * (bucket if only the bucketName parameter is specified, otherwise the object with the
3878      * specified key in the bucket).
3879      * </p>
3880      *
3881      * @param bucketName
3882      *            The name of the bucket whose ACL should be returned if the key
3883      *            parameter is not specified, otherwise the bucket containing
3884      *            the specified key.
3885      * @param key
3886      *            The object key whose ACL should be retrieve. If not specified,
3887      *            the bucket's ACL is returned.
3888      * @param versionId
3889      *            The version ID of the object version whose ACL is being
3890      *            retrieved.
3891      * @param originalRequest
3892      *            The original, user facing request object.
3893      *
3894      * @return The S3 ACL for the specified resource.
3895      */
3896     private AccessControlList getAcl(String bucketName, String key, String versionId,
3897             boolean isRequesterPays, AmazonWebServiceRequest originalRequest) {
3898         if (originalRequest == null) originalRequest = new GenericBucketRequest(bucketName);
3899
3900         Request<AmazonWebServiceRequest> request = createRequest(bucketName, key, originalRequest, HttpMethodName.GET);
3901
3902         if (bucketName != null && key != null) {
3903             request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetObjectAcl");
3904         } else if (bucketName != null) {
3905             request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketAcl");
3906         }
3907
3908         request.addParameter("acl", null);
3909         if (versionId != null) {
3910             request.addParameter("versionId", versionId);
3911         }
3912         populateRequesterPaysHeader(request, isRequesterPays);
3913
3914         @SuppressWarnings("unchecked")
3915         ResponseHeaderHandlerChain<AccessControlList> responseHandler = new ResponseHeaderHandlerChain<AccessControlList>(
3916                 new Unmarshallers.AccessControlListUnmarshaller(),
3917                 new S3RequesterChargedHeaderHandler<AccessControlList>());
3918
3919         return invoke(request, responseHandler, bucketName, key);
3920     }
3921
3922     /**
3923      * Sets the Canned ACL for the specified resource in S3. If only bucketName
3924      * is specified, the canned ACL will be applied to the bucket, otherwise if
3925      * bucketName and key are specified, the canned ACL will be applied to the
3926      * object.
3927      *
3928      * @param bucketName
3929      *            The name of the bucket containing the specified key, or if no
3930      *            key is listed, the bucket whose ACL will be set.
3931      * @param key
3932      *            The optional object key within the specified bucket whose ACL
3933      *            will be set. If not specified, the bucket ACL will be set.
3934      * @param versionId
3935      *            The version ID of the object version whose ACL is being set.
3936      * @param cannedAcl
3937      *            The canned ACL to apply to the resource.
3938      * @param originalRequest
3939      *            The original, user facing request object.
3940      */
3941     private void setAcl(String bucketName, String key, String versionId, CannedAccessControlList cannedAcl, boolean isRequesterPays,
3942             AmazonWebServiceRequest originalRequest) {
3943         if (originalRequest == null) originalRequest = new GenericBucketRequest(bucketName);
3944
3945         Request<AmazonWebServiceRequest> request = createRequest(bucketName, key, originalRequest, HttpMethodName.PUT);
3946
3947         if (bucketName != null && key != null) {
3948             request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutObjectAcl");
3949         } else if (bucketName != null) {
3950             request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketAcl");
3951         }
3952
3953         request.addParameter("acl", null);
3954         request.addHeader(Headers.S3_CANNED_ACL, cannedAcl.toString());
3955         if (versionId != null) request.addParameter("versionId", versionId);
3956         populateRequesterPaysHeader(request, isRequesterPays);
3957
3958         invoke(request, voidResponseHandler, bucketName, key);
3959     }
3960
3961     /**
3962      * Sets the ACL for the specified resource in S3. If only bucketName is
3963      * specified, the ACL will be applied to the bucket, otherwise if bucketName
3964      * and key are specified, the ACL will be applied to the object.
3965      *
3966      * @param bucketName
3967      *            The name of the bucket containing the specified key, or if no
3968      *            key is listed, the bucket whose ACL will be set.
3969      * @param key
3970      *            The optional object key within the specified bucket whose ACL
3971      *            will be set. If not specified, the bucket ACL will be set.
3972      * @param versionId
3973      *            The version ID of the object version whose ACL is being set.
3974      * @param acl
3975      *            The ACL to apply to the resource.
3976      * @param originalRequest
3977      *            The original, user facing request object.
3978      */
3979     private void setAcl(String bucketName, String key, String versionId, AccessControlList acl, boolean isRequesterPays,
3980             AmazonWebServiceRequest originalRequest) {
3981         if (originalRequest == null) originalRequest = new GenericBucketRequest(bucketName);
3982
3983         Request<AmazonWebServiceRequest> request = createRequest(bucketName, key, originalRequest, HttpMethodName.PUT);
3984
3985         if (bucketName != null && key != null) {
3986             request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutObjectAcl");
3987         } else if (bucketName != null) {
3988             request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketAcl");
3989         }
3990
3991         request.addParameter("acl", null);
3992         if (versionId != null) request.addParameter("versionId", versionId);
3993         populateRequesterPaysHeader(request, isRequesterPays);
3994
3995         byte[] aclAsXml = new AclXmlFactory().convertToXmlByteArray(acl);
3996         request.addHeader("Content-Type", "application/xml");
3997         request.addHeader("Content-Length", String.valueOf(aclAsXml.length));
3998         request.setContent(new ByteArrayInputStream(aclAsXml));
3999
4000         invoke(request, voidResponseHandler, bucketName, key);
4001     }
4002
4003     /**
4004      * Returns a "complete" S3 specific signer, taking into the S3 bucket, key,
4005      * and the current S3 client configuration into account.
4006      */
4007     protected Signer createSigner(final Request<?> request,
4008                                   final String bucketName,
4009                                   final String key) {
4010         return createSigner(request, bucketName, key, false);
4011     }
4012
4013     /**
4014      * Returns a "complete" S3 specific signer, taking into the S3 bucket, key,
4015      * and the current S3 client configuration into account.
4016      */
4017     protected Signer createSigner(final Request<?> request,
4018                                   final String bucketName,
4019                                   final String key,
4020                                   final boolean isAdditionalHeadRequestToFindRegion) {
4021
4022         // Instead of using request.getEndpoint() for this parameter, we use endpoint which is because
4023         // in accelerate mode, the endpoint in request is regionless. We need the client-wide endpoint
4024         // to fetch the region information and pick the correct signer.
4025         URI uri = clientOptions.isAccelerateModeEnabled() ? endpoint : request.getEndpoint();
4026
4027         Signer signer;
4028         if (isAccessPointArn(bucketName)) {
4029             Arn resourceArn = Arn.fromString(bucketName);
4030             S3Resource s3Resource = S3ArnConverter.getInstance().convertArn(resourceArn);
4031             String region = s3Resource.getRegion();
4032             String regionalEndpoint = RegionUtils.getRegion(region).getServiceEndpoint("s3");
4033             signer = getSignerByURI(URI.create(uri.getScheme() + "://" + regionalEndpoint));
4034         } else {
4035             signer = getSignerByURI(uri);
4036         }
4037
4038         if (!isSignerOverridden()) {
4039
4040             if ((signer instanceof AWSS3V4Signer) && bucketRegionShouldBeCached(request)) {
4041
4042                 String region = bucketRegionCache.get(bucketName);
4043                 if (region != null) {
4044                      // If cache contains the region for the bucket, create an endpoint for the region and
4045                      // update the request with that endpoint if accelerate mode is not enabled
4046                      request.addHandlerContext(HandlerContextKey.SIGNING_REGION, region);
4047                      if (!clientOptions.isAccelerateModeEnabled()) {
4048                          resolveRequestEndpoint(request, bucketName, key, RuntimeHttpUtils.toUri(RegionUtils.getRegion(region).getServiceEndpoint(S3_SERVICE_NAME), clientConfiguration));
4049                      }
4050                      return updateSigV4SignerWithRegion((AWSS3V4Signer) signer, region);
4051                 } else if (request.getOriginalRequest() instanceof GeneratePresignedUrlRequest) {
4052                     String signerRegion = getSignerRegion();
4053                     if (signerRegion == null) {
4054                         return createSigV2Signer(request, bucketName, key);
4055                     }
4056                     return updateSigV4SignerWithRegion((AWSS3V4Signer) signer, signerRegion);
4057                 } else if (isAdditionalHeadRequestToFindRegion) {
4058                     return updateSigV4SignerWithRegion((AWSS3V4Signer) signer, "us-east-1");
4059                 }
4060             }
4061
4062             String regionOverride = getSignerRegionOverride();
4063             if (regionOverride != null) {
4064                 return updateSigV4SignerWithRegion(new AWSS3V4Signer(), regionOverride);
4065             }
4066         }
4067
4068         if (signer instanceof S3Signer) {
4069             // The old S3Signer needs a method and path passed to its
4070             // constructor; if that's what we should use, getSigner()
4071             // will return a dummy instance and we need to create a
4072             // new one with the appropriate values for this request.
4073             return createSigV2Signer(request, bucketName, key);
4074         }
4075
4076         return signer;
4077     }
4078
4079     private S3Signer createSigV2Signer(final Request<?> request,
4080                                             final String bucketName,
4081                                             final String key) {
4082         String resourcePath = "/" +
4083                 ((bucketName != null) ? bucketName + "/" : "") +
4084                 ((key != null) ? key : "");
4085         return new S3Signer(request.getHttpMethod().toString(), resourcePath);
4086     }
4087
4088     private AWSS3V4Signer updateSigV4SignerWithRegion(final AWSS3V4Signer v4Signer, String region) {
4089         v4Signer.setServiceName(getServiceNameIntern());
4090         v4Signer.setRegionName(region);
4091         return v4Signer;
4092     }
4093
4094     /**
4095      * Return the region string that should be used for signing requests sent by
4096      * this client. This method can only return null if both of the following
4097      * are true:
4098      * (a) the user has never specified a region via setRegion/configureRegion/setSignerRegionOverride
4099      * (b) the user has specified a client endpoint that is known to be a global S3 endpoint
4100      */
4101     private String getSignerRegion() {
4102         String region = getSignerRegionOverride();
4103         if (region == null) {
4104             region = clientRegion;
4105         }
4106         return region;
4107     }
4108
4109     /**
4110      * Has signer been explicitly overriden in the configuration?
4111      */
4112     private boolean isSignerOverridden() {
4113         return clientConfiguration != null
4114           && clientConfiguration.getSignerOverride() != null;
4115     }
4116
4117     /**
4118      * <p>Returns true if the region required for signing could not be computed from the client or the request.</p>
4119      * <p>
4120      * This is the case when the standard endpoint is in use and neither an explicit region nor a signer override
4121      * have been provided by the user.
4122      * </p>
4123      */
4124     private boolean noExplicitRegionProvided(final Request<?> request) {
4125         return isStandardEndpoint(request.getEndpoint()) &&
4126                 getSignerRegion() == null;
4127     }
4128
4129     private boolean isStandardEndpoint(URI endpoint) {
4130         return endpoint.getHost().endsWith(Constants.S3_HOSTNAME);
4131     }
4132
4133     /**
4134      * Pre-signs the specified request, using a signature query-string
4135      * parameter.
4136      *
4137      * @param request
4138      *            The request to sign.
4139      * @param methodName
4140      *            The HTTP method (GET, PUT, DELETE, HEAD) for the specified
4141      *            request.
4142      * @param bucketName
4143      *            The name of the bucket involved in the request. If the request
4144      *            is not an operation on a bucket this parameter should be null.
4145      * @param key
4146      *            The object key involved in the request. If the request is not
4147      *            an operation on an object, this parameter should be null.
4148      * @param expiration
4149      *            The time at which the signed request is no longer valid, and
4150      *            will stop working.
4151      * @param subResource
4152      *            The optional sub-resource being requested as part of the
4153      *            request (e.g. "location", "acl", "logging", or "torrent").
4154      */
4155     protected <T> void presignRequest(Request<T> request, HttpMethod methodName,
4156             String bucketName, String key, Date expiration, String subResource) {
4157         // Run any additional request handlers if present
4158         beforeRequest(request);
4159
4160         String resourcePath = "/" +
4161             ((bucketName != null) ? bucketName + "/" : "") +
4162             ((key != null) ? SdkHttpUtils.urlEncode(key, true) : "") +
4163             ((subResource != null) ? "?" + subResource : "");
4164
4165         // Make sure the resource-path for signing does not contain
4166         // any consecutive "/"s.
4167         // Note that we should also follow the same rule to escape
4168         // consecutive "/"s when generating the presigned URL.
4169         // See ServiceUtils#convertRequestToUrl(...)
4170         resourcePath = resourcePath.replaceAll("(?<=/)/", "%2F");
4171
4172         new S3QueryStringSigner(methodName.toString(), resourcePath, expiration)
4173             .sign(request, CredentialUtils.getCredentialsProvider(request.getOriginalRequest(), awsCredentialsProvider).getCredentials());
4174
4175         // The Amazon S3 DevPay token header is a special exception and can be safely moved
4176         // from the request's headers into the query string to ensure that it travels along
4177         // with the pre-signed URL when it's sent back to Amazon S3.
4178         if (request.getHeaders().containsKey(Headers.SECURITY_TOKEN)) {
4179             String value = request.getHeaders().get(Headers.SECURITY_TOKEN);
4180             request.addParameter(Headers.SECURITY_TOKEN, value);
4181             request.getHeaders().remove(Headers.SECURITY_TOKEN);
4182         }
4183     }
4184
4185     private <T> void beforeRequest(Request<T> request) {
4186         if (requestHandler2s != null) {
4187             for (RequestHandler2 requestHandler2 : requestHandler2s) {
4188                 requestHandler2.beforeRequest(request);
4189             }
4190         }
4191     }
4192
4193     /**
4194      * <p>
4195      * Populates the specified request object with the appropriate headers from
4196      * the {@link ObjectMetadata} object.
4197      * </p>
4198      *
4199      * @param request
4200      *            The request to populate with headers.
4201      * @param metadata
4202      *            The metadata containing the header information to include in
4203      *            the request.
4204      */
4205     protected static void populateRequestMetadata(Request<?> request, ObjectMetadata metadata) {
4206         Map<String, Object> rawMetadata = metadata.getRawMetadata();
4207         if (rawMetadata != null) {
4208             for (Entry<String, Object> entry : rawMetadata.entrySet()) {
4209                 request.addHeader(entry.getKey(), entry.getValue().toString());
4210             }
4211         }
4212
4213         Date httpExpiresDate = metadata.getHttpExpiresDate();
4214         if (httpExpiresDate != null) {
4215             request.addHeader(Headers.EXPIRES, DateUtils.formatRFC822Date(httpExpiresDate));
4216         }
4217
4218         Map<String, String> userMetadata = metadata.getUserMetadata();
4219         if (userMetadata != null) {
4220             for (Entry<String, String> entry : userMetadata.entrySet()) {
4221                 String key = entry.getKey();
4222                 String value = entry.getValue();
4223                 if (key != null) key = key.trim();
4224                 if (value != null) value = value.trim();
4225                 request.addHeader(Headers.S3_USER_METADATA_PREFIX + key, value);
4226             }
4227         }
4228     }
4229
4230     /**
4231      * <p>
4232      * Populate the specified request with {@link Constants#REQUESTER_PAYS} to header {@link Headers#REQUESTER_PAYS_HEADER},
4233      * if isRequesterPays is true.
4234      * </p>
4235      *
4236      * @param request
4237      *              The specified request to populate.
4238      * @param isRequesterPays
4239      *              The flag whether to populate the header or not.
4240      */
4241     protected static void populateRequesterPaysHeader(Request<?> request, boolean isRequesterPays) {
4242         if (isRequesterPays) {
4243             request.addHeader(Headers.REQUESTER_PAYS_HEADER, Constants.REQUESTER_PAYS);
4244         }
4245     }
4246
4247     /**
4248      * <p>
4249      * Populates the specified request with the specified Multi-Factor
4250      * Authentication (MFA) details. This includes the MFA header with device serial
4251      * number and generated token. Since all requests which include the MFA
4252      * header must be sent over HTTPS, this operation also configures the request object to
4253      * use HTTPS instead of HTTP.
4254      * </p>
4255      *
4256      * @param request
4257      *            The request to populate.
4258      * @param mfa
4259      *            The Multi-Factor Authentication information.
4260      */
4261     private void populateRequestWithMfaDetails(Request<?> request, MultiFactorAuthentication mfa) {
4262         if (mfa == null) return;
4263
4264         String endpoint = request.getEndpoint().toString();
4265         if (endpoint.startsWith("http://")) {
4266             String httpsEndpoint = endpoint.replace("http://", "https://");
4267             request.setEndpoint(URI.create(httpsEndpoint));
4268             log.info("Overriding current endpoint to use HTTPS " +
4269                     "as required by S3 for requests containing an MFA header");
4270         }
4271
4272         request.addHeader(Headers.S3_MFA,
4273                 mfa.getDeviceSerialNumber() + " " + mfa.getToken());
4274     }
4275
4276     /**
4277      * <p>
4278      * Populates the specified request with the numerous options available in
4279      * <code>CopyObjectRequest</code>.
4280      * </p>
4281      *
4282      * @param request
4283      *            The request to populate with headers to represent all the
4284      *            options expressed in the <code>CopyObjectRequest</code> object.
4285      * @param copyObjectRequest
4286      *            The object containing all the options for copying an object in
4287      *            Amazon S3.
4288      */
4289     private void populateRequestWithCopyObjectParameters(Request<? extends AmazonWebServiceRequest> request, CopyObjectRequest copyObjectRequest) {
4290         String copySourceHeader =
4291              "/" + SdkHttpUtils.urlEncode(copyObjectRequest.getSourceBucketName(), true)
4292            + "/" + SdkHttpUtils.urlEncode(copyObjectRequest.getSourceKey(), true);
4293         if (copyObjectRequest.getSourceVersionId() != null) {
4294             copySourceHeader += "?versionId=" + copyObjectRequest.getSourceVersionId();
4295         }
4296         request.addHeader("x-amz-copy-source", copySourceHeader);
4297
4298         addDateHeader(request, Headers.COPY_SOURCE_IF_MODIFIED_SINCE,
4299                 copyObjectRequest.getModifiedSinceConstraint());
4300         addDateHeader(request, Headers.COPY_SOURCE_IF_UNMODIFIED_SINCE,
4301                 copyObjectRequest.getUnmodifiedSinceConstraint());
4302
4303         addStringListHeader(request, Headers.COPY_SOURCE_IF_MATCH,
4304                 copyObjectRequest.getMatchingETagConstraints());
4305         addStringListHeader(request, Headers.COPY_SOURCE_IF_NO_MATCH,
4306                 copyObjectRequest.getNonmatchingETagConstraints());
4307
4308         if (copyObjectRequest.getAccessControlList() != null) {
4309             addAclHeaders(request, copyObjectRequest.getAccessControlList());
4310         } else if (copyObjectRequest.getCannedAccessControlList() != null) {
4311             request.addHeader(Headers.S3_CANNED_ACL,
4312                     copyObjectRequest.getCannedAccessControlList().toString());
4313         }
4314
4315         if (copyObjectRequest.getStorageClass() != null) {
4316             request.addHeader(Headers.STORAGE_CLASS, copyObjectRequest.getStorageClass());
4317         }
4318
4319         if (copyObjectRequest.getRedirectLocation() != null) {
4320             request.addHeader(Headers.REDIRECT_LOCATION, copyObjectRequest.getRedirectLocation());
4321         }
4322
4323         populateRequesterPaysHeader(request, copyObjectRequest.isRequesterPays());
4324
4325         ObjectMetadata newObjectMetadata = copyObjectRequest.getNewObjectMetadata();
4326         if (copyObjectRequest.getMetadataDirective() != null) {
4327             request.addHeader(Headers.METADATA_DIRECTIVE, copyObjectRequest.getMetadataDirective());
4328         } else if (newObjectMetadata != null) {
4329             request.addHeader(Headers.METADATA_DIRECTIVE, "REPLACE");
4330         }
4331         if (newObjectMetadata != null) {
4332             populateRequestMetadata(request, newObjectMetadata);
4333         }
4334
4335         ObjectTagging newObjectTagging = copyObjectRequest.getNewObjectTagging();
4336         if (newObjectTagging != null) {
4337             request.addHeader(Headers.TAGGING_DIRECTIVE, "REPLACE");
4338             request.addHeader(Headers.S3_TAGGING, urlEncodeTags(newObjectTagging));
4339         }
4340
4341         // Populate the SSE-C parameters for the destination object
4342         populateSourceSSE_C(request, copyObjectRequest.getSourceSSECustomerKey());
4343         populateSSE_C(request, copyObjectRequest.getDestinationSSECustomerKey());
4344     }
4345
4346     /**
4347      * <p>
4348      * Populates the specified request with the numerous options available in
4349      * <code>CopyObjectRequest</code>.
4350      * </p>
4351      *
4352      * @param request
4353      *            The request to populate with headers to represent all the
4354      *            options expressed in the <code>CopyPartRequest</code> object.
4355      * @param copyPartRequest
4356      *            The object containing all the options for copying an object in
4357      *            Amazon S3.
4358      */
4359     private static void populateRequestWithCopyPartParameters(Request<?> request, CopyPartRequest copyPartRequest) {
4360         String copySourceHeader =
4361              "/" + SdkHttpUtils.urlEncode(copyPartRequest.getSourceBucketName(), true)
4362            + "/" + SdkHttpUtils.urlEncode(copyPartRequest.getSourceKey(), true);
4363         if (copyPartRequest.getSourceVersionId() != null) {
4364             copySourceHeader += "?versionId=" + copyPartRequest.getSourceVersionId();
4365         }
4366         request.addHeader("x-amz-copy-source", copySourceHeader);
4367
4368         addDateHeader(request, Headers.COPY_SOURCE_IF_MODIFIED_SINCE,
4369                 copyPartRequest.getModifiedSinceConstraint());
4370         addDateHeader(request, Headers.COPY_SOURCE_IF_UNMODIFIED_SINCE,
4371                 copyPartRequest.getUnmodifiedSinceConstraint());
4372
4373         addStringListHeader(request, Headers.COPY_SOURCE_IF_MATCH,
4374                 copyPartRequest.getMatchingETagConstraints());
4375         addStringListHeader(request, Headers.COPY_SOURCE_IF_NO_MATCH,
4376                 copyPartRequest.getNonmatchingETagConstraints());
4377
4378         if ( copyPartRequest.getFirstByte() != null && copyPartRequest.getLastByte() != null ) {
4379             String range = "bytes=" + copyPartRequest.getFirstByte() + "-" + copyPartRequest.getLastByte();
4380             request.addHeader(Headers.COPY_PART_RANGE, range);
4381         }
4382
4383         // Populate the SSE-C parameters for the destination object
4384         populateSourceSSE_C(request, copyPartRequest.getSourceSSECustomerKey());
4385         populateSSE_C(request, copyPartRequest.getDestinationSSECustomerKey());
4386     }
4387
4388     /**
4389      * <p>
4390      * Populates the specified request with the numerous attributes available in
4391      * <code>SSEWithCustomerKeyRequest</code>.
4392      * </p>
4393      *
4394      * @param request
4395      *            The request to populate with headers to represent all the
4396      *            options expressed in the
4397      *            <code>ServerSideEncryptionWithCustomerKeyRequest</code>
4398      *            object.
4399      * @param sseKey
4400      *            The request object for an S3 operation that allows server-side
4401      *            encryption using customer-provided keys.
4402      */
4403     private static void populateSSE_C(Request<?> request, SSECustomerKey sseKey) {
4404         if (sseKey == null) return;
4405
4406         addHeaderIfNotNull(request, Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM,
4407                 sseKey.getAlgorithm());
4408         addHeaderIfNotNull(request, Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
4409                 sseKey.getKey());
4410         addHeaderIfNotNull(request, Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
4411                 sseKey.getMd5());
4412         // Calculate the MD5 hash of the encryption key and fill it in the
4413         // header, if the user didn't specify it in the metadata
4414         if (sseKey.getKey() != null
4415                 && sseKey.getMd5() == null) {
4416             String encryptionKey_b64 = sseKey.getKey();
4417             byte[] encryptionKey = Base64.decode(encryptionKey_b64);
4418             request.addHeader(Headers.SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
4419                     Md5Utils.md5AsBase64(encryptionKey));
4420         }
4421     }
4422
4423     private static void populateSourceSSE_C(Request<?> request, SSECustomerKey sseKey) {
4424         if (sseKey == null) return;
4425
4426         // Populate the SSE-C parameters for the source object
4427         addHeaderIfNotNull(request, Headers.COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM,
4428                 sseKey.getAlgorithm());
4429         addHeaderIfNotNull(request, Headers.COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
4430                 sseKey.getKey());
4431         addHeaderIfNotNull(request, Headers.COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
4432                 sseKey.getMd5());
4433         // Calculate the MD5 hash of the encryption key and fill it in the
4434         // header, if the user didn't specify it in the metadata
4435         if (sseKey.getKey() != null
4436                 && sseKey.getMd5() == null) {
4437             String encryptionKey_b64 = sseKey.getKey();
4438             byte[] encryptionKey = Base64.decode(encryptionKey_b64);
4439             request.addHeader(Headers.COPY_SOURCE_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
4440                     Md5Utils.md5AsBase64(encryptionKey));
4441         }
4442     }
4443
4444     private static void populateSSE_KMS(Request<?> request,
4445             SSEAwsKeyManagementParams sseParams) {
4446
4447         if (sseParams != null) {
4448             addHeaderIfNotNull(request, Headers.SERVER_SIDE_ENCRYPTION,
4449                     sseParams.getEncryption());
4450             addHeaderIfNotNull(request,
4451                     Headers.SERVER_SIDE_ENCRYPTION_AWS_KMS_KEYID,
4452                     sseParams.getAwsKmsKeyId());
4453             addHeaderIfNotNull(request, Headers.SERVER_SIDE_ENCRYPTION_AWS_KMS_CONTEXT, sseParams.getAwsKmsEncryptionContext());
4454         }
4455     }
4456
4457     /**
4458      * Adds the part number to the specified request, if partNumber is not null.
4459      *
4460      * @param request
4461      *            The request to add the partNumber to.
4462      * @param partNumber
4463      *               The part number to be added.
4464      */
4465     private void addPartNumberIfNotNull(Request<?> request, Integer partNumber) {
4466         if (partNumber != null) {
4467             request.addParameter("partNumber", partNumber.toString());
4468         }
4469     }
4470
4471     /**
4472      * Adds the specified header to the specified request, if the header value
4473      * is not null.
4474      *
4475      * @param request
4476      *            The request to add the header to.
4477      * @param header
4478      *            The header name.
4479      * @param value
4480      *            The header value.
4481      */
4482     private static void addHeaderIfNotNull(Request<?> request, String header, String value) {
4483         if (value != null) {
4484             request.addHeader(header, value);
4485         }
4486     }
4487
4488     /**
4489      * Adds the specified parameter to the specified request, if the parameter
4490      * value is not null.
4491      *
4492      * @param request
4493      *            The request to add the parameter to.
4494      * @param paramName
4495      *            The parameter name.
4496      * @param paramValue
4497      *            The parameter value.
4498      */
4499     private static void addParameterIfNotNull(Request<?> request, String paramName, Integer paramValue) {
4500         if (paramValue != null) {
4501             addParameterIfNotNull(request, paramName, paramValue.toString());
4502         }
4503     }
4504
4505     /**
4506      * Adds the specified parameter to the specified request, if the parameter
4507      * value is not null.
4508      *
4509      * @param request
4510      *            The request to add the parameter to.
4511      * @param paramName
4512      *            The parameter name.
4513      * @param paramValue
4514      *            The parameter value.
4515      */
4516     private static void addParameterIfNotNull(Request<?> request, String paramName, String paramValue) {
4517         if (paramValue != null) {
4518             request.addParameter(paramName, paramValue);
4519         }
4520     }
4521
4522     /**
4523      * <p>
4524      * Adds the specified date header in RFC 822 date format to the specified
4525      * request.
4526      * This method will not add a date header if the specified date value is <code>null</code>.
4527      * </p>
4528      *
4529      * @param request
4530      *            The request to add the header to.
4531      * @param header
4532      *            The header name.
4533      * @param value
4534      *            The header value.
4535      */
4536     private static void addDateHeader(Request<?> request, String header, Date value) {
4537         if (value != null) {
4538             request.addHeader(header, ServiceUtils.formatRfc822Date(value));
4539         }
4540     }
4541
4542     /**
4543      * <p>
4544      * Adds the specified string list header, joined together separated with
4545      * commas, to the specified request.
4546      * This method will not add a string list header if the specified values
4547      * are <code>null</code> or empty.
4548      * </p>
4549      *
4550      * @param request
4551      *            The request to add the header to.
4552      * @param header
4553      *            The header name.
4554      * @param values
4555      *            The list of strings to join together for the header value.
4556      */
4557     private static void addStringListHeader(Request<?> request, String header, List<String> values) {
4558         if (values != null && !values.isEmpty()) {
4559             request.addHeader(header, ServiceUtils.join(values));
4560         }
4561     }
4562
4563     /**
4564      * <p>
4565      * Adds response headers parameters to the request given, if non-null.
4566      * </p>
4567      *
4568      * @param request
4569      *            The request to add the response header parameters to.
4570      * @param responseHeaders
4571      *            The full set of response headers to add, or null for none.
4572      */
4573     private static void addResponseHeaderParameters(Request<?> request, ResponseHeaderOverrides responseHeaders) {
4574         if ( responseHeaders != null ) {
4575             if ( responseHeaders.getCacheControl() != null ) {
4576                 request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_CACHE_CONTROL, responseHeaders.getCacheControl());
4577             }
4578             if ( responseHeaders.getContentDisposition() != null ) {
4579                 request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_DISPOSITION,
4580                         responseHeaders.getContentDisposition());
4581             }
4582             if ( responseHeaders.getContentEncoding() != null ) {
4583                 request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_ENCODING,
4584                         responseHeaders.getContentEncoding());
4585             }
4586             if ( responseHeaders.getContentLanguage() != null ) {
4587                 request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_LANGUAGE,
4588                         responseHeaders.getContentLanguage());
4589             }
4590             if ( responseHeaders.getContentType() != null ) {
4591                 request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_CONTENT_TYPE, responseHeaders.getContentType());
4592             }
4593             if ( responseHeaders.getExpires() != null ) {
4594                 request.addParameter(ResponseHeaderOverrides.RESPONSE_HEADER_EXPIRES, responseHeaders.getExpires());
4595             }
4596         }
4597     }
4598
4599     /**
4600      * Returns the URL to the key in the bucket given, using the client's scheme
4601      * and endpoint. Returns null if the given bucket and key cannot be
4602      * converted to a URL.
4603      */
4604     public String getResourceUrl(String bucketName, String key) {
4605         try {
4606             return getUrl(bucketName, key).toString();
4607         } catch ( Exception e ) {
4608             return null;
4609         }
4610     }
4611
4612     @Override
4613     public URL getUrl(String bucketName, String key) {
4614         Request<?> request = new DefaultRequest<Object>(Constants.S3_SERVICE_DISPLAY_NAME);
4615         resolveRequestEndpoint(request, bucketName, key, endpoint);
4616         return ServiceUtils.convertRequestToUrl(request, false, false);
4617     }
4618
4619     @Override
4620     public synchronized Region getRegion() {
4621         String authority = super.endpoint.getAuthority();
4622         if (Constants.S3_HOSTNAME.equals(authority)) {
4623             return Region.US_Standard;
4624         } else {
4625             Matcher m = Region.S3_REGIONAL_ENDPOINT_PATTERN.matcher(authority);
4626             if (m.matches()) {
4627                 return Region.fromValue(m.group(1));
4628             } else {
4629                 throw new IllegalStateException(
4630                     "S3 client with invalid S3 endpoint configured: " + authority);
4631             }
4632         }
4633     }
4634
4635     @Override
4636     public String getRegionName() {
4637         String authority = super.endpoint.getAuthority();
4638         if(Constants.S3_HOSTNAME.equals(authority)) {
4639             return "us-east-1";
4640         }
4641         Matcher m = Region.S3_REGIONAL_ENDPOINT_PATTERN.matcher(authority);
4642         try {
4643             m.matches();
4644             return RegionUtils.getRegion(m.group(1)).getName();
4645         } catch (Exception e) {
4646             throw new IllegalStateException("No valid region has been specified. Unable to return region name", e);
4647         }
4648     }
4649
4650     private static boolean isRegionFipsEnabled(String regionName) {
4651         return regionName.startsWith("fips-") || regionName.endsWith("-fips");
4652     }
4653
4654     /**
4655      * Creates and initializes a new request object for the specified S3
4656      * resource. This method is responsible for determining the right way to
4657      * address resources. For example, bucket names that are not DNS addressable
4658      * cannot be addressed in V2, virtual host, style, and instead must use V1,
4659      * path style. The returned request object has the service name, endpoint
4660      * and resource path correctly populated. Callers can take the request, add
4661      * any additional headers or parameters, then sign and execute the request.
4662      *
4663      * @param bucketName
4664      *            An optional parameter indicating the name of the bucket
4665      *            containing the resource involved in the request.
4666      * @param key
4667      *            An optional parameter indicating the key under which the
4668      *            desired resource is stored in the specified bucket.
4669      * @param originalRequest
4670      *            The original request, as created by the user.
4671      * @param httpMethod
4672      *            The HTTP method to use when sending the request.
4673      *
4674      * @return A new request object, populated with endpoint, resource path, and
4675      *         service name, ready for callers to populate any additional
4676      *         headers or parameters, and execute.
4677      */
4678     protected <X extends AmazonWebServiceRequest> Request<X> createRequest(String bucketName, String key, X originalRequest, HttpMethodName httpMethod) {
4679         return createRequest(bucketName, key, originalRequest, httpMethod, endpoint);
4680     }
4681
4682     // NOTE: New uses of this method are discouraged and flagged at build time.
4683     // Be careful not to change its signature.
4684     protected <X extends AmazonWebServiceRequest> Request<X> createRequest(String bucketName, String key, X originalRequest, HttpMethodName httpMethod, URI endpoint) {
4685         String signingRegion;
4686
4687         Request<X> request = new DefaultRequest<X>(originalRequest, Constants.S3_SERVICE_DISPLAY_NAME);
4688         request.setHttpMethod(httpMethod);
4689         request.addHandlerContext(S3HandlerContextKeys.IS_CHUNKED_ENCODING_DISABLED,
4690                                   clientOptions.isChunkedEncodingDisabled());
4691         request.addHandlerContext(S3HandlerContextKeys.IS_PAYLOAD_SIGNING_ENABLED,
4692                                   clientOptions.isPayloadSigningEnabled());
4693         request.addHandlerContext(HandlerContextKey.SERVICE_ID, SERVICE_ID);
4694
4695         // If the bucketName appears to be an ARN, parse the ARN as an S3 resource and rewrite target resource arguments
4696         // based on the parsed resource.
4697         if (isAccessPointArn(bucketName)) {
4698             Arn resourceArn = Arn.fromString(bucketName);
4699             S3Resource s3Resource = S3ArnConverter.getInstance().convertArn(resourceArn);
4700
4701             if (S3ResourceType.fromValue(s3Resource.getType()) != S3ResourceType.ACCESS_POINT) {
4702                 throw new IllegalArgumentException("An ARN was passed as a bucket parameter to an S3 operation, "
4703                                                    + "however it does not appear to be a valid S3 access point ARN.");
4704             }
4705
4706             if (isEndpointOverridden()) {
4707                 throw new IllegalArgumentException("An access point ARN cannot be passed as a bucket parameter to an S3"
4708                                                    + " operation if the S3 client has been configured with an endpoint "
4709                                                    + "override.");
4710             }
4711
4712             if (isRegionFipsEnabled(getRegionName())) {
4713                 throw new IllegalArgumentException("An access point ARN cannot be passed as a bucket parameter to an S3"
4714                                                    + " operation if the S3 client has been configured with a FIPS"
4715                                                    + " enabled region.");
4716             }
4717
4718             if (clientOptions.isAccelerateModeEnabled()) {
4719                 throw new IllegalArgumentException("An access point ARN cannot be passed as a bucket parameter to an S3 "
4720                                                    + "operation if the S3 client has been configured with accelerate mode"
4721                                                    + " enabled.");
4722             }
4723
4724             if (clientOptions.isPathStyleAccess()) {
4725                 throw new IllegalArgumentException("An access point ARN cannot be passed as a bucket parameter to an S3 "
4726                                                    + "operation if the S3 client has been configured with path style "
4727                                                    + "addressing enabled.");
4728             }
4729
4730             S3AccessPointResource s3EndpointResource = (S3AccessPointResource) s3Resource;
4731             com.amazonaws.regions.Region clientRegion = RegionUtils.getRegion(getRegionName());
4732             validateS3ResourceArn(resourceArn, clientRegion);
4733             endpoint = S3AccessPointBuilder.create()
4734                                            .withAccessPointName(s3EndpointResource.getAccessPointName())
4735                                            .withAccountId(s3EndpointResource.getAccountId())
4736                                            .withRegion(s3EndpointResource.getRegion())
4737                                            .withProtocol(clientConfiguration.getProtocol().toString())
4738                                            .withDomain(clientRegion.getDomain())
4739                                            .withDualstackEnabled(clientOptions.isDualstackEnabled())
4740                                            .toURI();
4741             signingRegion = s3EndpointResource.getRegion();
4742
4743             request.addHandlerContext(HandlerContextKey.SIGNING_REGION, signingRegion);
4744             resolveAccessPointEndpoint(request, null /* bucketName */, key, endpoint);
4745             return request;
4746         } else {
4747             signingRegion = getSigningRegion();
4748         }
4749
4750         // If the underlying AmazonS3Client has enabled accelerate mode and the original
4751         // request operation is accelerate mode supported, then the request will use the
4752         // s3-accelerate endpoint to performe the operations.
4753         if (clientOptions.isAccelerateModeEnabled() && !(originalRequest instanceof S3AccelerateUnsupported)) {
4754             if (clientOptions.isDualstackEnabled()) {
4755                 endpoint = RuntimeHttpUtils.toUri(Constants.S3_ACCELERATE_DUALSTACK_HOSTNAME, clientConfiguration);
4756             } else {
4757                 endpoint = RuntimeHttpUtils.toUri(Constants.S3_ACCELERATE_HOSTNAME, clientConfiguration);
4758             }
4759         }
4760
4761         resolveRequestEndpoint(request, bucketName, key, endpoint);
4762         request.addHandlerContext(HandlerContextKey.SIGNING_REGION, signingRegion);
4763         return request;
4764     }
4765
4766     private void validateS3ResourceArn(Arn resourceArn, com.amazonaws.regions.Region clientRegion) {
4767         String clientPartition = (clientRegion == null) ? null : clientRegion.getPartition();
4768
4769         if (clientPartition == null || !clientPartition.equals(resourceArn.getPartition())) {
4770             throw new IllegalArgumentException("The partition field of the ARN being passed as a bucket parameter to "
4771                     + "an S3 operation does not match the partition the S3 client has been configured with. Provided "
4772                                                + "partition: '" + resourceArn.getPartition() + "'; client partition: "
4773                                                + "'" + clientPartition + "'.");
4774         }
4775
4776         if (!clientOptions.isForceGlobalBucketAccessEnabled() && !useArnRegion()) {
4777             if (!clientRegion.getName().equals(resourceArn.getRegion())) {
4778                 throw new IllegalArgumentException("The region field of the ARN being passed as a bucket parameter to an "
4779                         + "S3 operation does not match the region the client was configured "
4780                         + "with. Provided region: '" + resourceArn.getRegion() + "'; client "
4781                         + "region: '" + clientRegion.getName() + "'.");
4782             }
4783         }
4784     }
4785
4786     private boolean useArnRegion() {
4787
4788         // If useArnRegion is false, it was not set to false by the customer, it was simply not enabled
4789         if (clientOptions.isUseArnRegion()) {
4790             return clientOptions.isUseArnRegion();
4791         }
4792
4793         return USE_ARN_REGION_RESOLVER.useArnRegion();
4794     }
4795
4796     /**
4797      * Short circuit endpoint logic when working with access points as the endpoint
4798      * is constructed in advance. This resolver will take the existing endpoint and append
4799      * the correct path.
4800      */
4801     private void resolveAccessPointEndpoint(Request<?> request, String bucketName, String key, URI endpoint) {
4802         ServiceEndpointBuilder builder = new IdentityEndpointBuilder(endpoint);
4803         buildEndpointResolver(builder, bucketName, key).resolveRequestEndpoint(request);
4804     }
4805
4806     /**
4807      * Configure the given request with an endpoint and resource path based on the bucket name and
4808      * key provided
4809      */
4810     private void resolveRequestEndpoint(Request<?> request, String bucketName, String key, URI endpoint) {
4811         ServiceEndpointBuilder builder = getBuilder(endpoint, endpoint.getScheme(), false);
4812         buildEndpointResolver(builder, bucketName, key).resolveRequestEndpoint(request);
4813     }
4814
4815     private S3RequestEndpointResolver buildDefaultEndpointResolver(String protocol, String bucketName, String key) {
4816         ServiceEndpointBuilder builder = getBuilder(endpoint, protocol, true);
4817         return new S3RequestEndpointResolver(builder, clientOptions.isPathStyleAccess(), bucketName, key);
4818     }
4819
4820     private ServiceEndpointBuilder getBuilder(URI endpoint, String protocol, boolean useDefaultBuilder) {
4821         if(clientOptions.isDualstackEnabled() && !clientOptions.isAccelerateModeEnabled()) {
4822             return new DualstackEndpointBuilder(getServiceNameIntern(), protocol, getRegion().toAWSRegion());
4823         } else {
4824             if(useDefaultBuilder) {
4825                 return new DefaultServiceEndpointBuilder(getServiceName(), protocol);
4826             } else {
4827                 return new IdentityEndpointBuilder(endpoint);
4828             }
4829         }
4830     }
4831
4832     public PresignedUrlDownloadResult download(PresignedUrlDownloadRequest presignedUrlDownloadRequest) throws SdkClientException {
4833         assertNotNull(presignedUrlDownloadRequest.getPresignedUrl(), "Presigned URL");
4834         final ProgressListener listener = presignedUrlDownloadRequest.getGeneralProgressListener();
4835
4836         Request<PresignedUrlDownloadRequest> request = createRequestForPresignedUrl(presignedUrlDownloadRequest, HttpMethodName.GET,
4837                                                                                     presignedUrlDownloadRequest.getPresignedUrl());
4838         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetObject");
4839         request.addHandlerContext(HandlerContextKey.HAS_STREAMING_OUTPUT, Boolean.TRUE);
4840
4841         // set range header if present on request
4842         long[] range = presignedUrlDownloadRequest.getRange();
4843         if (range != null) {
4844             request.addHeader(Headers.RANGE, "bytes=" + Long.toString(range[0]) + "-" + Long.toString(range[1]));
4845         }
4846
4847         try {
4848             publishProgress(listener, ProgressEventType.TRANSFER_STARTED_EVENT);
4849
4850             S3Object s3Object = client.execute(request,
4851                                                new S3ObjectResponseHandler(),
4852                                                errorResponseHandler,
4853                                                createExecutionContext(AmazonWebServiceRequest.NOOP, new NoOpSignerProvider()),
4854                                                requestConfigWithSkipAppendUriPath(request))
4855                                       .getAwsResponse();
4856
4857             boolean skipClientSideValidation = skipMd5CheckStrategy.skipClientSideValidation(presignedUrlDownloadRequest,
4858                                                                                              s3Object.getObjectMetadata());
4859             postProcessS3Object(s3Object, skipClientSideValidation, listener);
4860
4861             return new PresignedUrlDownloadResult().withS3Object(s3Object);
4862         } catch (AmazonS3Exception ase) {
4863             publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT);
4864             throw ase;
4865         }
4866     }
4867
4868     public void download(final PresignedUrlDownloadRequest presignedUrlDownloadRequest, final File destinationFile)
4869         throws SdkClientException {
4870         assertNotNull(destinationFile, "Destination file");
4871
4872         ServiceUtils.retryableDownloadS3ObjectToFile(destinationFile, new ServiceUtils.RetryableS3DownloadTask() {
4873
4874             @Override
4875             public S3Object getS3ObjectStream() {
4876                 return download(presignedUrlDownloadRequest).getS3Object();
4877             }
4878
4879             @Override
4880             public boolean needIntegrityCheck() {
4881                 return !skipMd5CheckStrategy.skipClientSideValidationPerRequest(presignedUrlDownloadRequest);
4882             }
4883
4884         }, ServiceUtils.OVERWRITE_MODE);
4885     }
4886
4887     public PresignedUrlUploadResult upload(PresignedUrlUploadRequest presignedUrlUploadRequest) {
4888         presignedUrlUploadRequest = beforeClientExecution(presignedUrlUploadRequest);
4889         rejectNull(presignedUrlUploadRequest, "The PresignedUrlUploadRequest object cannot be null");
4890         rejectNull(presignedUrlUploadRequest.getPresignedUrl(), "Presigned URL");
4891
4892         final File file = presignedUrlUploadRequest.getFile();
4893         final InputStream isOrig = presignedUrlUploadRequest.getInputStream();
4894         final ProgressListener listener = presignedUrlUploadRequest.getGeneralProgressListener();
4895
4896         ObjectMetadata metadata = presignedUrlUploadRequest.getMetadata();
4897         if (metadata == null)
4898             metadata = new ObjectMetadata();
4899
4900         Request<PresignedUrlUploadRequest> request = createRequestForPresignedUrl(presignedUrlUploadRequest,
4901                                                                                   presignedUrlUploadRequest.getHttpMethodName(),
4902                                                                                   presignedUrlUploadRequest.getPresignedUrl());
4903         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutObject");
4904
4905         // Make backward compatible with buffer size via system property
4906         final Integer bufsize = Constants.getS3StreamBufferSize();
4907         if (bufsize != null) {
4908             AmazonWebServiceRequest awsreq = request.getOriginalRequest();
4909             // Note awsreq is never null at this point even if the original
4910             // request was
4911             awsreq.getRequestClientOptions()
4912                   .setReadLimit(bufsize.intValue());
4913         }
4914
4915         return uploadObject(isOrig, file, metadata, listener, request, presignedUrlUploadRequest,
4916                             // server side validation is skipped so that SDK doesn't send Content-MD5 header.
4917                             // As this is signed header, only users know if this header is signed in the presigned url or not.
4918                             // To enable server side validation, users should set the md5 value through ObjectMetadata
4919                             true,
4920                             skipMd5CheckStrategy.skipClientSideValidationPerRequest(presignedUrlUploadRequest),
4921                             new PresignedUrlUploadStrategy(presignedUrlUploadRequest.getPresignedUrl()),
4922                             !isSigV2PresignedUrl(presignedUrlUploadRequest.getPresignedUrl()));
4923     }
4924
4925     /**
4926      * Request config used by the APIs that execute presigned urls. Without this config, SDK appends slash ("/") at the end
4927      * of uri while making the request which will fail for presigned urls.
4928      *
4929      * Used in {@link #download(PresignedUrlDownloadRequest)} and {@link #upload(PresignedUrlUploadRequest)} operations.
4930      */
4931     private RequestConfig requestConfigWithSkipAppendUriPath(Request request) {
4932         RequestConfig config = new AmazonWebServiceRequestAdapter(request.getOriginalRequest());
4933         config.getRequestClientOptions().setSkipAppendUriPath(true);
4934         return config;
4935     }
4936
4937     /**
4938      * Method to create {@link Request} object. This method is intended to be used only by operations that use S3
4939      * presigned URL.
4940      *
4941      * This method has special behavior that is not used by normal APIs (that don't use presigned url) like:
4942      * 1) Setting empty Content-Type header when url is signed with SigV2 signer
4943      * 2) Ignoring signer, credentials when constructing the request object
4944      *
4945      */
4946     private <X extends AmazonWebServiceRequest> Request<X> createRequestForPresignedUrl(X originalRequest,
4947                                                                                         HttpMethodName httpMethod,
4948                                                                                         URL endpoint) {
4949         Request<X> request = new DefaultRequest<X>(originalRequest, Constants.S3_SERVICE_DISPLAY_NAME);
4950         request.setHttpMethod(httpMethod);
4951
4952         try {
4953             request.setEndpoint(endpoint.toURI());
4954         } catch (URISyntaxException e) {
4955             throw new SdkClientException(e);
4956         }
4957
4958         if (originalRequest.getCustomRequestHeaders() != null) {
4959             for (Map.Entry<String, String> entry : originalRequest.getCustomRequestHeaders().entrySet()) {
4960                 request.addHeader(entry.getKey(), entry.getValue());
4961             }
4962         }
4963
4964         // Content-Type is signed in SigV2 signer
4965         // If no Content-Type is present, SDK puts a default type in Apache http layer, which will make the call fail with
4966         // Signature error. So set it to empty string for sigv2 presigned urls when user don't provide a Content-Type value
4967         if (request.getHeaders().get(Headers.CONTENT_TYPE) == null && isSigV2PresignedUrl(endpoint)) {
4968             request.addHeader(Headers.CONTENT_TYPE, "");
4969         }
4970
4971         request.addHandlerContext(S3HandlerContextKeys.IS_CHUNKED_ENCODING_DISABLED,
4972                                   Boolean.valueOf(clientOptions.isChunkedEncodingDisabled()));
4973         request.addHandlerContext(S3HandlerContextKeys.IS_PAYLOAD_SIGNING_ENABLED,
4974                                   Boolean.valueOf(clientOptions.isPayloadSigningEnabled()));
4975         request.addHandlerContext(HandlerContextKey.SERVICE_ID, SERVICE_ID);
4976
4977         return request;
4978     }
4979
4980     /**
4981      * SigV2 presigned url has "AWSAccessKeyId" in the params. Also doing "X-Amz-Algorithm" check to ensure
4982      * "AWSAccessKeyId=" is not present in the bucket or key name
4983      */
4984     private boolean isSigV2PresignedUrl(URL presignedUrl) {
4985         String url = presignedUrl.toString();
4986         return url.contains("AWSAccessKeyId=") && !presignedUrl.toString().contains("X-Amz-Algorithm=AWS4-HMAC-SHA256");
4987     }
4988
4989     private S3RequestEndpointResolver buildEndpointResolver(ServiceEndpointBuilder serviceEndpointBuilder, String bucketName, String key) {
4990         return new S3RequestEndpointResolver(serviceEndpointBuilder, clientOptions.isPathStyleAccess(), bucketName, key);
4991     }
4992
4993     @Override
4994     protected final SignerProvider createSignerProvider(Signer signer) {
4995         return new S3SignerProvider(this, signer);
4996     }
4997
4998     private <X, Y extends AmazonWebServiceRequest> X invoke(Request<Y> request,
4999                                                             Unmarshaller<X, InputStream> unmarshaller,
5000                                                             String bucketName,
5001                                                             String key) {
5002         return invoke(request, new S3XmlResponseHandler<X>(unmarshaller), bucketName, key);
5003     }
5004
5005     private <X, Y extends AmazonWebServiceRequest> X invoke(Request<Y> request,
5006                                                             HttpResponseHandler<AmazonWebServiceResponse<X>> responseHandler,
5007                                                             String bucket, String key) {
5008         return invoke(request, responseHandler, bucket, key, false);
5009     }
5010
5011     private <X, Y extends AmazonWebServiceRequest> X invoke(Request<Y> request,
5012                                                             HttpResponseHandler<AmazonWebServiceResponse<X>> responseHandler,
5013                                                             String bucket, String key, boolean isAdditionalHeadRequestToFindRegion) {
5014
5015
5016         AmazonWebServiceRequest originalRequest = request.getOriginalRequest();
5017         checkHttps(originalRequest);
5018         S3SignerProvider signerProvider = new S3SignerProvider(this, getSigner());
5019         ExecutionContext executionContext = createExecutionContext(originalRequest, signerProvider);
5020         AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics();
5021         // Binds the request metrics to the current request.
5022         request.setAWSRequestMetrics(awsRequestMetrics);
5023         // Having the ClientExecuteTime defined here is not ideal (for the
5024         // timing measurement should start as close to the top of the call
5025         // stack of the service client method as possible)
5026         // but definitely a safe compromise for S3 at least for now.
5027         // We can incrementally make it more elaborate should the need arise
5028         // for individual method.
5029         awsRequestMetrics.startEvent(Field.ClientExecuteTime);
5030         Response<X> response = null;
5031         try {
5032             request.setTimeOffset(timeOffset);
5033             /*
5034              * The string we sign needs to include the exact headers that we
5035              * send with the request, but the client runtime layer adds the
5036              * Content-Type header before the request is sent if one isn't set,
5037              * so we have to set something here otherwise the request will fail.
5038              */
5039             if (!request.getHeaders().containsKey(Headers.CONTENT_TYPE)) {
5040                 request.addHeader(Headers.CONTENT_TYPE,
5041                     "application/octet-stream");
5042             }
5043
5044             // Update the bucketRegionCache if we can't find region for the request
5045             if (!isAdditionalHeadRequestToFindRegion && shouldPerformHeadRequestToFindRegion(request, bucket)) {
5046                 fetchRegionFromCache(bucket);
5047             }
5048
5049             Signer signer = createSigner(request, bucket, key, isAdditionalHeadRequestToFindRegion);
5050             signerProvider.setSigner(signer);
5051
5052             // Retry V4 auth errors if signer is explicitly overridden and
5053             // signer is not a SigV4 signer.
5054             if (isSignerOverridden() && !(signer instanceof AWSS3V4Signer)) {
5055                 executionContext.setAuthErrorRetryStrategy(
5056                         new S3V4AuthErrorRetryStrategy(buildDefaultEndpointResolver(getProtocol(request), bucket, key)));
5057             }
5058
5059             executionContext.setCredentialsProvider(CredentialUtils.getCredentialsProvider(request.getOriginalRequest(), awsCredentialsProvider));
5060
5061             validateRequestBeforeTransmit(request);
5062             response = client.execute(request, responseHandler, errorResponseHandler, executionContext);
5063
5064             return response.getAwsResponse();
5065         } catch (ResetException ex) {
5066             ex.setExtraInfo("If the request involves an input stream, the maximum stream buffer size can be configured via request.getRequestClientOptions().setReadLimit(int)");
5067             throw ex;
5068         } catch (AmazonS3Exception ase) {
5069             /**
5070              * This is to handle the edge case: when the bucket is deleted and recreated in a different region,
5071              * the cache still has the old region info.
5072              * If region is not specified, the first request to this newly created bucket will fail because it used
5073              * the outdated region present in cache. Here we update the cache with correct region. The subsequent
5074              * requests will succeed.
5075              * The recommended practice for any request is to provide region info always.
5076              */
5077             if (ase.getStatusCode() == 301) {
5078                 if (ase.getAdditionalDetails() != null) {
5079                     String region = ase.getAdditionalDetails().get(Headers.S3_BUCKET_REGION);
5080                     bucketRegionCache.put(bucket, region);
5081                     ase.setErrorMessage("The bucket is in this region: " + region +
5082                                         ". Please use this region to retry the request");
5083                 }
5084             }
5085             throw ase;
5086         } finally {
5087             endClientExecution(awsRequestMetrics, request, response);
5088         }
5089     }
5090
5091     private void validateRequestBeforeTransmit(Request<?> request) {
5092         boolean implicitCrossRegionForbidden = areImplicitGlobalClientsDisabled();
5093         boolean explicitCrossRegionEnabled = clientOptions.isForceGlobalBucketAccessEnabled();
5094
5095         // The region must be set if implicit cross region clients are not allowed
5096         if (noExplicitRegionProvided(request) && implicitCrossRegionForbidden && !explicitCrossRegionEnabled) {
5097             String error = String.format("While the %s system property is enabled, Amazon S3 clients cannot be used without " +
5098                                          "first configuring a region or explicitly enabling global bucket access discovery " +
5099                                          "in the S3 client builder.",
5100                                          SDKGlobalConfiguration.DISABLE_S3_IMPLICIT_GLOBAL_CLIENTS_SYSTEM_PROPERTY);
5101             throw new IllegalStateException(error);
5102         }
5103     }
5104
5105     /**
5106      * Returns true in the event that the {@link SDKGlobalConfiguration#DISABLE_S3_IMPLICIT_GLOBAL_CLIENTS_SYSTEM_PROPERTY}
5107      * is non-null and not "false".
5108      *
5109      * If this system property is set, S3 clients may not act in a cross-region manner unless cross-region behavior is
5110      * explicitly enabled, using options like {@link AmazonS3ClientBuilder#enableForceGlobalBucketAccess()}.
5111      */
5112     private boolean areImplicitGlobalClientsDisabled() {
5113         String setting = System.getProperty(SDKGlobalConfiguration.DISABLE_S3_IMPLICIT_GLOBAL_CLIENTS_SYSTEM_PROPERTY);
5114         return setting != null && !setting.equals("false");
5115     }
5116
5117     private boolean shouldPerformHeadRequestToFindRegion(Request<?> request, String bucket) {
5118         return bucket != null && !isAccessPointArn(bucket) &&
5119                !(request.getOriginalRequest() instanceof CreateBucketRequest) &&
5120                bucketRegionShouldBeCached(request);
5121     }
5122
5123     private boolean isAccessPointArn(String s) {
5124         return s != null && s.startsWith("arn:") && s.contains(":accesspoint");
5125     }
5126
5127     private boolean bucketRegionShouldBeCached(Request<?> request) {
5128         return clientOptions.isForceGlobalBucketAccessEnabled() || noExplicitRegionProvided(request);
5129     }
5130
5131     @Override
5132     public void enableRequesterPays(String bucketName) {
5133         RequestPaymentConfiguration configuration = new RequestPaymentConfiguration(
5134                 Payer.Requester);
5135
5136         setBucketRequestPayment(new SetRequestPaymentConfigurationRequest(
5137                 bucketName, configuration));
5138     }
5139
5140     @Override
5141     public void disableRequesterPays(String bucketName) {
5142         RequestPaymentConfiguration configuration = new RequestPaymentConfiguration(
5143                 Payer.BucketOwner);
5144
5145         setBucketRequestPayment(new SetRequestPaymentConfigurationRequest(
5146                 bucketName, configuration));
5147     }
5148
5149     @Override
5150     public boolean isRequesterPaysEnabled(String bucketName) {
5151         RequestPaymentConfiguration configuration = getBucketRequestPayment(new GetRequestPaymentConfigurationRequest(
5152                 bucketName));
5153         return (configuration.getPayer() == Payer.Requester);
5154     }
5155
5156     /**
5157      * Sets the request payment configuration for a given Amazon S3 bucket.
5158      * This operation can be done only by the owner of the Amazon S3 bucket.
5159      * <p>
5160      * When the request payment configuration for a Amazon S3 bucket is set to
5161      * <code>Requester</code>, the requester instead of the bucket owner pays
5162      * the cost of the request and the data download from the bucket. The bucket
5163      * owner always pays the cost of storing data.
5164      */
5165     private void setBucketRequestPayment(
5166             SetRequestPaymentConfigurationRequest setRequestPaymentConfigurationRequest) {
5167
5168         String bucketName = setRequestPaymentConfigurationRequest
5169                 .getBucketName();
5170         RequestPaymentConfiguration configuration = setRequestPaymentConfigurationRequest
5171                 .getConfiguration();
5172
5173         rejectNull(bucketName,
5174                 "The bucket name parameter must be specified while setting the Requester Pays.");
5175
5176         rejectNull(
5177                 configuration,
5178                 "The request payment configuration parameter must be specified when setting the Requester Pays.");
5179
5180         Request<SetRequestPaymentConfigurationRequest> request = createRequest(
5181                 bucketName, null, setRequestPaymentConfigurationRequest,
5182                 HttpMethodName.PUT);
5183         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketRequestPayment");
5184         request.addParameter("requestPayment", null);
5185         request.addHeader("Content-Type", "application/xml");
5186
5187         byte[] bytes = requestPaymentConfigurationXmlFactory
5188                 .convertToXmlByteArray(configuration);
5189         request.setContent(new ByteArrayInputStream(bytes));
5190
5191         invoke(request, voidResponseHandler, bucketName, null);
5192     }
5193
5194     /**
5195      * Retrieves the request payment configuration for a given Amazon S3 bucket.
5196      * <p>
5197      * When the request payment configuration for a Amazon S3 bucket is
5198      * <code>Requester</code>, the requester instead of the bucket owner pays
5199      * the cost of the request and the data download from the bucket. The bucket
5200      * owner always pays the cost of storing data.
5201      */
5202     private RequestPaymentConfiguration getBucketRequestPayment(
5203             GetRequestPaymentConfigurationRequest getRequestPaymentConfigurationRequest) {
5204
5205         String bucketName = getRequestPaymentConfigurationRequest
5206                 .getBucketName();
5207
5208         rejectNull(
5209                 bucketName,
5210                 "The bucket name parameter must be specified while getting the Request Payment Configuration.");
5211
5212         Request<GetRequestPaymentConfigurationRequest> request = createRequest(
5213                 bucketName, null, getRequestPaymentConfigurationRequest,
5214                 HttpMethodName.GET);
5215         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketRequestPayment");
5216         request.addParameter("requestPayment", null);
5217         request.addHeader("Content-Type", "application/xml");
5218
5219         return invoke(request,
5220                 new Unmarshallers.RequestPaymentConfigurationUnmarshaller(),
5221                 bucketName, null);
5222     }
5223
5224     private void setZeroContentLength(Request<?> req) {
5225         // https://github.com/aws/aws-sdk-java/pull/215
5226         // http://aws.amazon.com/articles/1109#14
5227         req.addHeader(Headers.CONTENT_LENGTH, String.valueOf(0));
5228     }
5229
5230     /**
5231      * Throws {@link IllegalArgumentException} if SSE customer key is in use
5232      * without https.
5233      */
5234     private void checkHttps(AmazonWebServiceRequest req) {
5235         if (req instanceof SSECustomerKeyProvider) {
5236             SSECustomerKeyProvider p = (SSECustomerKeyProvider) req;
5237             if (p.getSSECustomerKey() != null)
5238                 assertHttps();
5239         } else if (req instanceof CopyObjectRequest) {
5240             CopyObjectRequest cor = (CopyObjectRequest) req;
5241             if (cor.getSourceSSECustomerKey() != null
5242             ||  cor.getDestinationSSECustomerKey() != null) {
5243                 assertHttps();
5244             }
5245         } else if (req instanceof CopyPartRequest) {
5246             CopyPartRequest cpr = (CopyPartRequest) req;
5247             if (cpr.getSourceSSECustomerKey() != null
5248             ||  cpr.getDestinationSSECustomerKey() != null) {
5249                 assertHttps();
5250             }
5251         }
5252
5253         if (req instanceof SSEAwsKeyManagementParamsProvider) {
5254             SSEAwsKeyManagementParamsProvider p = (SSEAwsKeyManagementParamsProvider) req;
5255             if (p.getSSEAwsKeyManagementParams() != null)
5256                 assertHttps();
5257         }
5258     }
5259
5260     private void assertHttps() {
5261         URI endpoint = this.endpoint;
5262         String scheme = endpoint == null ? null : endpoint.getScheme();
5263         if (!Protocol.HTTPS.toString().equalsIgnoreCase(scheme)) {
5264             throw new IllegalArgumentException(
5265                     "HTTPS must be used when sending customer encryption keys (SSE-C) to S3, in order to protect your encryption keys.");
5266         }
5267     }
5268
5269     /**
5270      * For testing
5271      */
5272     synchronized URI getEndpoint() {
5273         return endpoint;
5274     }
5275
5276     private static String getProtocol(Request<?> request) {
5277         if (request == null || request.getEndpoint() == null) {
5278             return null;
5279         }
5280         return request.getEndpoint().getScheme();
5281     }
5282
5283     /**
5284      * Creates and returns a multi-part upload initiation request from the given upload-object
5285      * request.
5286      */
5287     protected final InitiateMultipartUploadRequest newInitiateMultipartUploadRequest(
5288             UploadObjectRequest req) {
5289         return new InitiateMultipartUploadRequest(
5290                 req.getBucketName(), req.getKey(), req.getMetadata())
5291             .withRedirectLocation(req.getRedirectLocation())
5292             .withSSEAwsKeyManagementParams(req.getSSEAwsKeyManagementParams())
5293             .withSSECustomerKey(req.getSSECustomerKey())
5294             .withStorageClass(req.getStorageClass())
5295             .withAccessControlList(req.getAccessControlList())
5296             .withCannedACL(req.getCannedAcl())
5297             .withGeneralProgressListener(req.getGeneralProgressListener())
5298             .withRequestMetricCollector(req.getRequestMetricCollector())
5299             ;
5300     }
5301
5302     /**
5303      * Used for performance testing purposes only.
5304      */
5305     private void putLocalObject(final UploadObjectRequest reqIn,
5306             OutputStream os) throws IOException {
5307         UploadObjectRequest req = reqIn.clone();
5308
5309         final File fileOrig = req.getFile();
5310         final InputStream isOrig = req.getInputStream();
5311
5312         if (isOrig == null) {
5313             if (fileOrig == null)
5314                 throw new IllegalArgumentException("Either a file lor input stream must be specified");
5315             req.setInputStream(new FileInputStream(fileOrig));
5316             req.setFile(null);
5317         }
5318
5319         try {
5320             IOUtils.copy(req.getInputStream(), os);
5321         } finally {
5322             cleanupDataSource(req, fileOrig, isOrig,
5323                     req.getInputStream(), log);
5324             IOUtils.closeQuietly(os, log);
5325         }
5326         return;
5327     }
5328
5329     /**
5330      * Used for performance testing purposes only.  Hence package private.
5331      * This method is subject to removal anytime without notice.
5332      */
5333     CompleteMultipartUploadResult uploadObject(final UploadObjectRequest req)
5334             throws IOException, InterruptedException, ExecutionException {
5335         // Set up the pipeline for concurrent encrypt and upload
5336         // Set up a thread pool for this pipeline
5337         ExecutorService es = req.getExecutorService();
5338         final boolean defaultExecutorService = es == null;
5339         if (es == null)
5340             es = Executors.newFixedThreadPool(clientConfiguration.getMaxConnections());
5341         UploadObjectObserver observer = req.getUploadObjectObserver();
5342         if (observer == null)
5343             observer = new UploadObjectObserver();
5344         // initialize the observer
5345         observer.init(req, this, this, es);
5346         // Initiate upload
5347         observer.onUploadInitiation(req);
5348         final List<PartETag> partETags = new ArrayList<PartETag>();
5349         MultiFileOutputStream mfos = req.getMultiFileOutputStream();
5350         if (mfos == null)
5351             mfos = new MultiFileOutputStream();
5352         try {
5353             // initialize the multi-file output stream
5354             mfos.init(observer, req.getPartSize(), req.getDiskLimit());
5355             // Kicks off the encryption-upload pipeline;
5356             // Note mfos is automatically closed upon method completion.
5357             putLocalObject(req, mfos);
5358             // block till all part have been uploaded
5359             for (Future<UploadPartResult> future: observer.getFutures()) {
5360                 UploadPartResult partResult = future.get();
5361                 partETags.add(new PartETag(partResult.getPartNumber(), partResult.getETag()));
5362             }
5363         } finally {
5364             if (defaultExecutorService)
5365                 es.shutdownNow();   // shut down the locally created thread pool
5366             mfos.cleanup();       // delete left-over temp files
5367         }
5368         // Complete upload
5369         return observer.onCompletion(partETags);
5370     }
5371
5372     @Override
5373     public void setBucketReplicationConfiguration(String bucketName,
5374             BucketReplicationConfiguration configuration)
5375             throws AmazonServiceException, SdkClientException {
5376         setBucketReplicationConfiguration(new SetBucketReplicationConfigurationRequest(
5377                 bucketName, configuration));
5378     }
5379
5380     @Override
5381     public void setBucketReplicationConfiguration(
5382             SetBucketReplicationConfigurationRequest setBucketReplicationConfigurationRequest)
5383             throws AmazonServiceException, SdkClientException {
5384         setBucketReplicationConfigurationRequest = beforeClientExecution(setBucketReplicationConfigurationRequest);
5385         rejectNull(setBucketReplicationConfigurationRequest,
5386                 "The set bucket replication configuration request object must be specified.");
5387
5388         final String bucketName = setBucketReplicationConfigurationRequest
5389                 .getBucketName();
5390
5391         final BucketReplicationConfiguration bucketReplicationConfiguration = setBucketReplicationConfigurationRequest
5392                 .getReplicationConfiguration();
5393
5394         rejectNull(
5395                 bucketName,
5396                 "The bucket name parameter must be specified when setting replication configuration.");
5397         rejectNull(
5398                 bucketReplicationConfiguration,
5399                 "The replication configuration parameter must be specified when setting replication configuration.");
5400
5401         Request<SetBucketReplicationConfigurationRequest> request = createRequest(
5402                 bucketName, null, setBucketReplicationConfigurationRequest,
5403                 HttpMethodName.PUT);
5404         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketReplication");
5405         request.addParameter("replication", null);
5406
5407         final byte[] bytes = bucketConfigurationXmlFactory
5408                 .convertToXmlByteArray(bucketReplicationConfiguration);
5409
5410         addHeaderIfNotNull(request, Headers.OBJECT_LOCK_TOKEN, setBucketReplicationConfigurationRequest.getToken());
5411         request.addHeader("Content-Length", String.valueOf(bytes.length));
5412         request.addHeader("Content-Type", "application/xml");
5413         request.setContent(new ByteArrayInputStream(bytes));
5414
5415
5416         try {
5417             request.addHeader("Content-MD5",
5418                     BinaryUtils.toBase64(Md5Utils.computeMD5Hash(bytes)));
5419         } catch (Exception e) {
5420             throw new SdkClientException(
5421                     "Not able to compute MD5 of the replication rule configuration. Exception Message : "
5422                             + e.getMessage(), e);
5423         }
5424         invoke(request, voidResponseHandler, bucketName, null);
5425     }
5426
5427     @Override
5428     public BucketReplicationConfiguration getBucketReplicationConfiguration(
5429             String bucketName) throws AmazonServiceException,
5430             SdkClientException {
5431       return getBucketReplicationConfiguration(new GetBucketReplicationConfigurationRequest(bucketName));
5432     }
5433
5434     @Override
5435     public BucketReplicationConfiguration getBucketReplicationConfiguration(
5436                 GetBucketReplicationConfigurationRequest getBucketReplicationConfigurationRequest)
5437                     throws AmazonServiceException, SdkClientException {
5438         getBucketReplicationConfigurationRequest = beforeClientExecution(getBucketReplicationConfigurationRequest);
5439         rejectNull(
5440                 getBucketReplicationConfigurationRequest,
5441                 "The bucket request parameter must be specified when retrieving replication configuration");
5442         String bucketName = getBucketReplicationConfigurationRequest.getBucketName();
5443         rejectNull(
5444                 bucketName,
5445                 "The bucket request must specify a bucket name when retrieving replication configuration");
5446
5447         Request<GetBucketReplicationConfigurationRequest> request = createRequest(bucketName, null,
5448                           getBucketReplicationConfigurationRequest, HttpMethodName.GET);
5449         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketReplication");
5450         request.addParameter("replication", null);
5451
5452         return invoke(request,
5453                 new Unmarshallers.BucketReplicationConfigurationUnmarshaller(),
5454                 bucketName, null);
5455     }
5456
5457     @Override
5458     public void deleteBucketReplicationConfiguration(String bucketName)
5459             throws AmazonServiceException, SdkClientException {
5460         deleteBucketReplicationConfiguration(new
5461                 DeleteBucketReplicationConfigurationRequest(bucketName));
5462     }
5463
5464     @Override
5465     public void deleteBucketReplicationConfiguration
5466             (DeleteBucketReplicationConfigurationRequest
5467                      deleteBucketReplicationConfigurationRequest)
5468             throws AmazonServiceException, SdkClientException {
5469         deleteBucketReplicationConfigurationRequest = beforeClientExecution(deleteBucketReplicationConfigurationRequest);
5470         final String bucketName = deleteBucketReplicationConfigurationRequest.getBucketName();
5471         rejectNull(
5472                 bucketName,
5473                 "The bucket name parameter must be specified when deleting replication configuration");
5474
5475         Request<DeleteBucketReplicationConfigurationRequest> request = createRequest(bucketName, null,
5476                 deleteBucketReplicationConfigurationRequest, HttpMethodName
5477                         .DELETE);
5478         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteBucketReplication");
5479         request.addParameter("replication", null);
5480
5481         invoke(request, voidResponseHandler, bucketName, null);
5482     }
5483
5484     @Override
5485     public DeleteBucketMetricsConfigurationResult deleteBucketMetricsConfiguration(
5486             String bucketName, String id) throws AmazonServiceException, SdkClientException {
5487         return deleteBucketMetricsConfiguration(new DeleteBucketMetricsConfigurationRequest(bucketName, id));
5488     }
5489
5490     @Override
5491     public DeleteBucketMetricsConfigurationResult deleteBucketMetricsConfiguration(
5492             DeleteBucketMetricsConfigurationRequest deleteBucketMetricsConfigurationRequest)
5493             throws AmazonServiceException, SdkClientException {
5494         deleteBucketMetricsConfigurationRequest = beforeClientExecution(deleteBucketMetricsConfigurationRequest);
5495         rejectNull(deleteBucketMetricsConfigurationRequest, "The request cannot be null");
5496         final String bucketName = assertStringNotEmpty(deleteBucketMetricsConfigurationRequest.getBucketName(), "BucketName");
5497         final String id = assertStringNotEmpty(deleteBucketMetricsConfigurationRequest.getId(), "Metrics Id");
5498
5499         Request<DeleteBucketMetricsConfigurationRequest> request =
5500                 createRequest(bucketName, null, deleteBucketMetricsConfigurationRequest, HttpMethodName.DELETE);
5501         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteBucketMetricsConfiguration");
5502         request.addParameter("metrics", null);
5503         request.addParameter("id", id);
5504
5505         return invoke(request, new Unmarshallers.DeleteBucketMetricsConfigurationUnmarshaller(), bucketName, null);
5506     }
5507
5508     @Override
5509     public GetBucketMetricsConfigurationResult getBucketMetricsConfiguration(
5510             String bucketName, String id) throws AmazonServiceException, SdkClientException {
5511         return getBucketMetricsConfiguration(new GetBucketMetricsConfigurationRequest(bucketName, id));
5512     }
5513
5514     @Override
5515     public GetBucketMetricsConfigurationResult getBucketMetricsConfiguration(
5516             GetBucketMetricsConfigurationRequest getBucketMetricsConfigurationRequest)
5517             throws AmazonServiceException, SdkClientException {
5518         getBucketMetricsConfigurationRequest = beforeClientExecution(getBucketMetricsConfigurationRequest);
5519         rejectNull(getBucketMetricsConfigurationRequest, "The request cannot be null");
5520         final String bucketName = assertStringNotEmpty(getBucketMetricsConfigurationRequest.getBucketName(), "BucketName");
5521         final String id = assertStringNotEmpty(getBucketMetricsConfigurationRequest.getId(), "Metrics Id");
5522
5523         Request<GetBucketMetricsConfigurationRequest> request =
5524                 createRequest(bucketName, null, getBucketMetricsConfigurationRequest, HttpMethodName.GET);
5525         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketMetricsConfiguration");
5526         request.addParameter("metrics", null);
5527         request.addParameter("id", id);
5528
5529         return invoke(request, new Unmarshallers.GetBucketMetricsConfigurationUnmarshaller(), bucketName, null);
5530     }
5531
5532     public SetBucketMetricsConfigurationResult setBucketMetricsConfiguration(
5533             String bucketName, MetricsConfiguration metricsConfiguration)
5534             throws AmazonServiceException, SdkClientException {
5535         return setBucketMetricsConfiguration(new SetBucketMetricsConfigurationRequest(bucketName, metricsConfiguration));
5536     }
5537
5538     @Override
5539     public SetBucketMetricsConfigurationResult setBucketMetricsConfiguration(
5540             SetBucketMetricsConfigurationRequest setBucketMetricsConfigurationRequest)
5541             throws AmazonServiceException, SdkClientException {
5542         setBucketMetricsConfigurationRequest = beforeClientExecution(setBucketMetricsConfigurationRequest);
5543         new SetBucketMetricsConfigurationRequest();
5544         rejectNull(setBucketMetricsConfigurationRequest, "The request cannot be null");
5545         final String bucketName = assertStringNotEmpty(setBucketMetricsConfigurationRequest.getBucketName(), "BucketName");
5546         final MetricsConfiguration metricsConfiguration = assertNotNull(
5547                 setBucketMetricsConfigurationRequest.getMetricsConfiguration(), "Metrics Configuration");
5548         final String id = assertNotNull(metricsConfiguration.getId(), "Metrics Id");
5549
5550         Request<SetBucketMetricsConfigurationRequest> request =
5551                 createRequest(bucketName, null, setBucketMetricsConfigurationRequest, HttpMethodName.PUT);
5552         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketMetricsConfiguration");
5553         request.addParameter("metrics", null);
5554         request.addParameter("id", id);
5555
5556         byte[] bytes = bucketConfigurationXmlFactory.convertToXmlByteArray(metricsConfiguration);
5557         request.addHeader("Content-Length", String.valueOf(bytes.length));
5558         request.addHeader("Content-Type", "application/xml");
5559         request.setContent(new ByteArrayInputStream(bytes));
5560
5561         return invoke(request, new Unmarshallers.SetBucketMetricsConfigurationUnmarshaller(), bucketName, null);
5562     }
5563
5564     @Override
5565     public ListBucketMetricsConfigurationsResult listBucketMetricsConfigurations(
5566             ListBucketMetricsConfigurationsRequest listBucketMetricsConfigurationsRequest)
5567             throws AmazonServiceException, SdkClientException {
5568         listBucketMetricsConfigurationsRequest = beforeClientExecution(listBucketMetricsConfigurationsRequest);
5569         rejectNull(listBucketMetricsConfigurationsRequest, "The request cannot be null");
5570         final String bucketName = assertStringNotEmpty(listBucketMetricsConfigurationsRequest.getBucketName(), "BucketName");
5571
5572         Request<ListBucketMetricsConfigurationsRequest> request =
5573                 createRequest(bucketName, null, listBucketMetricsConfigurationsRequest, HttpMethodName.GET);
5574         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "ListBucketMetricsConfigurations");
5575         request.addParameter("metrics", null);
5576         addParameterIfNotNull(request, "continuation-token", listBucketMetricsConfigurationsRequest.getContinuationToken());
5577
5578         return invoke(request, new Unmarshallers.ListBucketMetricsConfigurationsUnmarshaller(), bucketName, null);
5579     }
5580
5581     @Override
5582     public DeleteBucketAnalyticsConfigurationResult deleteBucketAnalyticsConfiguration(
5583             String bucketName, String id) throws AmazonServiceException, SdkClientException {
5584         return deleteBucketAnalyticsConfiguration(new DeleteBucketAnalyticsConfigurationRequest(bucketName, id));
5585     }
5586
5587     @Override
5588     public DeleteBucketAnalyticsConfigurationResult deleteBucketAnalyticsConfiguration(
5589             DeleteBucketAnalyticsConfigurationRequest deleteBucketAnalyticsConfigurationRequest)
5590             throws AmazonServiceException, SdkClientException {
5591         deleteBucketAnalyticsConfigurationRequest = beforeClientExecution(deleteBucketAnalyticsConfigurationRequest);
5592         rejectNull(deleteBucketAnalyticsConfigurationRequest, "The request cannot be null");
5593         final String bucketName = assertStringNotEmpty(
5594                 deleteBucketAnalyticsConfigurationRequest.getBucketName(), "BucketName");
5595         final String id = assertStringNotEmpty(
5596                 deleteBucketAnalyticsConfigurationRequest.getId(), "Analytics Id");
5597
5598         Request<DeleteBucketAnalyticsConfigurationRequest> request =
5599                 createRequest(bucketName, null, deleteBucketAnalyticsConfigurationRequest, HttpMethodName.DELETE);
5600         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteBucketAnalyticsConfiguration");
5601         request.addParameter("analytics", null);
5602         request.addParameter("id", id);
5603
5604         return invoke(request, new Unmarshallers.DeleteBucketAnalyticsConfigurationUnmarshaller(), bucketName, null);
5605     }
5606
5607     @Override
5608     public GetBucketAnalyticsConfigurationResult getBucketAnalyticsConfiguration(
5609             String bucketName, String id) throws AmazonServiceException, SdkClientException {
5610         return getBucketAnalyticsConfiguration(new GetBucketAnalyticsConfigurationRequest(bucketName, id));
5611     }
5612
5613     @Override
5614     public GetBucketAnalyticsConfigurationResult getBucketAnalyticsConfiguration(
5615             GetBucketAnalyticsConfigurationRequest getBucketAnalyticsConfigurationRequest)
5616             throws AmazonServiceException, SdkClientException {
5617         getBucketAnalyticsConfigurationRequest = beforeClientExecution(getBucketAnalyticsConfigurationRequest);
5618
5619         rejectNull(getBucketAnalyticsConfigurationRequest, "The request cannot be null");
5620         final String bucketName = assertStringNotEmpty(
5621                 getBucketAnalyticsConfigurationRequest.getBucketName(), "BucketName");
5622         final String id = assertStringNotEmpty(
5623                 getBucketAnalyticsConfigurationRequest.getId(), "Analytics Id");
5624
5625         Request<GetBucketAnalyticsConfigurationRequest> request =
5626                 createRequest(bucketName, null, getBucketAnalyticsConfigurationRequest, HttpMethodName.GET);
5627         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketAnalyticsConfiguration");
5628         request.addParameter("analytics", null);
5629         request.addParameter("id", id);
5630
5631         return invoke(request, new Unmarshallers.GetBucketAnalyticsConfigurationUnmarshaller(), bucketName, null);
5632     }
5633
5634     @Override
5635     public SetBucketAnalyticsConfigurationResult setBucketAnalyticsConfiguration(
5636             String bucketName, AnalyticsConfiguration analyticsConfiguration)
5637             throws AmazonServiceException, SdkClientException {
5638         return setBucketAnalyticsConfiguration(
5639                 new SetBucketAnalyticsConfigurationRequest(bucketName, analyticsConfiguration));
5640     }
5641
5642     @Override
5643     public SetBucketAnalyticsConfigurationResult setBucketAnalyticsConfiguration(
5644             SetBucketAnalyticsConfigurationRequest setBucketAnalyticsConfigurationRequest)
5645             throws AmazonServiceException, SdkClientException {
5646         setBucketAnalyticsConfigurationRequest = beforeClientExecution(setBucketAnalyticsConfigurationRequest);
5647         rejectNull(setBucketAnalyticsConfigurationRequest, "The request cannot be null");
5648         final String bucketName = assertStringNotEmpty(
5649                 setBucketAnalyticsConfigurationRequest.getBucketName(), "BucketName");
5650         final AnalyticsConfiguration analyticsConfiguration = assertNotNull(
5651                 setBucketAnalyticsConfigurationRequest.getAnalyticsConfiguration(), "Analytics Configuration");
5652         final String id = assertNotNull(analyticsConfiguration.getId(), "Analytics Id");
5653
5654         Request<SetBucketAnalyticsConfigurationRequest> request =
5655                 createRequest(bucketName, null, setBucketAnalyticsConfigurationRequest, HttpMethodName.PUT);
5656         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketAnalyticsConfiguration");
5657         request.addParameter("analytics", null);
5658         request.addParameter("id", id);
5659
5660         byte[] bytes = bucketConfigurationXmlFactory.convertToXmlByteArray(analyticsConfiguration);
5661         request.addHeader("Content-Length", String.valueOf(bytes.length));
5662         request.addHeader("Content-Type", "application/xml");
5663         request.setContent(new ByteArrayInputStream(bytes));
5664
5665         return invoke(request, new Unmarshallers.SetBucketAnalyticsConfigurationUnmarshaller(), bucketName, null);
5666     }
5667
5668     @Override
5669     public ListBucketAnalyticsConfigurationsResult listBucketAnalyticsConfigurations(
5670             ListBucketAnalyticsConfigurationsRequest listBucketAnalyticsConfigurationsRequest)
5671             throws AmazonServiceException, SdkClientException {
5672         listBucketAnalyticsConfigurationsRequest = beforeClientExecution(listBucketAnalyticsConfigurationsRequest);
5673         rejectNull(listBucketAnalyticsConfigurationsRequest, "The request cannot be null");
5674         final String bucketName = assertStringNotEmpty(
5675                 listBucketAnalyticsConfigurationsRequest.getBucketName(), "BucketName");
5676
5677         Request<ListBucketAnalyticsConfigurationsRequest> request =
5678                 createRequest(bucketName, null, listBucketAnalyticsConfigurationsRequest, HttpMethodName.GET);
5679         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "ListBucketAnalyticsConfigurations");
5680         request.addParameter("analytics", null);
5681         addParameterIfNotNull(request, "continuation-token", listBucketAnalyticsConfigurationsRequest.getContinuationToken());
5682
5683         return invoke(request, new Unmarshallers.ListBucketAnalyticsConfigurationUnmarshaller(), bucketName, null);
5684     }
5685
5686     @Override
5687     public DeleteBucketInventoryConfigurationResult deleteBucketInventoryConfiguration(
5688             String bucketName, String id) throws AmazonServiceException, SdkClientException {
5689         return deleteBucketInventoryConfiguration(
5690                 new DeleteBucketInventoryConfigurationRequest(bucketName, id));
5691     }
5692
5693     @Override
5694     public DeleteBucketInventoryConfigurationResult deleteBucketInventoryConfiguration(
5695             DeleteBucketInventoryConfigurationRequest deleteBucketInventoryConfigurationRequest)
5696             throws AmazonServiceException, SdkClientException {
5697         deleteBucketInventoryConfigurationRequest = beforeClientExecution(deleteBucketInventoryConfigurationRequest);
5698         rejectNull(deleteBucketInventoryConfigurationRequest, "The request cannot be null");
5699         String bucketName = assertStringNotEmpty(deleteBucketInventoryConfigurationRequest.getBucketName(), "BucketName");
5700         String id = assertStringNotEmpty(deleteBucketInventoryConfigurationRequest.getId(), "Inventory id");
5701
5702         Request<DeleteBucketInventoryConfigurationRequest> request = createRequest(bucketName, null, deleteBucketInventoryConfigurationRequest, HttpMethodName.DELETE);
5703         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteBucketInventoryConfiguration");
5704         request.addParameter("inventory", null);
5705         request.addParameter("id", id);
5706
5707         return invoke(request, new Unmarshallers.DeleteBucketInventoryConfigurationUnmarshaller(), bucketName, null);
5708     }
5709
5710     @Override
5711     public GetBucketInventoryConfigurationResult getBucketInventoryConfiguration(
5712             String bucketName, String id) throws AmazonServiceException, SdkClientException {
5713         return getBucketInventoryConfiguration(
5714                 new GetBucketInventoryConfigurationRequest(bucketName, id));
5715     }
5716
5717     @Override
5718     public GetBucketInventoryConfigurationResult getBucketInventoryConfiguration(
5719             GetBucketInventoryConfigurationRequest getBucketInventoryConfigurationRequest)
5720             throws AmazonServiceException, SdkClientException {
5721         getBucketInventoryConfigurationRequest = beforeClientExecution(getBucketInventoryConfigurationRequest);
5722         rejectNull(getBucketInventoryConfigurationRequest, "The request cannot be null");
5723         String bucketName = assertStringNotEmpty(getBucketInventoryConfigurationRequest.getBucketName(), "BucketName");
5724         String id = assertStringNotEmpty(getBucketInventoryConfigurationRequest.getId(), "Inventory id");
5725
5726         Request<GetBucketInventoryConfigurationRequest> request = createRequest(bucketName, null, getBucketInventoryConfigurationRequest, HttpMethodName.GET);
5727         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetBucketInventoryConfiguration");
5728         request.addParameter("inventory", null);
5729         request.addParameter("id", id);
5730
5731         return invoke(request, new Unmarshallers.GetBucketInventoryConfigurationUnmarshaller(), bucketName, null);
5732     }
5733
5734     @Override
5735     public SetBucketInventoryConfigurationResult setBucketInventoryConfiguration(
5736             String bucketName, InventoryConfiguration inventoryConfiguration)
5737             throws AmazonServiceException, SdkClientException {
5738         return setBucketInventoryConfiguration(
5739                 new SetBucketInventoryConfigurationRequest(bucketName, inventoryConfiguration));
5740     }
5741
5742     @Override
5743     public SetBucketInventoryConfigurationResult setBucketInventoryConfiguration(
5744             SetBucketInventoryConfigurationRequest setBucketInventoryConfigurationRequest)
5745             throws AmazonServiceException, SdkClientException {
5746         setBucketInventoryConfigurationRequest = beforeClientExecution(setBucketInventoryConfigurationRequest);
5747         rejectNull(setBucketInventoryConfigurationRequest, "The request cannot be null");
5748         final String bucketName = assertStringNotEmpty(setBucketInventoryConfigurationRequest.getBucketName(), "BucketName");
5749         final InventoryConfiguration inventoryConfiguration = assertNotNull(setBucketInventoryConfigurationRequest.getInventoryConfiguration(),
5750                 "InventoryConfiguration");
5751         final String id = assertNotNull(inventoryConfiguration.getId(), "Inventory id");
5752
5753         Request<SetBucketInventoryConfigurationRequest> request = createRequest(bucketName, null, setBucketInventoryConfigurationRequest, HttpMethodName.PUT);
5754         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "PutBucketInventoryConfiguration");
5755         request.addParameter("inventory", null);
5756         request.addParameter("id", id);
5757
5758         final byte[] bytes = bucketConfigurationXmlFactory.convertToXmlByteArray(inventoryConfiguration);
5759         request.addHeader("Content-Length", String.valueOf(bytes.length));
5760         request.addHeader("Content-Type", "application/xml");
5761         request.setContent(new ByteArrayInputStream(bytes));
5762
5763         return invoke(request, new Unmarshallers.SetBucketInventoryConfigurationUnmarshaller(), bucketName, null);
5764     }
5765
5766     @Override
5767     public ListBucketInventoryConfigurationsResult listBucketInventoryConfigurations(ListBucketInventoryConfigurationsRequest listBucketInventoryConfigurationsRequest)
5768             throws AmazonServiceException, SdkClientException {
5769         listBucketInventoryConfigurationsRequest = beforeClientExecution(listBucketInventoryConfigurationsRequest);
5770         rejectNull(listBucketInventoryConfigurationsRequest, "The request cannot be null");
5771         final String bucketName = assertStringNotEmpty(listBucketInventoryConfigurationsRequest.getBucketName(), "BucketName");
5772
5773         Request<ListBucketInventoryConfigurationsRequest> request = createRequest(bucketName, null, listBucketInventoryConfigurationsRequest, HttpMethodName.GET);
5774         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "ListBucketInventoryConfigurations");
5775         request.addParameter("inventory", null);
5776         addParameterIfNotNull(request, "continuation-token", listBucketInventoryConfigurationsRequest.getContinuationToken());
5777
5778         return invoke(request, new Unmarshallers.ListBucketInventoryConfigurationsUnmarshaller(), bucketName, null);
5779     }
5780
5781     /**
5782      * Specifically made package access for testing.
5783      * Used for internal consumption of AWS SDK.
5784      *
5785      * Tries to determine the service endpoint for the bucket name.
5786      * Returns the endpoint configured in the client if the region cannot be determined.
5787      */
5788     URI resolveServiceEndpointFromBucketName(String bucketName) {
5789
5790         if (getSignerRegion() != null || isSignerOverridden()) return endpoint;
5791
5792         final String regionStr = fetchRegionFromCache(bucketName);
5793         return resolveServiceEndpointFromRegion(regionStr);
5794     }
5795
5796     private URI resolveServiceEndpointFromRegion(String regionName) {
5797         final com.amazonaws.regions.Region region = RegionUtils.getRegion(regionName);
5798
5799         if (region == null) {
5800             log.warn("Region information for "
5801                     + regionName
5802                     + " is not available. Please upgrade to latest version of AWS Java SDK");
5803         }
5804
5805         return region != null
5806                 ? RuntimeHttpUtils.toUri(region.getServiceEndpoint(S3_SERVICE_NAME), clientConfiguration)
5807                 : endpoint;
5808     }
5809
5810     /**
5811      * Fetches the region of the bucket from the cache maintained. If the cache
5812      * doesn't have an entry, fetches the region from Amazon S3 and updates the
5813      * cache.
5814      */
5815     private String fetchRegionFromCache(String bucketName) {
5816         String bucketRegion = bucketRegionCache.get(bucketName);
5817         if (bucketRegion == null) {
5818             if (log.isDebugEnabled()) {
5819                 log.debug("Bucket region cache doesn't have an entry for " + bucketName
5820                         + ". Trying to get bucket region from Amazon S3.");
5821             }
5822
5823             bucketRegion = getBucketRegionViaHeadRequest(bucketName);
5824
5825             if (bucketRegion != null) {
5826                 bucketRegionCache.put(bucketName, bucketRegion);
5827             }
5828         }
5829         if (log.isDebugEnabled()) {
5830             log.debug("Region for " + bucketName + " is " + bucketRegion);
5831         }
5832         return bucketRegion;
5833     }
5834
5835     /**
5836      * Retrieves the region of the bucket by making a HeadBucket request to us-west-1 region.
5837      *
5838      * Currently S3 doesn't return region in a HEAD Bucket request if the bucket
5839      * owner has enabled bucket to accept only SigV4 requests via bucket
5840      * policies.
5841      */
5842     private String getBucketRegionViaHeadRequest(String bucketName) {
5843         String bucketRegion = null;
5844
5845         try {
5846             Request<HeadBucketRequest> request = createRequest(bucketName, null,
5847                     new HeadBucketRequest(bucketName), HttpMethodName.HEAD);
5848             request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "HeadBucket");
5849
5850             HeadBucketResult result = invoke(request, new HeadBucketResultHandler(), bucketName, null, true);
5851             bucketRegion = result.getBucketRegion();
5852         } catch (AmazonS3Exception exception) {
5853             if (exception.getAdditionalDetails() != null) {
5854                 bucketRegion = exception.getAdditionalDetails().get(
5855                     Headers.S3_BUCKET_REGION);
5856             }
5857         }
5858
5859         if (bucketRegion == null && log.isDebugEnabled()) {
5860             log.debug("Not able to derive region of the " + bucketName + " from the HEAD Bucket requests.");
5861         }
5862
5863         return bucketRegion;
5864     }
5865
5866     public AmazonS3Waiters waiters(){
5867         if (waiters == null) {
5868             synchronized (this) {
5869                 if (waiters == null) {
5870                     waiters = new AmazonS3Waiters(this);
5871                 }
5872             }
5873         }
5874         return waiters;
5875     }
5876
5877     private String urlEncodeTags(ObjectTagging tagging) {
5878         if (tagging == null || tagging.getTagSet() == null) return null;
5879
5880         StringBuilder sb = new StringBuilder();
5881
5882         Iterator<Tag> tagIter = tagging.getTagSet().iterator();
5883         while (tagIter.hasNext()) {
5884             Tag tag = tagIter.next();
5885             sb.append(SdkHttpUtils.urlEncode(tag.getKey(), false)).append('=').append(SdkHttpUtils.urlEncode(tag.getValue(), false));
5886             if (tagIter.hasNext()) {
5887                 sb.append("&");
5888             }
5889         }
5890
5891         return sb.toString();
5892     }
5893
5894     private void setContent(Request<?> request, byte[] content, String contentType, boolean setMd5) {
5895         request.setContent(new ByteArrayInputStream(content));
5896         request.addHeader("Content-Length", Integer.toString(content.length));
5897         request.addHeader("Content-Type", contentType);
5898         if (setMd5) {
5899             try {
5900                 byte[] md5 = Md5Utils.computeMD5Hash(content);
5901                 String md5Base64 = BinaryUtils.toBase64(md5);
5902                 request.addHeader("Content-MD5", md5Base64);
5903             } catch ( Exception e ) {
5904                 throw new AmazonClientException("Couldn't compute md5 sum", e);
5905             }
5906         }
5907     }
5908
5909     private Request<RestoreObjectRequest> createRestoreObjectRequest(RestoreObjectRequest restoreObjectRequest) {
5910         String bucketName = restoreObjectRequest.getBucketName();
5911         String key = restoreObjectRequest.getKey();
5912         String versionId = restoreObjectRequest.getVersionId();
5913
5914         Request<RestoreObjectRequest> request = createRequest(
5915             bucketName, key, restoreObjectRequest, HttpMethodName.POST);
5916         request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "RestoreObject");
5917         request.addParameter("restore", null);
5918         if (versionId != null) {
5919             request.addParameter("versionId", versionId);
5920         }
5921
5922         populateRequesterPaysHeader(request, restoreObjectRequest.isRequesterPays());
5923         byte[] content = RequestXmlFactory.convertToXmlByteArray(restoreObjectRequest);
5924         setContent(request, content, "application/xml", true);
5925         return request;
5926     }
5927
5928     private static void populateObjectLockHeaders(Request<?> request, String mode, Date retainUntil, String status) {
5929         addHeaderIfNotNull(request, Headers.OBJECT_LOCK_MODE, mode);
5930         if (retainUntil != null) {
5931             request.addHeader(Headers.OBJECT_LOCK_RETAIN_UNTIL_DATE, ServiceUtils.formatIso8601Date(retainUntil));
5932         }
5933         addHeaderIfNotNull(request, Headers.OBJECT_LOCK_LEGAL_HOLD_STATUS, status);
5934     }
5935
5936     /**
5937      * Upload strategy to use in {@link #putObject(PutObjectRequest)} API
5938      */
5939     private class PutObjectStrategy implements UploadObjectStrategy<PutObjectRequest, PutObjectResult> {
5940         private final String bucketName;
5941         private final String key;
5942
5943         private PutObjectStrategy(String bucketName, String key) {
5944             this.bucketName = bucketName;
5945             this.key = key;
5946         }
5947
5948         @Override
5949         public ObjectMetadata invokeServiceCall(Request<PutObjectRequest> request) {
5950             return invoke(request, new S3MetadataResponseHandler(), bucketName, key);
5951         }
5952
5953         @Override
5954         public PutObjectResult createResult(ObjectMetadata metadata, String contentMd5) {
5955             PutObjectResult result = createPutObjectResult(metadata);
5956             result.setContentMd5(contentMd5);
5957             return result;
5958         }
5959
5960         @Override
5961         public String md5ValidationErrorSuffix() {
5962             return ", bucketName: " + bucketName + ", key: " + key;
5963         }
5964     }
5965
5966     /**
5967      * Upload strategy to use in {@link #upload(PresignedUrlUploadRequest)} API
5968      */
5969     private class PresignedUrlUploadStrategy implements UploadObjectStrategy<PresignedUrlUploadRequest, PresignedUrlUploadResult> {
5970         private final URL url;
5971
5972         private PresignedUrlUploadStrategy(URL url) {
5973             this.url = url;
5974         }
5975
5976         @Override
5977         public ObjectMetadata invokeServiceCall(Request<PresignedUrlUploadRequest> request) {
5978             return client.execute(request,
5979                                   new S3MetadataResponseHandler(),
5980                                   errorResponseHandler,
5981                                   createExecutionContext(AmazonWebServiceRequest.NOOP, new NoOpSignerProvider()),
5982                                   requestConfigWithSkipAppendUriPath(request))
5983                          .getAwsResponse();
5984         }
5985
5986         @Override
5987         public PresignedUrlUploadResult createResult(ObjectMetadata metadata, String contentMd5) {
5988             return createPresignedUrlUploadResult(metadata, contentMd5);
5989         }
5990
5991         @Override
5992         public String md5ValidationErrorSuffix() {
5993             return ", object presigned url: " + url;
5994         }
5995     }
5996
5997     private PresignedUrlUploadResult createPresignedUrlUploadResult(ObjectMetadata metadata, String contentMd5) {
5998         final PresignedUrlUploadResult result = new PresignedUrlUploadResult();
5999         result.setMetadata(metadata);
6000         result.setContentMd5(contentMd5);
6001         return result;
6002     }
6003
6004 }
6005