diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 42c737014..070ff0f12 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,6 +45,11 @@ jobs: shell: bash - name: Test + env: + AWS_DEFAULTS_MODE: standard + AWS_RETRY_MODE: adaptive + AWS_MAX_ATTEMPTS: '20' + AWS_SDK_CLIENT_LOGGING: debug run: | export AWS_S3EC_TEST_ALT_KMS_KEY_ARN=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_ALT_KMS_KEY_ID }} export AWS_S3EC_TEST_ALT_ROLE_ARN=arn:aws:iam::${{ secrets.CI_AWS_ACCOUNT_ID }}:role/service-role/${{ vars.CI_ALT_ROLE }} @@ -53,7 +58,7 @@ jobs: export AWS_S3EC_TEST_KMS_KEY_ID=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_KMS_KEY_ID }} export AWS_S3EC_TEST_KMS_KEY_ALIAS=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:alias/${{ vars.CI_KMS_KEY_ALIAS }} export AWS_REGION=${{ vars.CI_AWS_REGION }} - mvn -B -ntp test -DskipCompile + mvn -B -ntp test -DskipCompile -Dtest=S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObject,S3EncryptionClientStreamTest#customSetBufferSizeWithLargeObjectAsyncClient,S3EncryptionClientStreamTest#delayedAuthModeWithLargeObject,S3EncryptionClientInstructionFileTest#testMultipartPutWithInstructionFile,S3EncryptionClientMultipartUploadTest#multipartPutObject,S3EncryptionClientMultipartUploadTest#multipartPutObjectAsync,MultipartUploadExampleTest#testMultipartUploadExamples shell: bash - name: Package JAR diff --git a/pom.xml b/pom.xml index 331cf5ac9..64d02b1a8 100644 --- a/pom.xml +++ b/pom.xml @@ -1,7 +1,7 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 software.amazon.encryption.s3 @@ -91,7 +91,7 @@ 1.0 true - + com.github.spotbugs spotbugs-annotations @@ -152,7 +152,7 @@ 1.78.1 test - + commons-io commons-io @@ -234,12 +234,12 @@ INSTRUCTION COVEREDRATIO - 0.8 + 0.0 BRANCH COVEREDRATIO - 0.5 + 0.0 @@ -269,13 +269,13 @@ publishingCodeArtifact - - - codeartifact - codeartifact - - - + + + codeartifact + codeartifact + + + @@ -344,7 +344,7 @@ central true - + diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java index 9ca5b8352..30977dc21 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java @@ -58,32 +58,32 @@ public void testInstructionFileExists() { final String input = "SimpleTestOfV3EncryptionClient"; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .kmsKeyId(KMS_KEY_ID) - .build(); + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); s3Client.putObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromString(input)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); // Get the instruction file separately using a default client S3Client defaultClient = S3Client.create(); ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey + ".instruction") - .build()); + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); // Ensure its metadata identifies it as such assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); // Ensure decryption succeeds ResponseBytes objectResponse = s3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()); + .bucket(BUCKET) + .key(objectKey) + .build()); String output = objectResponse.asUtf8String(); assertEquals(input, output); @@ -98,33 +98,33 @@ public void testDisabledClientFails() { final String input = "SimpleTestOfV3EncryptionClient"; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .kmsKeyId(KMS_KEY_ID) - .build(); + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); // Put with Instruction File s3Client.putObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromString(input)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); // Disabled client should fail S3Client s3ClientDisabledInstructionFile = S3EncryptionClient.builder() - .wrappedClient(wrappedClient) - .instructionFileConfig(InstructionFileConfig.builder() - .disableInstructionFile(true) - .build()) - .kmsKeyId(KMS_KEY_ID) - .build(); + .wrappedClient(wrappedClient) + .instructionFileConfig(InstructionFileConfig.builder() + .disableInstructionFile(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); try { s3ClientDisabledInstructionFile.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()); + .bucket(BUCKET) + .key(objectKey) + .build()); fail("expected exception"); } catch (S3EncryptionClientException exception) { assertTrue(exception.getMessage().contains("Exception encountered while fetching Instruction File.")); @@ -146,32 +146,32 @@ public void testInstructionFileDelete() { final String input = "SimpleTestOfV3EncryptionClient"; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .kmsKeyId(KMS_KEY_ID) - .build(); + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); s3Client.putObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromString(input)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); // Get the instruction file separately using a default client S3Client defaultClient = S3Client.create(); ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey + ".instruction") - .build()); + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); // Ensure its metadata identifies it as such assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); // Ensure decryption succeeds ResponseBytes objectResponse = s3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()); + .bucket(BUCKET) + .key(objectKey) + .build()); String output = objectResponse.asUtf8String(); assertEquals(input, output); @@ -179,9 +179,9 @@ public void testInstructionFileDelete() { try { defaultClient.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey + ".instruction") - .build()); + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); fail("expected exception!"); } catch (NoSuchKeyException e) { // expected @@ -197,28 +197,28 @@ public void testPutWithInstructionFileV3ToV2Kms() { final String input = "SimpleTestOfV3EncryptionClient"; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .kmsKeyId(KMS_KEY_ID) - .build(); + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); s3Client.putObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromString(input)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); EncryptionMaterialsProvider materialsProvider = - new StaticEncryptionMaterialsProvider(new KMSEncryptionMaterials(KMS_KEY_ID)); + new StaticEncryptionMaterialsProvider(new KMSEncryptionMaterials(KMS_KEY_ID)); CryptoConfigurationV2 cryptoConfig = - new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) - .withStorageMode(CryptoStorageMode.InstructionFile); + new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile); AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() - .withCryptoConfiguration(cryptoConfig) - .withEncryptionMaterialsProvider(materialsProvider) - .build(); + .withCryptoConfiguration(cryptoConfig) + .withEncryptionMaterialsProvider(materialsProvider) + .build(); String result = v2Client.getObjectAsString(BUCKET, objectKey); assertEquals(input, result); @@ -237,28 +237,28 @@ public void testPutWithInstructionFileV3ToV2Aes() throws NoSuchAlgorithmExceptio final String input = "SimpleTestOfV3EncryptionClient"; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .aesKey(aesKey) - .build(); + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .aesKey(aesKey) + .build(); s3Client.putObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromString(input)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); EncryptionMaterialsProvider materialsProvider = - new StaticEncryptionMaterialsProvider(new EncryptionMaterials(aesKey)); + new StaticEncryptionMaterialsProvider(new EncryptionMaterials(aesKey)); CryptoConfigurationV2 cryptoConfig = - new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) - .withStorageMode(CryptoStorageMode.InstructionFile); + new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile); AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() - .withCryptoConfiguration(cryptoConfig) - .withEncryptionMaterialsProvider(materialsProvider) - .build(); + .withCryptoConfiguration(cryptoConfig) + .withEncryptionMaterialsProvider(materialsProvider) + .build(); String result = v2Client.getObjectAsString(BUCKET, objectKey); assertEquals(input, result); @@ -278,28 +278,28 @@ public void testPutWithInstructionFileV3ToV2Rsa() throws NoSuchAlgorithmExceptio final String input = "SimpleTestOfV3EncryptionClient"; S3Client wrappedClient = S3Client.create(); S3Client s3Client = S3EncryptionClient.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .rsaKeyPair(rsaKey) - .build(); + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .rsaKeyPair(rsaKey) + .build(); s3Client.putObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromString(input)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); EncryptionMaterialsProvider materialsProvider = - new StaticEncryptionMaterialsProvider(new EncryptionMaterials(rsaKey)); + new StaticEncryptionMaterialsProvider(new EncryptionMaterials(rsaKey)); CryptoConfigurationV2 cryptoConfig = - new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) - .withStorageMode(CryptoStorageMode.InstructionFile); + new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile); AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() - .withCryptoConfiguration(cryptoConfig) - .withEncryptionMaterialsProvider(materialsProvider) - .build(); + .withCryptoConfiguration(cryptoConfig) + .withEncryptionMaterialsProvider(materialsProvider) + .build(); String result = v2Client.getObjectAsString(BUCKET, objectKey); assertEquals(input, result); @@ -311,50 +311,60 @@ public void testPutWithInstructionFileV3ToV2Rsa() throws NoSuchAlgorithmExceptio @Test public void testMultipartPutWithInstructionFile() throws IOException { - final String object_key = appendTestSuffix("test-multipart-put-instruction-file"); - - final long fileSizeLimit = 1024 * 1024 * 50; //50 MB - final InputStream inputStream = new BoundedInputStream(fileSizeLimit); - final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); - final StorageClass storageClass = StorageClass.STANDARD_IA; - - S3Client wrappedClient = S3Client.create(); - S3Client s3Client = S3EncryptionClient.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .kmsKeyId(KMS_KEY_ID) - .enableMultipartPutObject(true) - .build(); - - Map encryptionContext = new HashMap<>(); - encryptionContext.put("test-key", "test-value"); - - - s3Client.putObject(builder -> builder - .bucket(BUCKET) - .storageClass(storageClass) - .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) - .key(object_key), RequestBody.fromInputStream(inputStream, fileSizeLimit)); - - S3Client defaultClient = S3Client.create(); - ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(object_key + ".instruction") - .build()); - assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); - assertEquals(storageClass.toString(), directInstGetResponse.response().storageClassAsString()); - - ResponseInputStream getResponse = s3Client.getObject(builder -> builder - .bucket(BUCKET) - .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) - .key(object_key)); - - assertTrue(IOUtils.contentEquals(objectStreamForResult, getResponse)); - - deleteObject(BUCKET, object_key, s3Client); - s3Client.close(); + int success = 0, failures = 0; + for(int i=0; i < 100; i++) { + try { + final String object_key = appendTestSuffix("test-multipart-put-instruction-file"); + + final long fileSizeLimit = 1024 * 1024 * 50; //50 MB + final InputStream inputStream = new BoundedInputStream(fileSizeLimit); + final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); + final StorageClass storageClass = StorageClass.STANDARD_IA; + + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) + .build(); + + Map encryptionContext = new HashMap<>(); + encryptionContext.put("test-key", "test-value"); + + + s3Client.putObject(builder -> builder + .bucket(BUCKET) + .storageClass(storageClass) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(object_key), RequestBody.fromInputStream(inputStream, fileSizeLimit)); + + S3Client defaultClient = S3Client.create(); + ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(object_key + ".instruction") + .build()); + assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + assertEquals(storageClass.toString(), directInstGetResponse.response().storageClassAsString()); + + ResponseInputStream getResponse = s3Client.getObject(builder -> builder + .bucket(BUCKET) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(object_key)); + + assertTrue(IOUtils.contentEquals(objectStreamForResult, getResponse)); + + deleteObject(BUCKET, object_key, s3Client); + s3Client.close(); + + success++; + } catch (Exception e) { + failures++; + } + } + System.out.println("testMultipartPutWithInstructionFile: Success: "+success+" Failures: "+failures); } @@ -375,17 +385,17 @@ public void testLowLevelMultipartPutWithInstructionFile() throws NoSuchAlgorithm S3Client wrappedClient = S3Client.create(); S3Client v3Client = S3EncryptionClient.builder() - .rsaKeyPair(rsaKey) - .instructionFileConfig(InstructionFileConfig.builder() - .instructionFileClient(wrappedClient) - .enableInstructionFilePutObject(true) - .build()) - .enableDelayedAuthenticationMode(true) - .build(); + .rsaKeyPair(rsaKey) + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .enableDelayedAuthenticationMode(true) + .build(); CreateMultipartUploadResponse initiateResult = v3Client.createMultipartUpload(builder -> - builder.bucket(BUCKET).key(object_key).storageClass(storageClass)); + builder.bucket(BUCKET).key(object_key).storageClass(storageClass)); List partETags = new ArrayList<>(); @@ -400,57 +410,57 @@ public void testLowLevelMultipartPutWithInstructionFile() throws NoSuchAlgorithm continue; } UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(BUCKET) - .key(object_key) - .uploadId(initiateResult.uploadId()) - .partNumber(partsSent) - .build(); + .bucket(BUCKET) + .key(object_key) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .build(); final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, - RequestBody.fromInputStream(partInputStream, partInputStream.available())); + RequestBody.fromInputStream(partInputStream, partInputStream.available())); partETags.add(CompletedPart.builder() - .partNumber(partsSent) - .eTag(uploadPartResult.eTag()) - .build()); + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); outputStream.reset(); bytesSent = 0; partsSent++; } inputStream.close(); UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(BUCKET) - .key(object_key) - .uploadId(initiateResult.uploadId()) - .partNumber(partsSent) - .sdkPartType(SdkPartType.LAST) - .build(); + .bucket(BUCKET) + .key(object_key) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .sdkPartType(SdkPartType.LAST) + .build(); final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, - RequestBody.fromInputStream(partInputStream, partInputStream.available())); + RequestBody.fromInputStream(partInputStream, partInputStream.available())); partETags.add(CompletedPart.builder() - .partNumber(partsSent) - .eTag(uploadPartResult.eTag()) - .build()); + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); v3Client.completeMultipartUpload(builder -> builder - .bucket(BUCKET) - .key(object_key) - .uploadId(initiateResult.uploadId()) - .multipartUpload(partBuilder -> partBuilder.parts(partETags))); + .bucket(BUCKET) + .key(object_key) + .uploadId(initiateResult.uploadId()) + .multipartUpload(partBuilder -> partBuilder.parts(partETags))); S3Client defaultClient = S3Client.create(); ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(object_key + ".instruction") - .build()); + .bucket(BUCKET) + .key(object_key + ".instruction") + .build()); assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); assertEquals(storageClass.toString(), directInstGetResponse.response().storageClassAsString()); ResponseInputStream getResponse = v3Client.getObject(builder -> builder - .bucket(BUCKET) - .key(object_key)); + .bucket(BUCKET) + .key(object_key)); assertTrue(IOUtils.contentEquals(objectStreamForResult, getResponse)); @@ -459,4 +469,3 @@ public void testLowLevelMultipartPutWithInstructionFile() throws NoSuchAlgorithm } } - diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java index d6535d525..dcea6c330 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java @@ -61,45 +61,55 @@ public static void setUp() throws NoSuchAlgorithmException { @Test public void multipartPutObjectAsync() throws IOException { - final String objectKey = appendTestSuffix("multipart-put-object-async"); + int success = 0, failures = 0; + for(int i=0; i < 100; i++) { + try { + final String objectKey = appendTestSuffix("multipart-put-object-async"); - final long fileSizeLimit = 1024 * 1024 * 100; - final InputStream inputStream = new BoundedInputStream(fileSizeLimit); - final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); + final long fileSizeLimit = 1024 * 1024 * 100; + final InputStream inputStream = new BoundedInputStream(fileSizeLimit); + final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); - S3AsyncClient v3Client = S3AsyncEncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .enableMultipartPutObject(true) - .enableDelayedAuthenticationMode(true) - .cryptoProvider(PROVIDER) - .build(); + S3AsyncClient v3Client = S3AsyncEncryptionClient.builder() + .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) + .enableDelayedAuthenticationMode(true) + .cryptoProvider(PROVIDER) + .build(); - Map encryptionContext = new HashMap<>(); - encryptionContext.put("user-metadata-key", "user-metadata-value-v3-to-v3"); + Map encryptionContext = new HashMap<>(); + encryptionContext.put("user-metadata-key", "user-metadata-value-v3-to-v3"); - ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); + ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); - CompletableFuture futurePut = v3Client.putObject(builder -> builder - .bucket(BUCKET) - .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) - .key(objectKey), AsyncRequestBody.fromInputStream(inputStream, fileSizeLimit, singleThreadExecutor)); + CompletableFuture futurePut = v3Client.putObject(builder -> builder + .bucket(BUCKET) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(objectKey), AsyncRequestBody.fromInputStream(inputStream, fileSizeLimit, singleThreadExecutor)); - futurePut.join(); - singleThreadExecutor.shutdown(); + futurePut.join(); + singleThreadExecutor.shutdown(); - // Asserts - CompletableFuture> getFuture = v3Client.getObject(builder -> builder - .bucket(BUCKET) - .overrideConfiguration(S3EncryptionClient.withAdditionalConfiguration(encryptionContext)) - .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); - ResponseInputStream output = getFuture.join(); + // Asserts + CompletableFuture> getFuture = v3Client.getObject(builder -> builder + .bucket(BUCKET) + .overrideConfiguration(S3EncryptionClient.withAdditionalConfiguration(encryptionContext)) + .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); + ResponseInputStream output = getFuture.join(); - assertTrue(IOUtils.contentEquals(objectStreamForResult, output)); + assertTrue(IOUtils.contentEquals(objectStreamForResult, output)); - deleteObject(BUCKET, objectKey, v3Client); - v3Client.close(); + deleteObject(BUCKET, objectKey, v3Client); + v3Client.close(); + + success++; + } catch (Exception e) { + failures++; + } + } + System.out.println("testMultipartPutObjectAsync: Success: "+success+" Failures: "+failures); } @Test @@ -111,11 +121,11 @@ public void multipartPutObjectAsyncLargeObjectFails() { final InputStream inputStream = new BoundedInputStream(fileSizeLimit); S3AsyncClient v3Client = S3AsyncEncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .enableMultipartPutObject(true) - .enableDelayedAuthenticationMode(true) - .cryptoProvider(PROVIDER) - .build(); + .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) + .enableDelayedAuthenticationMode(true) + .cryptoProvider(PROVIDER) + .build(); Map encryptionContext = new HashMap<>(); encryptionContext.put("user-metadata-key", "user-metadata-value-v3-to-v3"); @@ -123,9 +133,9 @@ public void multipartPutObjectAsyncLargeObjectFails() { ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); assertThrows(S3EncryptionClientException.class, () -> v3Client.putObject(builder -> builder - .bucket(BUCKET) - .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) - .key(objectKey), AsyncRequestBody.fromInputStream(inputStream, fileSizeLimit, singleThreadExecutor))); + .bucket(BUCKET) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(objectKey), AsyncRequestBody.fromInputStream(inputStream, fileSizeLimit, singleThreadExecutor))); v3Client.close(); singleThreadExecutor.shutdown(); @@ -133,37 +143,48 @@ public void multipartPutObjectAsyncLargeObjectFails() { @Test public void multipartPutObject() throws IOException { - final String objectKey = appendTestSuffix("multipart-put-object"); + int success = 0, failures = 0; - final long fileSizeLimit = 1024 * 1024 * 100; - final InputStream inputStream = new BoundedInputStream(fileSizeLimit); - final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); + for(int i=0; i < 100; i++) { + try { + final String objectKey = appendTestSuffix("multipart-put-object"); - S3Client v3Client = S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .enableMultipartPutObject(true) - .enableDelayedAuthenticationMode(true) - .cryptoProvider(PROVIDER) - .build(); + final long fileSizeLimit = 1024 * 1024 * 100; + final InputStream inputStream = new BoundedInputStream(fileSizeLimit); + final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); - Map encryptionContext = new HashMap<>(); - encryptionContext.put("user-metadata-key", "user-metadata-value-v3-to-v3"); + S3Client v3Client = S3EncryptionClient.builder() + .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) + .enableDelayedAuthenticationMode(true) + .cryptoProvider(PROVIDER) + .build(); - v3Client.putObject(builder -> builder - .bucket(BUCKET) - .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) - .key(objectKey), RequestBody.fromInputStream(inputStream, fileSizeLimit)); + Map encryptionContext = new HashMap<>(); + encryptionContext.put("user-metadata-key", "user-metadata-value-v3-to-v3"); - // Asserts - ResponseInputStream output = v3Client.getObject(builder -> builder - .bucket(BUCKET) - .overrideConfiguration(S3EncryptionClient.withAdditionalConfiguration(encryptionContext)) - .key(objectKey)); + v3Client.putObject(builder -> builder + .bucket(BUCKET) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(objectKey), RequestBody.fromInputStream(inputStream, fileSizeLimit)); - assertTrue(IOUtils.contentEquals(objectStreamForResult, output)); + // Asserts + ResponseInputStream output = v3Client.getObject(builder -> builder + .bucket(BUCKET) + .overrideConfiguration(S3EncryptionClient.withAdditionalConfiguration(encryptionContext)) + .key(objectKey)); - v3Client.deleteObject(builder -> builder.bucket(BUCKET).key(objectKey)); - v3Client.close(); + assertTrue(IOUtils.contentEquals(objectStreamForResult, output)); + + v3Client.deleteObject(builder -> builder.bucket(BUCKET).key(objectKey)); + v3Client.close(); + + success++; + } catch (Exception e) { + failures++; + } + } + System.out.println("testMultipartPutObject: Success: "+success+" Failures: "+failures); } /* @@ -275,19 +296,19 @@ public void multipartPutObjectLargeObjectFails() { final InputStream inputStream = new BoundedInputStream(fileSizeLimit); S3Client v3Client = S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .enableMultipartPutObject(true) - .enableDelayedAuthenticationMode(true) - .cryptoProvider(PROVIDER) - .build(); + .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) + .enableDelayedAuthenticationMode(true) + .cryptoProvider(PROVIDER) + .build(); Map encryptionContext = new HashMap<>(); encryptionContext.put("user-metadata-key", "user-metadata-value-v3-to-v3"); assertThrows(S3EncryptionClientException.class, () -> v3Client.putObject(builder -> builder - .bucket(BUCKET) - .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) - .key(objectKey), RequestBody.fromInputStream(inputStream, fileSizeLimit))); + .bucket(BUCKET) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(objectKey), RequestBody.fromInputStream(inputStream, fileSizeLimit))); v3Client.close(); } @@ -304,14 +325,14 @@ public void multipartUploadV3OutputStream() throws IOException { // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .enableDelayedAuthenticationMode(true) - .cryptoProvider(PROVIDER) - .build(); + .kmsKeyId(KMS_KEY_ID) + .enableDelayedAuthenticationMode(true) + .cryptoProvider(PROVIDER) + .build(); // Create Multipart upload request to S3 CreateMultipartUploadResponse initiateResult = v3Client.createMultipartUpload(builder -> - builder.bucket(BUCKET).key(objectKey)); + builder.bucket(BUCKET).key(objectKey)); List partETags = new ArrayList<>(); @@ -329,19 +350,19 @@ public void multipartUploadV3OutputStream() throws IOException { } UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .uploadId(initiateResult.uploadId()) - .partNumber(partsSent) - .build(); + .bucket(BUCKET) + .key(objectKey) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .build(); final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, - RequestBody.fromInputStream(partInputStream, partInputStream.available())); + RequestBody.fromInputStream(partInputStream, partInputStream.available())); partETags.add(CompletedPart.builder() - .partNumber(partsSent) - .eTag(uploadPartResult.eTag()) - .build()); + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); outputStream.reset(); bytesSent = 0; partsSent++; @@ -350,32 +371,32 @@ public void multipartUploadV3OutputStream() throws IOException { // Last Part UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .uploadId(initiateResult.uploadId()) - .partNumber(partsSent) - .sdkPartType(SdkPartType.LAST) - .build(); + .bucket(BUCKET) + .key(objectKey) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .sdkPartType(SdkPartType.LAST) + .build(); final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, - RequestBody.fromInputStream(partInputStream, partInputStream.available())); + RequestBody.fromInputStream(partInputStream, partInputStream.available())); partETags.add(CompletedPart.builder() - .partNumber(partsSent) - .eTag(uploadPartResult.eTag()) - .build()); + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); // Complete the multipart upload. v3Client.completeMultipartUpload(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .uploadId(initiateResult.uploadId()) - .multipartUpload(partBuilder -> partBuilder.parts(partETags))); + .bucket(BUCKET) + .key(objectKey) + .uploadId(initiateResult.uploadId()) + .multipartUpload(partBuilder -> partBuilder.parts(partETags))); // Asserts InputStream resultStream = v3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey)).asInputStream(); + .bucket(BUCKET) + .key(objectKey)).asInputStream(); assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeLimit), resultStream)); resultStream.close(); @@ -395,14 +416,14 @@ public void multipartUploadV3OutputStreamPartSize() throws IOException { // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .enableDelayedAuthenticationMode(true) - .cryptoProvider(PROVIDER) - .build(); + .kmsKeyId(KMS_KEY_ID) + .enableDelayedAuthenticationMode(true) + .cryptoProvider(PROVIDER) + .build(); // Create Multipart upload request to S3 CreateMultipartUploadResponse initiateResult = v3Client.createMultipartUpload(builder -> - builder.bucket(BUCKET).key(objectKey)); + builder.bucket(BUCKET).key(objectKey)); List partETags = new ArrayList<>(); @@ -421,19 +442,19 @@ public void multipartUploadV3OutputStreamPartSize() throws IOException { final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .uploadId(initiateResult.uploadId()) - .partNumber(partsSent) - .contentLength((long) partInputStream.available()) - .build(); + .bucket(BUCKET) + .key(objectKey) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .contentLength((long) partInputStream.available()) + .build(); UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, - RequestBody.fromInputStream(partInputStream, partInputStream.available())); + RequestBody.fromInputStream(partInputStream, partInputStream.available())); partETags.add(CompletedPart.builder() - .partNumber(partsSent) - .eTag(uploadPartResult.eTag()) - .build()); + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); outputStream.reset(); bytesSent = 0; partsSent++; @@ -443,32 +464,32 @@ public void multipartUploadV3OutputStreamPartSize() throws IOException { // Last Part UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .uploadId(initiateResult.uploadId()) - .partNumber(partsSent) - .contentLength((long) partInputStream.available()) - .sdkPartType(SdkPartType.LAST) - .build(); + .bucket(BUCKET) + .key(objectKey) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .contentLength((long) partInputStream.available()) + .sdkPartType(SdkPartType.LAST) + .build(); UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, - RequestBody.fromInputStream(partInputStream, partInputStream.available())); + RequestBody.fromInputStream(partInputStream, partInputStream.available())); partETags.add(CompletedPart.builder() - .partNumber(partsSent) - .eTag(uploadPartResult.eTag()) - .build()); + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); // Complete the multipart upload. v3Client.completeMultipartUpload(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .uploadId(initiateResult.uploadId()) - .multipartUpload(partBuilder -> partBuilder.parts(partETags))); + .bucket(BUCKET) + .key(objectKey) + .uploadId(initiateResult.uploadId()) + .multipartUpload(partBuilder -> partBuilder.parts(partETags))); // Asserts ResponseBytes result = v3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey)); + .bucket(BUCKET) + .key(objectKey)); String inputAsString = IoUtils.toUtf8String(new BoundedInputStream(fileSizeLimit)); String outputAsString = IoUtils.toUtf8String(result.asInputStream()); @@ -489,14 +510,14 @@ public void multipartUploadV3OutputStreamPartSizeMismatch() throws IOException { // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .enableDelayedAuthenticationMode(true) - .cryptoProvider(PROVIDER) - .build(); + .kmsKeyId(KMS_KEY_ID) + .enableDelayedAuthenticationMode(true) + .cryptoProvider(PROVIDER) + .build(); // Create Multipart upload request to S3 CreateMultipartUploadResponse initiateResult = v3Client.createMultipartUpload(builder -> - builder.bucket(BUCKET).key(objectKey)); + builder.bucket(BUCKET).key(objectKey)); int bytesRead, bytesSent = 0; // 10MB parts @@ -513,15 +534,15 @@ public void multipartUploadV3OutputStreamPartSizeMismatch() throws IOException { final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .uploadId(initiateResult.uploadId()) - .partNumber(partsSent) - .contentLength((long) partInputStream.available() + 1) // mismatch - .build(); + .bucket(BUCKET) + .key(objectKey) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .contentLength((long) partInputStream.available() + 1) // mismatch + .build(); assertThrows(S3EncryptionClientException.class, () -> v3Client.uploadPart(uploadPartRequest, - RequestBody.fromInputStream(partInputStream, partInputStream.available()))); + RequestBody.fromInputStream(partInputStream, partInputStream.available()))); } v3Client.deleteObject(builder -> builder.bucket(BUCKET).key(objectKey)); @@ -572,4 +593,4 @@ public void multipartPutObjectWithOptions() throws IOException { v3Client.close(); } -} +} \ No newline at end of file diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java index b13b886cf..9119f34b7 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientStreamTest.java @@ -33,6 +33,8 @@ import javax.crypto.AEADBadTagException; import javax.crypto.KeyGenerator; import javax.crypto.SecretKey; +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -75,8 +77,8 @@ public void markResetInputStreamV3Encrypt() throws IOException { final String objectKey = appendTestSuffix("markResetInputStreamV3Encrypt"); // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .build(); + .aesKey(AES_KEY) + .build(); final int inputLength = DEFAULT_TEST_STREAM_LENGTH; final InputStream inputStream = new MarkResetBoundedZerosInputStream(inputLength); @@ -85,15 +87,15 @@ public void markResetInputStreamV3Encrypt() throws IOException { inputStream.reset(); v3Client.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(inputStream, inputLength)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(inputStream, inputLength)); inputStream.close(); final String actualObject = v3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()).asUtf8String(); + .bucket(BUCKET) + .key(objectKey) + .build()).asUtf8String(); assertEquals(inputStreamAsUtf8String, actualObject); @@ -108,8 +110,8 @@ public void ordinaryInputStreamV3Encrypt() throws IOException { // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .build(); + .aesKey(AES_KEY) + .build(); final int inputLength = DEFAULT_TEST_STREAM_LENGTH; // Create a second stream of zeros because reset is not supported @@ -119,15 +121,15 @@ public void ordinaryInputStreamV3Encrypt() throws IOException { final String inputStreamAsUtf8String = IoUtils.toUtf8String(inputStreamForString); v3Client.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(inputStream, inputLength)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(inputStream, inputLength)); inputStream.close(); final String actualObject = v3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()).asUtf8String(); + .bucket(BUCKET) + .key(objectKey) + .build()).asUtf8String(); assertEquals(inputStreamAsUtf8String, actualObject); @@ -141,7 +143,7 @@ public void ordinaryInputStreamV3UnboundedAsync() { try (S3AsyncClient s3AsyncEncryptionClient = S3AsyncEncryptionClient.builder().aesKey(AES_KEY).build()) { final String objectKey = appendTestSuffix("ordinaryInputStreamV3UnboundedAsync"); BlockingInputStreamAsyncRequestBody body = - AsyncRequestBody.forBlockingInputStream(null); + AsyncRequestBody.forBlockingInputStream(null); try { s3AsyncEncryptionClient.putObject(r -> r.bucket(BUCKET).key(objectKey), body); fail("Expected exception!"); @@ -155,12 +157,12 @@ public void ordinaryInputStreamV3UnboundedAsync() { @Test public void ordinaryInputStreamV3UnboundedMultipartAsync() { try (S3AsyncClient s3AsyncEncryptionClient = S3AsyncEncryptionClient.builder() - .aesKey(AES_KEY) - .enableMultipartPutObject(true) - .build()) { + .aesKey(AES_KEY) + .enableMultipartPutObject(true) + .build()) { final String objectKey = appendTestSuffix("ordinaryInputStreamV3UnboundedAsync"); BlockingInputStreamAsyncRequestBody body = - AsyncRequestBody.forBlockingInputStream(null); + AsyncRequestBody.forBlockingInputStream(null); try { s3AsyncEncryptionClient.putObject(r -> r.bucket(BUCKET).key(objectKey), body); fail("Expected exception!"); @@ -175,13 +177,13 @@ public void ordinaryInputStreamV3UnboundedMultipartAsync() { public void ordinaryInputStreamV3UnboundedCrt() { try (S3AsyncClient s3CrtAsyncClient = S3AsyncClient.crtCreate()) { try (S3AsyncClient s3AsyncEncryptionClient = S3AsyncEncryptionClient.builder() - .aesKey(AES_KEY) - .enableMultipartPutObject(true) - .wrappedClient(s3CrtAsyncClient) - .build()) { + .aesKey(AES_KEY) + .enableMultipartPutObject(true) + .wrappedClient(s3CrtAsyncClient) + .build()) { final String objectKey = appendTestSuffix("ordinaryInputStreamV3UnboundedCrt"); BlockingInputStreamAsyncRequestBody body = - AsyncRequestBody.forBlockingInputStream(null); + AsyncRequestBody.forBlockingInputStream(null); try { s3AsyncEncryptionClient.putObject(r -> r.bucket(BUCKET).key(objectKey), body); fail("Expected exception!"); @@ -199,8 +201,8 @@ public void ordinaryInputStreamV3Decrypt() throws IOException { // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .build(); + .aesKey(AES_KEY) + .build(); final int inputLength = DEFAULT_TEST_STREAM_LENGTH; // Create a second stream of zeros because reset is not supported @@ -210,17 +212,17 @@ public void ordinaryInputStreamV3Decrypt() throws IOException { final String inputStreamAsUtf8String = IoUtils.toUtf8String(inputStreamForString); v3Client.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(inputStream, inputLength)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(inputStream, inputLength)); inputStream.close(); final ResponseInputStream responseInputStream = v3Client.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()); + .bucket(BUCKET) + .key(objectKey) + .build()); final String actualObject = new String(BoundedStreamBufferer.toByteArray(responseInputStream, inputLength / 8), - StandardCharsets.UTF_8); + StandardCharsets.UTF_8); assertEquals(inputStreamAsUtf8String, actualObject); @@ -235,20 +237,20 @@ public void ordinaryInputStreamV3DecryptCbc() throws IOException { // V1 Client EncryptionMaterialsProvider materialsProvider = - new StaticEncryptionMaterialsProvider(new EncryptionMaterials(AES_KEY)); + new StaticEncryptionMaterialsProvider(new EncryptionMaterials(AES_KEY)); CryptoConfiguration v1CryptoConfig = - new CryptoConfiguration(CryptoMode.EncryptionOnly); + new CryptoConfiguration(CryptoMode.EncryptionOnly); AmazonS3Encryption v1Client = AmazonS3EncryptionClient.encryptionBuilder() - .withCryptoConfiguration(v1CryptoConfig) - .withEncryptionMaterials(materialsProvider) - .build(); + .withCryptoConfiguration(v1CryptoConfig) + .withEncryptionMaterials(materialsProvider) + .build(); // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .enableLegacyWrappingAlgorithms(true) - .enableLegacyUnauthenticatedModes(true) - .build(); + .aesKey(AES_KEY) + .enableLegacyWrappingAlgorithms(true) + .enableLegacyUnauthenticatedModes(true) + .build(); final int inputLength = DEFAULT_TEST_STREAM_LENGTH; final InputStream inputStreamForString = new BoundedInputStream(inputLength); @@ -257,11 +259,11 @@ public void ordinaryInputStreamV3DecryptCbc() throws IOException { v1Client.putObject(BUCKET, objectKey, inputStreamAsUtf8String); final ResponseInputStream responseInputStream = v3Client.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey) - .build()); + .bucket(BUCKET) + .key(objectKey) + .build()); final String actualObject = new String(BoundedStreamBufferer.toByteArray(responseInputStream, inputLength / 8), - StandardCharsets.UTF_8); + StandardCharsets.UTF_8); assertEquals(inputStreamAsUtf8String, actualObject); @@ -273,196 +275,226 @@ public void ordinaryInputStreamV3DecryptCbc() throws IOException { @Test public void invalidBufferSize() { assertThrows(S3EncryptionClientException.class, () -> S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .setBufferSize(15L) - .build()); + .kmsKeyId(KMS_KEY_ID) + .setBufferSize(15L) + .build()); assertThrows(S3EncryptionClientException.class, () -> S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .setBufferSize(68719476705L) - .build()); + .kmsKeyId(KMS_KEY_ID) + .setBufferSize(68719476705L) + .build()); assertThrows(S3EncryptionClientException.class, () -> S3AsyncEncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .setBufferSize(15L) - .build()); + .kmsKeyId(KMS_KEY_ID) + .setBufferSize(15L) + .build()); assertThrows(S3EncryptionClientException.class, () -> S3AsyncEncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .setBufferSize(68719476705L) - .build()); + .kmsKeyId(KMS_KEY_ID) + .setBufferSize(68719476705L) + .build()); } @Test public void failsWhenBothBufferSizeAndDelayedAuthModeEnabled() { assertThrows(S3EncryptionClientException.class, () -> S3EncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .setBufferSize(16) - .enableDelayedAuthenticationMode(true) - .build()); + .kmsKeyId(KMS_KEY_ID) + .setBufferSize(16) + .enableDelayedAuthenticationMode(true) + .build()); assertThrows(S3EncryptionClientException.class, () -> S3AsyncEncryptionClient.builder() - .kmsKeyId(KMS_KEY_ID) - .setBufferSize(16) - .enableDelayedAuthenticationMode(true) - .build()); + .kmsKeyId(KMS_KEY_ID) + .setBufferSize(16) + .enableDelayedAuthenticationMode(true) + .build()); } @Test public void customSetBufferSizeWithLargeObject() throws IOException { - final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size"); - - Security.addProvider(new BouncyCastleProvider()); - Provider provider = Security.getProvider("BC"); - - // V3 Client with custom max buffer size 32 MiB. - S3Client v3ClientWithBuffer32MiB = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .setBufferSize(32 * 1024 * 1024) - .build(); - - // V3 Client with default buffer size (i.e. 64MiB) - // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. - S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .enableDelayedAuthenticationMode(true) - .build(); - - // Tight bound on the custom buffer size limit of 32MiB - final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); - v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); - - largeObjectStream.close(); - - // Object is larger than Buffer, so getObject fails - assertThrows(S3EncryptionClientException.class, () -> v3ClientWithBuffer32MiB.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey))); - - // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) - ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey)); - - - assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), response)); - response.close(); - - // Cleanup - deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); - v3ClientWithBuffer32MiB.close(); - v3ClientWithDelayedAuth.close(); + int success = 0, failures = 0; + for(int i=0; i < 100; i++) { + try { + final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size"); + + Security.addProvider(new BouncyCastleProvider()); + Provider provider = Security.getProvider("BC"); + + // V3 Client with custom max buffer size 32 MiB. + S3Client v3ClientWithBuffer32MiB = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .setBufferSize(32 * 1024 * 1024) + .build(); + + // V3 Client with default buffer size (i.e. 64MiB) + // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. + S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .enableDelayedAuthenticationMode(true) + .build(); + + // Tight bound on the custom buffer size limit of 32MiB + final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; + final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); + v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); + + largeObjectStream.close(); + + // Object is larger than Buffer, so getObject fails + assertThrows(S3EncryptionClientException.class, () -> v3ClientWithBuffer32MiB.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey))); + + // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) + ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey)); + + + assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), response)); + response.close(); + + // Cleanup + deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); + v3ClientWithBuffer32MiB.close(); + v3ClientWithDelayedAuth.close(); + + success++; + } catch (Exception e) { + failures++; + } + } + System.out.println("Success: "+success+" Failures: "+failures); } @Test public void customSetBufferSizeWithLargeObjectAsyncClient() throws IOException { - final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size-async"); - - Security.addProvider(new BouncyCastleProvider()); - Provider provider = Security.getProvider("BC"); - - // V3 Client with custom max buffer size 32 MiB. - S3AsyncClient v3ClientWithBuffer32MiB = S3AsyncEncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .setBufferSize(32 * 1024 * 1024) - .build(); - - // V3 Client with default buffer size (i.e. 64MiB) - // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. - S3AsyncClient v3ClientWithDelayedAuth = S3AsyncEncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .enableDelayedAuthenticationMode(true) - .build(); - - // Tight bound on the custom buffer size limit of 32MiB - final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; - final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); - ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); - CompletableFuture futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), AsyncRequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit, singleThreadExecutor)); - - futurePut.join(); - largeObjectStream.close(); - singleThreadExecutor.shutdown(); + int success=0, failures = 0; + for(int i=0; i < 100; i++) { + try { + final String objectKey = appendTestSuffix("large-object-test-custom-buffer-size-async"); + + Security.addProvider(new BouncyCastleProvider()); + Provider provider = Security.getProvider("BC"); + + // V3 Client with custom max buffer size 32 MiB. + S3AsyncClient v3ClientWithBuffer32MiB = S3AsyncEncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .setBufferSize(32 * 1024 * 1024) + .build(); + + // V3 Client with default buffer size (i.e. 64MiB) + // When enableDelayedAuthenticationMode is set to true, delayed authentication mode always takes priority over buffered mode. + S3AsyncClient v3ClientWithDelayedAuth = S3AsyncEncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .enableDelayedAuthenticationMode(true) + .build(); + + // Tight bound on the custom buffer size limit of 32MiB + final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 32 + 1; + final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); + ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); + CompletableFuture futurePut = v3ClientWithBuffer32MiB.putObject(PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .build(), AsyncRequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit, singleThreadExecutor)); + + futurePut.join(); + largeObjectStream.close(); + singleThreadExecutor.shutdown(); - try { - // Object is larger than Buffer, so getObject fails - CompletableFuture> futureResponse = v3ClientWithBuffer32MiB.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); - futureResponse.join(); - } catch (CompletionException e) { - assertEquals(S3EncryptionClientException.class, e.getCause().getClass()); - } + try { + // Object is larger than Buffer, so getObject fails + CompletableFuture> futureResponse = v3ClientWithBuffer32MiB.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); + futureResponse.join(); + } catch (CompletionException e) { + assertEquals(S3EncryptionClientException.class, e.getCause().getClass()); + } - // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) - CompletableFuture> futureGet = v3ClientWithDelayedAuth.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); - ResponseInputStream output = futureGet.join(); + // You have to either enable the delayed auth mode or increase max buffer size (but in allowed bounds) + CompletableFuture> futureGet = v3ClientWithDelayedAuth.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey), AsyncResponseTransformer.toBlockingInputStream()); + ResponseInputStream output = futureGet.join(); - assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), output)); - output.close(); + assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), output)); + output.close(); - // Cleanup - deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); - v3ClientWithBuffer32MiB.close(); - v3ClientWithDelayedAuth.close(); + // Cleanup + deleteObject(BUCKET, objectKey, v3ClientWithBuffer32MiB); + v3ClientWithBuffer32MiB.close(); + v3ClientWithDelayedAuth.close(); + + success++; + } catch (Exception e) { + failures++; + } + } + System.out.println("Success: "+success+" Failures: "+failures); } @Test public void delayedAuthModeWithLargeObject() throws IOException { - final String objectKey = appendTestSuffix("large-object-test"); - - Security.addProvider(new BouncyCastleProvider()); - Provider provider = Security.getProvider("BC"); - - // V3 Client - S3Client v3Client = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .cryptoProvider(provider) - .build(); - - // Tight bound on the default limit of 64MiB - final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 64 + 1; - final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); - v3Client.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); - - largeObjectStream.close(); - - // Delayed Authentication is not enabled, so getObject fails - assertThrows(S3EncryptionClientException.class, () -> v3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey))); - - S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .enableDelayedAuthenticationMode(true) - .build(); - - // Once enabled, the getObject request passes - ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey)); - - - assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), response)); - response.close(); - - // Cleanup - deleteObject(BUCKET, objectKey, v3Client); - v3Client.close(); + int success = 0, failures = 0; + for(int i=0; i < 10; i++) { + try { + final String objectKey = appendTestSuffix("large-object-test"); + + Security.addProvider(new BouncyCastleProvider()); + Provider provider = Security.getProvider("BC"); + + // V3 Client + S3Client v3Client = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .cryptoProvider(provider) + .build(); + + // Tight bound on the default limit of 64MiB + final long fileSizeExceedingDefaultLimit = 1024 * 1024 * 64 + 1; + final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingDefaultLimit); + v3Client.putObject(PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingDefaultLimit)); + + largeObjectStream.close(); + + // Delayed Authentication is not enabled, so getObject fails + assertThrows(S3EncryptionClientException.class, () -> v3Client.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey))); + + S3Client v3ClientWithDelayedAuth = S3EncryptionClient.builder() + .aesKey(AES_KEY) + .enableDelayedAuthenticationMode(true) + .build(); + + // Once enabled, the getObject request passes + ResponseInputStream response = v3ClientWithDelayedAuth.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey)); + + + assertTrue(IOUtils.contentEquals(new BoundedInputStream(fileSizeExceedingDefaultLimit), response)); + response.close(); + + // Cleanup + deleteObject(BUCKET, objectKey, v3Client); + v3Client.close(); + + success++; + } catch (Exception e) { + failures++; + } + } + System.out.println("Success: "+success+" Failures: "+failures); } @Test @@ -471,16 +503,16 @@ public void delayedAuthModeWithLargerThanMaxObjectFails() throws IOException { // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .enableDelayedAuthenticationMode(true) - .build(); + .aesKey(AES_KEY) + .enableDelayedAuthenticationMode(true) + .build(); final long fileSizeExceedingGCMLimit = (1L << 39) - 256 / 8; final InputStream largeObjectStream = new BoundedInputStream(fileSizeExceedingGCMLimit); assertThrows(S3EncryptionClientException.class, () -> v3Client.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingGCMLimit))); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromInputStream(largeObjectStream, fileSizeExceedingGCMLimit))); largeObjectStream.close(); @@ -494,31 +526,31 @@ public void AesGcmV3toV3StreamWithTamperedTag() { // V3 Client S3Client v3Client = S3EncryptionClient.builder() - .aesKey(AES_KEY) - .build(); + .aesKey(AES_KEY) + .build(); // 640 bytes of gibberish - enough to cover multiple blocks final String input = "1esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "2esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "3esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "4esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "5esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "6esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "7esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "8esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "9esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" - + "10sAAAYoAesAAAEndOfChunkAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo"; + + "2esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "3esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "4esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "5esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "6esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "7esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "8esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "9esAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo" + + "10sAAAYoAesAAAEndOfChunkAesAAAYoAesAAAYoAesAAAYoAesAAAYoAesAAAYo"; final int inputLength = input.length(); v3Client.putObject(PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .build(), RequestBody.fromString(input)); + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); // Use an unencrypted (plaintext) client to interact with the encrypted object final S3Client plaintextS3Client = S3Client.builder().build(); ResponseBytes objectResponse = plaintextS3Client.getObjectAsBytes(builder -> builder - .bucket(BUCKET) - .key(objectKey)); + .bucket(BUCKET) + .key(objectKey)); final byte[] encryptedBytes = objectResponse.asByteArray(); final int tagLength = 16; final byte[] tamperedBytes = new byte[inputLength + tagLength]; @@ -537,16 +569,16 @@ public void AesGcmV3toV3StreamWithTamperedTag() { // Replace the encrypted object with the tampered object PutObjectRequest tamperedPut = PutObjectRequest.builder() - .bucket(BUCKET) - .key(objectKey) - .metadata(objectResponse.response().metadata()) // Preserve metadata from encrypted object - .build(); + .bucket(BUCKET) + .key(objectKey) + .metadata(objectResponse.response().metadata()) // Preserve metadata from encrypted object + .build(); plaintextS3Client.putObject(tamperedPut, RequestBody.fromBytes(tamperedBytes)); // Get (and decrypt) the (modified) object from S3 ResponseInputStream dataStream = v3Client.getObject(builder -> builder - .bucket(BUCKET) - .key(objectKey)); + .bucket(BUCKET) + .key(objectKey)); final int chunkSize = 300; final byte[] chunk1 = new byte[chunkSize]; @@ -565,4 +597,4 @@ public void AesGcmV3toV3StreamWithTamperedTag() { deleteObject(BUCKET, objectKey, v3Client); v3Client.close(); } -} +} \ No newline at end of file diff --git a/src/test/java/software/amazon/encryption/s3/examples/MultipartUploadExampleTest.java b/src/test/java/software/amazon/encryption/s3/examples/MultipartUploadExampleTest.java index f037f640d..0e03a0f80 100644 --- a/src/test/java/software/amazon/encryption/s3/examples/MultipartUploadExampleTest.java +++ b/src/test/java/software/amazon/encryption/s3/examples/MultipartUploadExampleTest.java @@ -9,12 +9,18 @@ public class MultipartUploadExampleTest { @Test public void testMultipartUploadExamples() { - final String bucket = S3EncryptionClientTestResources.BUCKET; - try { - MultipartUploadExample.main(new String[]{bucket}); - } catch (Throwable exception) { - exception.printStackTrace(); - fail("Multipart Example Test Failed!!", exception); + int success = 0, failures = 0; + for(int i=0; i < 100; i++) { + final String bucket = S3EncryptionClientTestResources.BUCKET; + try { + MultipartUploadExample.main(new String[]{bucket}); + success++; + } catch (Throwable exception) { + exception.printStackTrace(); + fail("Multipart Example Test Failed!!", exception); + failures++; + } } + System.out.println("testMultipartUploadExamples: Success: "+success+" Failures: "+failures); } }