From e866c54a45e1f716a7cbeef8c1067f0e73e7cd0d Mon Sep 17 00:00:00 2001 From: "IIPL\\14261" Date: Wed, 4 Sep 2024 21:33:05 +0530 Subject: [PATCH 1/2] Add functionality to upload files to S3 with optional public access Implement file download functionality from S3 as a stream --- pom.xml | 5 ++ src/main/java/com/app/config/AwsS3Config.java | 45 ++++++++++ .../java/com/app/controller/S3Controller.java | 41 ++++++++++ src/main/java/com/app/service/S3Service.java | 12 +++ .../java/com/app/service/S3ServiceImpl.java | 82 +++++++++++++++++++ src/main/resources/application-local.yml | 7 ++ 6 files changed, 192 insertions(+) create mode 100644 src/main/java/com/app/config/AwsS3Config.java create mode 100644 src/main/java/com/app/controller/S3Controller.java create mode 100644 src/main/java/com/app/service/S3Service.java create mode 100644 src/main/java/com/app/service/S3ServiceImpl.java diff --git a/pom.xml b/pom.xml index 2d8e109..0766dd1 100644 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,11 @@ org.springframework.boot spring-boot-starter-log4j2 + + software.amazon.awssdk + s3 + 2.25.70 + org.springframework.boot spring-boot-starter-test diff --git a/src/main/java/com/app/config/AwsS3Config.java b/src/main/java/com/app/config/AwsS3Config.java new file mode 100644 index 0000000..952544c --- /dev/null +++ b/src/main/java/com/app/config/AwsS3Config.java @@ -0,0 +1,45 @@ +package com.app.config; + + +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +@Configuration +@ConditionalOnProperty(name = "aws.s3.enabled", havingValue = "true") +@Log4j2 +public class AwsS3Config { + + @Value("${aws.s3.region:us-east-1}") + private String awsRegion; + @Value("${aws.s3.accessKeyId:us-east-1}") + private String accessKeyId; + @Value("${aws.s3.secretAccessKey}") + private String secretAccessKey; + + @Bean + public S3Client s3Client() { + try { + log.info("Trying to S3Client create."); + return S3Client.builder() + .region(Region.of(awsRegion)) + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey))) + .overrideConfiguration(ClientOverrideConfiguration.builder() + .retryPolicy(r -> r.numRetries(3)) + .build()) + .build(); + } catch (Exception e) { + log.error("Failed to create S3Client.", e); + throw e; + } finally { + log.info("S3Client created successfully."); + } + } +} diff --git a/src/main/java/com/app/controller/S3Controller.java b/src/main/java/com/app/controller/S3Controller.java new file mode 100644 index 0000000..f6c5ad6 --- /dev/null +++ b/src/main/java/com/app/controller/S3Controller.java @@ -0,0 +1,41 @@ +package com.app.controller; + +import com.app.service.S3Service; +import lombok.RequiredArgsConstructor; +import org.springframework.core.io.InputStreamResource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.io.InputStream; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/files") +public class S3Controller { + + private final S3Service s3Service; + + @PostMapping("/upload") + public ResponseEntity uploadFile(@RequestPart("file") MultipartFile file, + @RequestParam(value = "isReadPublicly", defaultValue = "false") boolean isReadPublicly) { + boolean isUploaded = s3Service.uploadFile(file, isReadPublicly); + if (isUploaded) { + return ResponseEntity.ok("File uploaded successfully: " + file.getOriginalFilename()); + } else { + return ResponseEntity.status(500).body("Failed to upload file: " + file.getOriginalFilename()); + } + } + + @GetMapping("/download/{key}") + public ResponseEntity downloadFile(@PathVariable String key) { + InputStream fileStream = s3Service.downloadFileAsStream(key); + InputStreamResource resource = new InputStreamResource(fileStream); + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + key) + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .body(resource); + } +} diff --git a/src/main/java/com/app/service/S3Service.java b/src/main/java/com/app/service/S3Service.java new file mode 100644 index 0000000..5f9d2f2 --- /dev/null +++ b/src/main/java/com/app/service/S3Service.java @@ -0,0 +1,12 @@ +package com.app.service; + +import org.springframework.web.multipart.MultipartFile; + +import java.io.InputStream; + +public interface S3Service { + + boolean uploadFile(MultipartFile file, boolean isReadPublicly); + + InputStream downloadFileAsStream(String key); +} diff --git a/src/main/java/com/app/service/S3ServiceImpl.java b/src/main/java/com/app/service/S3ServiceImpl.java new file mode 100644 index 0000000..7651f3a --- /dev/null +++ b/src/main/java/com/app/service/S3ServiceImpl.java @@ -0,0 +1,82 @@ +package com.app.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; + +@Service +@Log4j2 +@RequiredArgsConstructor +public class S3ServiceImpl implements S3Service { + + private final S3Client s3Client; + + @Value("${aws.s3.bucket-name}") + private String bucketName; + + + @Override + public boolean uploadFile(MultipartFile file, boolean isReadPublicly) { + log.info("Started uploading file '{}' to S3 Bucket '{}'", file.getOriginalFilename(), bucketName); + PutObjectRequest putObjectRequest; + if (isReadPublicly) { + putObjectRequest = PutObjectRequest.builder() + .bucket(bucketName) + .key(file.getOriginalFilename()).acl("public-read") + .build(); + } else { + putObjectRequest = PutObjectRequest.builder() + .bucket(bucketName) + .key(file.getOriginalFilename()) + .build(); + } + try { + s3Client.putObject(putObjectRequest, RequestBody.fromBytes(file.getBytes())); + log.info("Successfully uploaded file to S3. Bucket: {}, Key: {}", bucketName, file.getOriginalFilename()); + return true; + } catch (Exception e) { + log.error("Failed to upload file to S3. Bucket: {}, Key: {}", bucketName, file.getOriginalFilename(), e); + return false; + } + } + + @Override + public InputStream downloadFileAsStream(String key) { + try { + GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + + ResponseBytes getObjectResponse = s3Client.getObjectAsBytes(getObjectRequest); + if (getObjectResponse == null) { + log.warn("Failed to get file from S3 bucket: Response is null"); + return new ByteArrayInputStream(new byte[0]); + } + + log.info("Successfully getting file in bytes from S3 bucket."); + byte[] fileBytes = getObjectResponse.asByteArray(); + return new ByteArrayInputStream(fileBytes); + + } catch (S3Exception e) { + log.error("Failed to fetch object from S3 Bucket: {}, Key: {}", bucketName, key, e); + throw e; + } catch (SdkException e) { + log.error("Error while downloading file from S3 Bucket: {}, Key: {}", bucketName, key, e); + throw e; + } + } +} diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index d2f8b2d..11b1e06 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -2,3 +2,10 @@ app: logs: path: C:/${spring.application.name}/logs +aws: + s3: + enabled: true + region: us-east-1 + accessKeyId: <> + secretAccessKey: <> + bucket-name: <> \ No newline at end of file From 39da02e57b33046bc0df37d3dbc0a6944a2a2b08 Mon Sep 17 00:00:00 2001 From: "IIPL\\14261" Date: Wed, 4 Sep 2024 21:52:53 +0530 Subject: [PATCH 2/2] Add asynchronous file upload and download functionality using S3AsyncClient --- src/main/java/com/app/config/AwsS3Config.java | 11 +- .../java/com/app/controller/S3Controller.java | 35 ++++-- src/main/java/com/app/service/S3Service.java | 6 +- .../java/com/app/service/S3ServiceImpl.java | 105 ++++++++++-------- 4 files changed, 97 insertions(+), 60 deletions(-) diff --git a/src/main/java/com/app/config/AwsS3Config.java b/src/main/java/com/app/config/AwsS3Config.java index 952544c..37657fe 100644 --- a/src/main/java/com/app/config/AwsS3Config.java +++ b/src/main/java/com/app/config/AwsS3Config.java @@ -10,7 +10,7 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3AsyncClient; @Configuration @ConditionalOnProperty(name = "aws.s3.enabled", havingValue = "true") @@ -25,10 +25,11 @@ public class AwsS3Config { private String secretAccessKey; @Bean - public S3Client s3Client() { + public S3AsyncClient s3Client() { try { log.info("Trying to S3Client create."); - return S3Client.builder() + + return S3AsyncClient.builder() .region(Region.of(awsRegion)) .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey))) .overrideConfiguration(ClientOverrideConfiguration.builder() @@ -36,10 +37,10 @@ public S3Client s3Client() { .build()) .build(); } catch (Exception e) { - log.error("Failed to create S3Client.", e); + log.error("Failed to create S3AsyncClient.", e); throw e; } finally { - log.info("S3Client created successfully."); + log.info("S3AsyncClient created successfully."); } } } diff --git a/src/main/java/com/app/controller/S3Controller.java b/src/main/java/com/app/controller/S3Controller.java index f6c5ad6..f395121 100644 --- a/src/main/java/com/app/controller/S3Controller.java +++ b/src/main/java/com/app/controller/S3Controller.java @@ -1,6 +1,7 @@ package com.app.controller; import com.app.service.S3Service; +import jakarta.servlet.http.HttpServletResponse; import lombok.RequiredArgsConstructor; import org.springframework.core.io.InputStreamResource; import org.springframework.http.HttpHeaders; @@ -8,8 +9,11 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; +import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.util.concurrent.CompletableFuture; @RestController @RequiredArgsConstructor @@ -30,12 +34,29 @@ public ResponseEntity uploadFile(@RequestPart("file") MultipartFile file } @GetMapping("/download/{key}") - public ResponseEntity downloadFile(@PathVariable String key) { - InputStream fileStream = s3Service.downloadFileAsStream(key); - InputStreamResource resource = new InputStreamResource(fileStream); - return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + key) - .contentType(MediaType.APPLICATION_OCTET_STREAM) - .body(resource); + public StreamingResponseBody downloadFile(@PathVariable String key, HttpServletResponse httpResponse) { + + httpResponse.setContentType("application/octet-stream"); + httpResponse.setHeader("Content-Disposition", String.format("inline; filename=\"%s\"", key)); + + CompletableFuture byteArrayInputStreamCompletableFuture = s3Service.downloadFileAsStream(key); + + return outputStream -> { + ByteArrayInputStream byteArrayInputStream = byteArrayInputStreamCompletableFuture.join(); + if (byteArrayInputStream != null) { + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = byteArrayInputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + outputStream.flush(); + } else { + // Handle the case where the stream is null + httpResponse.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + String errorMessage = "Failed to download the key. Please try again later."; + outputStream.write(errorMessage.getBytes()); + outputStream.flush(); + } + }; } } diff --git a/src/main/java/com/app/service/S3Service.java b/src/main/java/com/app/service/S3Service.java index 5f9d2f2..775dd0f 100644 --- a/src/main/java/com/app/service/S3Service.java +++ b/src/main/java/com/app/service/S3Service.java @@ -2,11 +2,13 @@ import org.springframework.web.multipart.MultipartFile; -import java.io.InputStream; +import java.io.ByteArrayInputStream; +import java.util.concurrent.CompletableFuture; public interface S3Service { + boolean uploadFile(MultipartFile file, boolean isReadPublicly); - InputStream downloadFileAsStream(String key); + CompletableFuture downloadFileAsStream(String key); } diff --git a/src/main/java/com/app/service/S3ServiceImpl.java b/src/main/java/com/app/service/S3ServiceImpl.java index 7651f3a..d47fd12 100644 --- a/src/main/java/com/app/service/S3ServiceImpl.java +++ b/src/main/java/com/app/service/S3ServiceImpl.java @@ -5,46 +5,62 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; -import software.amazon.awssdk.core.ResponseBytes; -import software.amazon.awssdk.core.exception.SdkException; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.CompletedUpload; +import software.amazon.awssdk.transfer.s3.model.Upload; +import software.amazon.awssdk.transfer.s3.model.UploadRequest; +import software.amazon.awssdk.transfer.s3.progress.LoggingTransferListener; import java.io.ByteArrayInputStream; -import java.io.InputStream; +import java.io.ByteArrayOutputStream; +import java.util.concurrent.CompletableFuture; @Service @Log4j2 @RequiredArgsConstructor public class S3ServiceImpl implements S3Service { - private final S3Client s3Client; + private final S3AsyncClient s3AsyncClient; @Value("${aws.s3.bucket-name}") private String bucketName; + private S3TransferManager createTransferManager() { + return S3TransferManager.builder() + .s3Client(s3AsyncClient) + .build(); + } + + @Override public boolean uploadFile(MultipartFile file, boolean isReadPublicly) { log.info("Started uploading file '{}' to S3 Bucket '{}'", file.getOriginalFilename(), bucketName); - PutObjectRequest putObjectRequest; - if (isReadPublicly) { - putObjectRequest = PutObjectRequest.builder() - .bucket(bucketName) - .key(file.getOriginalFilename()).acl("public-read") - .build(); - } else { - putObjectRequest = PutObjectRequest.builder() - .bucket(bucketName) - .key(file.getOriginalFilename()) - .build(); - } - try { - s3Client.putObject(putObjectRequest, RequestBody.fromBytes(file.getBytes())); + try (S3TransferManager transferManager = createTransferManager()) { + UploadRequest uploadRequest; + if (isReadPublicly) { + uploadRequest = UploadRequest.builder() + .putObjectRequest(builder -> builder.bucket(bucketName).key(file.getOriginalFilename()).acl("public-read")) + .requestBody(AsyncRequestBody.fromBytes(file.getBytes())) + .addTransferListener(LoggingTransferListener.create()) // For logging progress + .build(); + } else { + uploadRequest = UploadRequest.builder() + .putObjectRequest(builder -> builder.bucket(bucketName).key(file.getOriginalFilename())) + .requestBody(AsyncRequestBody.fromBytes(file.getBytes())) + .addTransferListener(LoggingTransferListener.create()) // For logging progress + .build(); + } + // Start the file upload + Upload upload = transferManager.upload(uploadRequest); + + // Wait for the upload to complete + CompletableFuture uploadCompletion = upload.completionFuture(); + uploadCompletion.join(); log.info("Successfully uploaded file to S3. Bucket: {}, Key: {}", bucketName, file.getOriginalFilename()); return true; } catch (Exception e) { @@ -54,29 +70,26 @@ public boolean uploadFile(MultipartFile file, boolean isReadPublicly) { } @Override - public InputStream downloadFileAsStream(String key) { - try { - GetObjectRequest getObjectRequest = GetObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build(); - - ResponseBytes getObjectResponse = s3Client.getObjectAsBytes(getObjectRequest); - if (getObjectResponse == null) { - log.warn("Failed to get file from S3 bucket: Response is null"); - return new ByteArrayInputStream(new byte[0]); - } + public CompletableFuture downloadFileAsStream(String key) { + GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); - log.info("Successfully getting file in bytes from S3 bucket."); - byte[] fileBytes = getObjectResponse.asByteArray(); - return new ByteArrayInputStream(fileBytes); - - } catch (S3Exception e) { - log.error("Failed to fetch object from S3 Bucket: {}, Key: {}", bucketName, key, e); - throw e; - } catch (SdkException e) { - log.error("Error while downloading file from S3 Bucket: {}, Key: {}", bucketName, key, e); - throw e; - } + // Download the file directly into a ByteArrayOutputStream + return s3AsyncClient.getObject(getObjectRequest, AsyncResponseTransformer.toBytes()) + .thenApply(response -> { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try { + byteArrayOutputStream.write(response.asByteArray()); + } catch (Exception e) { + log.error("Failed to write response to ByteArrayOutputStream. Bucket: {}, Key: {}", bucketName, key, e); + } + return new ByteArrayInputStream(byteArrayOutputStream.toByteArray()); + }) + .exceptionally(e -> { + log.error("Failed to download file from S3. Bucket: {}, Key: {}", bucketName, key, e); + return null; + }); } }