From bc2fa0a2a1a1fae6dd235bbe56388c4ef08a73a3 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Mon, 6 Oct 2025 17:39:59 -0400 Subject: [PATCH 01/11] feat: s3 transfer manager client --- aws-runtime/aws-http/api/aws-http.api | 1 + .../AwsBusinessMetricsUtils.kt | 1 + hll/build.gradle.kts | 2 +- .../api/s3-transfer-manager.api | 4 ++ hll/s3-transfer-manager/build.gradle.kts | 19 ++++++++ .../BusinessMetricInterceptor.kt | 16 +++++++ .../MultiPartDownloadType.kt | 16 +++++++ .../s3transfermanager/S3TransferManager.kt | 44 +++++++++++++++++++ .../s3transfermanager/TransferInterceptor.kt | 5 +++ settings.gradle.kts | 1 + 10 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 hll/s3-transfer-manager/api/s3-transfer-manager.api create mode 100644 hll/s3-transfer-manager/build.gradle.kts create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptor.kt create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/MultiPartDownloadType.kt create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index fea5bdc046c..3bbefbda4c8 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -171,6 +171,7 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexi public final class aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric : java/lang/Enum, aws/smithy/kotlin/runtime/businessmetrics/BusinessMetric { public static final field DDB_MAPPER Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; public static final field S3_EXPRESS_BUCKET Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; + public static final field S3_TRANSFER Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; public static fun getEntries ()Lkotlin/enums/EnumEntries; public fun getIdentifier ()Ljava/lang/String; public fun toString ()Ljava/lang/String; diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt index 239686ee3df..f806b908149 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt @@ -62,6 +62,7 @@ internal fun formatMetrics(metrics: MutableSet, logger: Logger): public enum class AwsBusinessMetric(public override val identifier: String) : BusinessMetric { S3_EXPRESS_BUCKET("J"), DDB_MAPPER("d"), + S3_TRANSFER("G"), ; @InternalApi diff --git a/hll/build.gradle.kts b/hll/build.gradle.kts index e6c5661b02a..a2e924f4cba 100644 --- a/hll/build.gradle.kts +++ b/hll/build.gradle.kts @@ -45,7 +45,7 @@ val hllPreviewVersion = if (sdkVersion.contains("-SNAPSHOT")) { // e.g. 1.3.29-b subprojects { group = "aws.sdk.kotlin" - version = hllPreviewVersion + version = if (name == "s3-transfer-manager") sdkVersion else hllPreviewVersion // TODO Use configurePublishing when migrating to Sonatype Publisher API / JReleaser configurePublishing("aws-sdk-kotlin") } diff --git a/hll/s3-transfer-manager/api/s3-transfer-manager.api b/hll/s3-transfer-manager/api/s3-transfer-manager.api new file mode 100644 index 00000000000..0a0abe81f34 --- /dev/null +++ b/hll/s3-transfer-manager/api/s3-transfer-manager.api @@ -0,0 +1,4 @@ +public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager { + public fun ()V +} + diff --git a/hll/s3-transfer-manager/build.gradle.kts b/hll/s3-transfer-manager/build.gradle.kts new file mode 100644 index 00000000000..c68662a3d64 --- /dev/null +++ b/hll/s3-transfer-manager/build.gradle.kts @@ -0,0 +1,19 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +description = "S3 Transfer Manager for the AWS SDK for Kotlin" +extra["displayName"] = "AWS :: SDK :: Kotlin :: HLL :: S3TransferManager" +extra["moduleName"] = "aws.sdk.kotlin.hll.s3transfermanager" + +kotlin { + sourceSets { + commonMain { + dependencies { + implementation(project(":aws-runtime:aws-http")) + implementation(project(":services:s3")) + } + } + } +} \ No newline at end of file diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptor.kt new file mode 100644 index 00000000000..65c5b9a536c --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptor.kt @@ -0,0 +1,16 @@ +package aws.sdk.kotlin.hll.s3transfermanager + +import aws.sdk.kotlin.runtime.http.interceptors.businessmetrics.AwsBusinessMetric +import aws.smithy.kotlin.runtime.businessmetrics.emitBusinessMetric +import aws.smithy.kotlin.runtime.client.RequestInterceptorContext +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor + +/** + * An interceptor that emits the S3 Transfer Manager business metric + */ +internal object BusinessMetricInterceptor : HttpInterceptor { + override suspend fun modifyBeforeSerialization(context: RequestInterceptorContext): Any { + context.executionContext.emitBusinessMetric(AwsBusinessMetric.S3_TRANSFER) + return context.request + } +} \ No newline at end of file diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/MultiPartDownloadType.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/MultiPartDownloadType.kt new file mode 100644 index 00000000000..9b1c0718f6b --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/MultiPartDownloadType.kt @@ -0,0 +1,16 @@ +package aws.sdk.kotlin.hll.s3transfermanager + +/** + * TODO + */ +public sealed interface MultiPartDownloadType + +/** + * TODO + */ +public object Range : MultiPartDownloadType + +/** + * TODO + */ +public object Part : MultiPartDownloadType \ No newline at end of file diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt new file mode 100644 index 00000000000..0c449eab606 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt @@ -0,0 +1,44 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager + +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.withConfig + +/** + * High level utility for managing transfers to Amazon S3. + */ +public class S3TransferManager private constructor( + public val client: S3Client, + public val partSize: Long?, + public val multipartUploadThreshold: Long?, + public val multipartDownloadType: MultiPartDownloadType?, + public val interceptors: MutableList?, +) { + public companion object { + public operator fun invoke(block: Builder.() -> Unit): S3TransferManager = + Builder().apply(block).build() + } + + public class Builder { + public var client: S3Client? = null + public var partSize: Long? = 8_000_000L + public var multipartUploadThreshold: Long? = 16_000_000L + public var multipartDownloadType: MultiPartDownloadType? = Part + public var interceptors: MutableList? = mutableListOf() + + internal fun build(): S3TransferManager = + S3TransferManager( + client = client?.withConfig { interceptors += BusinessMetricInterceptor } ?: error("client must be set"), + partSize = partSize, + multipartUploadThreshold = multipartUploadThreshold, + multipartDownloadType = multipartDownloadType, + interceptors = interceptors + ) + } + + public fun x(): String = "" // TODO +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt new file mode 100644 index 00000000000..1c46d19bf8b --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt @@ -0,0 +1,5 @@ +package aws.sdk.kotlin.hll.s3transfermanager + +public class TransferInterceptor { + // TODO +} \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 826adedd84d..bd31e6e53c4 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -57,6 +57,7 @@ include(":aws-runtime:aws-http") include(":hll") include(":hll:hll-codegen") include(":hll:hll-mapping-core") +include(":hll:s3-transfer-manager") include(":services") include(":tests") include(":tests:codegen") From 6895fc3ea927747c8be0e1d58cd0a3108acf295a Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 7 Oct 2025 17:08:37 -0400 Subject: [PATCH 02/11] saving work - upload file --- .../api/s3-transfer-manager.api | 294 +++++++++++++++ hll/s3-transfer-manager/build.gradle.kts | 4 +- .../BusinessMetricInterceptor.kt | 7 +- .../MultiPartDownloadType.kt | 16 - .../s3transfermanager/S3TransferManager.kt | 117 +++++- .../s3transfermanager/TransferInterceptor.kt | 66 +++- .../model/MultiPartDownloadType.kt | 21 ++ .../hll/s3transfermanager/model/UploadFile.kt | 339 ++++++++++++++++++ 8 files changed, 829 insertions(+), 35 deletions(-) delete mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/MultiPartDownloadType.kt create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType.kt create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFile.kt diff --git a/hll/s3-transfer-manager/api/s3-transfer-manager.api b/hll/s3-transfer-manager/api/s3-transfer-manager.api index 0a0abe81f34..23ab43edc64 100644 --- a/hll/s3-transfer-manager/api/s3-transfer-manager.api +++ b/hll/s3-transfer-manager/api/s3-transfer-manager.api @@ -1,4 +1,298 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager { + public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Companion; + public synthetic fun (Laws/sdk/kotlin/services/s3/S3Client;JJLaws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType;Ljava/util/List;Lkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getClient ()Laws/sdk/kotlin/services/s3/S3Client; + public final fun getInterceptors ()Ljava/util/List; + public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType; + public final fun getMultipartUploadThreshold ()J + public final fun getPartSize ()J + public final fun uploadFile (Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Builder { + public fun ()V + public final fun getClient ()Laws/sdk/kotlin/services/s3/S3Client; + public final fun getInterceptors ()Ljava/util/List; + public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType; + public final fun getMultipartUploadThreshold ()Ljava/lang/Long; + public final fun getPartSize ()Ljava/lang/Long; + public final fun setClient (Laws/sdk/kotlin/services/s3/S3Client;)V + public final fun setInterceptors (Ljava/util/List;)V + public final fun setMultipartDownloadType (Laws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType;)V + public final fun setMultipartUploadThreshold (Ljava/lang/Long;)V + public final fun setPartSize (Ljava/lang/Long;)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Companion { + public final fun invoke (Lkotlin/jvm/functions/Function1;)Laws/sdk/kotlin/hll/s3transfermanager/S3TransferManager; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/TransferContext { public fun ()V + public fun (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;[BLjava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;)V + public synthetic fun (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;[BLjava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun component1 ()Ljava/lang/Object; + public final fun component2 ()Ljava/lang/Object; + public final fun component3 ()Ljava/lang/Long; + public final fun component4 ()[B + public final fun component5 ()Ljava/lang/Long; + public final fun component6 ()Ljava/lang/Long; + public final fun component7 ()Ljava/lang/String; + public final fun component8 ()Ljava/lang/Long; + public final fun copy (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;[BLjava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static synthetic fun copy$default (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;[BLjava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;ILjava/lang/Object;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun equals (Ljava/lang/Object;)Z + public final fun getCurrentBytes ()[B + public final fun getCurrentFile ()Ljava/lang/String; + public final fun getRequest ()Ljava/lang/Object; + public final fun getResponse ()Ljava/lang/Object; + public final fun getTransferableBytes ()Ljava/lang/Long; + public final fun getTransferableFiles ()Ljava/lang/Long; + public final fun getTransferredBytes ()Ljava/lang/Long; + public final fun getTransferredFiles ()Ljava/lang/Long; + public fun hashCode ()I + public final fun setCurrentBytes ([B)V + public final fun setCurrentFile (Ljava/lang/String;)V + public final fun setRequest (Ljava/lang/Object;)V + public final fun setResponse (Ljava/lang/Object;)V + public final fun setTransferableBytes (Ljava/lang/Long;)V + public final fun setTransferableFiles (Ljava/lang/Long;)V + public final fun setTransferredBytes (Ljava/lang/Long;)V + public final fun setTransferredFiles (Ljava/lang/Long;)V + public fun toString ()Ljava/lang/String; +} + +public abstract interface class aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor { + public fun modifyAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun modifyAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun modifyAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun modifyAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun modifyBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun modifyBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun modifyBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun modifyBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun readAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public fun readAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public fun readAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public fun readAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public fun readBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public fun readBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public fun readBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public fun readBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor$DefaultImpls { + public static fun modifyAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun modifyAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun modifyAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun modifyAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun modifyBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun modifyBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun modifyBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun modifyBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun readAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public static fun readAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public static fun readAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public static fun readAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public static fun readBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public static fun readBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public static fun readBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public static fun readBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V +} + +public abstract interface class aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType { +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/Part : aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType { + public static final field INSTANCE Laws/sdk/kotlin/hll/s3transfermanager/model/Part; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/Range : aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType { + public static final field INSTANCE Laws/sdk/kotlin/hll/s3transfermanager/model/Range; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest { + public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$Companion; + public synthetic fun (Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/String;Ljava/lang/Boolean;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;JLjava/lang/String;Ljava/lang/String;Laws/smithy/kotlin/runtime/time/Instant;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus;Laws/sdk/kotlin/services/s3/model/ObjectLockMode;Laws/smithy/kotlin/runtime/time/Instant;Laws/sdk/kotlin/services/s3/model/RequestPayer;Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/StorageClass;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getAcl ()Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl; + public final fun getBody ()Laws/smithy/kotlin/runtime/content/ByteStream; + public final fun getBucket ()Ljava/lang/String; + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getCacheControl ()Ljava/lang/String; + public final fun getChecksumAlgorithm ()Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getContentDisposition ()Ljava/lang/String; + public final fun getContentEncoding ()Ljava/lang/String; + public final fun getContentLanguage ()Ljava/lang/String; + public final fun getContentLength ()J + public final fun getContentType ()Ljava/lang/String; + public final fun getExpectedBucketOwner ()Ljava/lang/String; + public final fun getExpires ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getGrantFullControl ()Ljava/lang/String; + public final fun getGrantRead ()Ljava/lang/String; + public final fun getGrantReadAcp ()Ljava/lang/String; + public final fun getGrantWriteAcp ()Ljava/lang/String; + public final fun getIfMatch ()Ljava/lang/String; + public final fun getIfNoneMatch ()Ljava/lang/String; + public final fun getKey ()Ljava/lang/String; + public final fun getMetadata ()Ljava/util/Map; + public final fun getObjectLockLegalHoldStatus ()Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus; + public final fun getObjectLockMode ()Laws/sdk/kotlin/services/s3/model/ObjectLockMode; + public final fun getObjectLockRetainUntilDate ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getRequestPayer ()Laws/sdk/kotlin/services/s3/model/RequestPayer; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSource ()Ljava/lang/String; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKey ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getStorageClass ()Laws/sdk/kotlin/services/s3/model/StorageClass; + public final fun getTagging ()Ljava/lang/String; + public final fun getWebsiteRedirectLocation ()Ljava/lang/String; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$Builder { + public fun ()V + public final fun getAcl ()Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl; + public final fun getBody ()Laws/smithy/kotlin/runtime/content/ByteStream; + public final fun getBucket ()Ljava/lang/String; + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getCacheControl ()Ljava/lang/String; + public final fun getChecksumAlgorithm ()Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getContentDisposition ()Ljava/lang/String; + public final fun getContentEncoding ()Ljava/lang/String; + public final fun getContentLanguage ()Ljava/lang/String; + public final fun getContentLength ()Ljava/lang/Long; + public final fun getContentType ()Ljava/lang/String; + public final fun getExpectedBucketOwner ()Ljava/lang/String; + public final fun getExpires ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getGrantFullControl ()Ljava/lang/String; + public final fun getGrantRead ()Ljava/lang/String; + public final fun getGrantReadAcp ()Ljava/lang/String; + public final fun getGrantWriteAcp ()Ljava/lang/String; + public final fun getIfMatch ()Ljava/lang/String; + public final fun getIfNoneMatch ()Ljava/lang/String; + public final fun getKey ()Ljava/lang/String; + public final fun getMetadata ()Ljava/util/Map; + public final fun getObjectLockLegalHoldStatus ()Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus; + public final fun getObjectLockMode ()Laws/sdk/kotlin/services/s3/model/ObjectLockMode; + public final fun getObjectLockRetainUntilDate ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getRequestPayer ()Laws/sdk/kotlin/services/s3/model/RequestPayer; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSource ()Ljava/lang/String; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKey ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getStorageClass ()Laws/sdk/kotlin/services/s3/model/StorageClass; + public final fun getTagging ()Ljava/lang/String; + public final fun getWebsiteRedirectLocation ()Ljava/lang/String; + public final fun setAcl (Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl;)V + public final fun setBody (Laws/smithy/kotlin/runtime/content/ByteStream;)V + public final fun setBucket (Ljava/lang/String;)V + public final fun setBucketKeyEnabled (Ljava/lang/Boolean;)V + public final fun setCacheControl (Ljava/lang/String;)V + public final fun setChecksumAlgorithm (Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm;)V + public final fun setChecksumCrc32 (Ljava/lang/String;)V + public final fun setChecksumCrc32C (Ljava/lang/String;)V + public final fun setChecksumCrc64Nvme (Ljava/lang/String;)V + public final fun setChecksumSha1 (Ljava/lang/String;)V + public final fun setChecksumSha256 (Ljava/lang/String;)V + public final fun setContentDisposition (Ljava/lang/String;)V + public final fun setContentEncoding (Ljava/lang/String;)V + public final fun setContentLanguage (Ljava/lang/String;)V + public final fun setContentLength (Ljava/lang/Long;)V + public final fun setContentType (Ljava/lang/String;)V + public final fun setExpectedBucketOwner (Ljava/lang/String;)V + public final fun setExpires (Laws/smithy/kotlin/runtime/time/Instant;)V + public final fun setGrantFullControl (Ljava/lang/String;)V + public final fun setGrantRead (Ljava/lang/String;)V + public final fun setGrantReadAcp (Ljava/lang/String;)V + public final fun setGrantWriteAcp (Ljava/lang/String;)V + public final fun setIfMatch (Ljava/lang/String;)V + public final fun setIfNoneMatch (Ljava/lang/String;)V + public final fun setKey (Ljava/lang/String;)V + public final fun setMetadata (Ljava/util/Map;)V + public final fun setObjectLockLegalHoldStatus (Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus;)V + public final fun setObjectLockMode (Laws/sdk/kotlin/services/s3/model/ObjectLockMode;)V + public final fun setObjectLockRetainUntilDate (Laws/smithy/kotlin/runtime/time/Instant;)V + public final fun setRequestPayer (Laws/sdk/kotlin/services/s3/model/RequestPayer;)V + public final fun setServerSideEncryption (Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;)V + public final fun setSource (Ljava/lang/String;)V + public final fun setSseCustomerAlgorithm (Ljava/lang/String;)V + public final fun setSseCustomerKey (Ljava/lang/String;)V + public final fun setSseCustomerKeyMd5 (Ljava/lang/String;)V + public final fun setSsekmsEncryptionContext (Ljava/lang/String;)V + public final fun setSsekmsKeyId (Ljava/lang/String;)V + public final fun setStorageClass (Laws/sdk/kotlin/services/s3/model/StorageClass;)V + public final fun setTagging (Ljava/lang/String;)V + public final fun setWebsiteRedirectLocation (Ljava/lang/String;)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$Companion { + public final fun invoke (Lkotlin/jvm/functions/Function1;)Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse { + public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse$Companion; + public fun (Ljava/lang/Boolean;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ChecksumType;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/RequestCharged;Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;Ljava/lang/String;Ljava/lang/String;)V + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getChecksumType ()Laws/sdk/kotlin/services/s3/model/ChecksumType; + public final fun getETag ()Ljava/lang/String; + public final fun getExpiration ()Ljava/lang/String; + public final fun getRequestCharged ()Laws/sdk/kotlin/services/s3/model/RequestCharged; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getVersionId ()Ljava/lang/String; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse$Builder { + public fun ()V + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getChecksumType ()Laws/sdk/kotlin/services/s3/model/ChecksumType; + public final fun getETag ()Ljava/lang/String; + public final fun getExpiration ()Ljava/lang/String; + public final fun getRequestCharged ()Laws/sdk/kotlin/services/s3/model/RequestCharged; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getVersionId ()Ljava/lang/String; + public final fun setBucketKeyEnabled (Ljava/lang/Boolean;)V + public final fun setChecksumCrc32 (Ljava/lang/String;)V + public final fun setChecksumCrc32C (Ljava/lang/String;)V + public final fun setChecksumCrc64Nvme (Ljava/lang/String;)V + public final fun setChecksumSha1 (Ljava/lang/String;)V + public final fun setChecksumSha256 (Ljava/lang/String;)V + public final fun setChecksumType (Laws/sdk/kotlin/services/s3/model/ChecksumType;)V + public final fun setETag (Ljava/lang/String;)V + public final fun setExpiration (Ljava/lang/String;)V + public final fun setRequestCharged (Laws/sdk/kotlin/services/s3/model/RequestCharged;)V + public final fun setServerSideEncryption (Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;)V + public final fun setSsekmsKeyId (Ljava/lang/String;)V + public final fun setVersionId (Ljava/lang/String;)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse$Companion { + public final fun invoke (Lkotlin/jvm/functions/Function1;)Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse; } diff --git a/hll/s3-transfer-manager/build.gradle.kts b/hll/s3-transfer-manager/build.gradle.kts index c68662a3d64..0125d442603 100644 --- a/hll/s3-transfer-manager/build.gradle.kts +++ b/hll/s3-transfer-manager/build.gradle.kts @@ -12,8 +12,8 @@ kotlin { commonMain { dependencies { implementation(project(":aws-runtime:aws-http")) - implementation(project(":services:s3")) + implementation(project(":services:s3")) // TODO: Hardcode an S3 Client version to avoid breakages } } } -} \ No newline at end of file +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptor.kt index 65c5b9a536c..f9f2228364b 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptor.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptor.kt @@ -1,3 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + package aws.sdk.kotlin.hll.s3transfermanager import aws.sdk.kotlin.runtime.http.interceptors.businessmetrics.AwsBusinessMetric @@ -13,4 +18,4 @@ internal object BusinessMetricInterceptor : HttpInterceptor { context.executionContext.emitBusinessMetric(AwsBusinessMetric.S3_TRANSFER) return context.request } -} \ No newline at end of file +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/MultiPartDownloadType.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/MultiPartDownloadType.kt deleted file mode 100644 index 9b1c0718f6b..00000000000 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/MultiPartDownloadType.kt +++ /dev/null @@ -1,16 +0,0 @@ -package aws.sdk.kotlin.hll.s3transfermanager - -/** - * TODO - */ -public sealed interface MultiPartDownloadType - -/** - * TODO - */ -public object Range : MultiPartDownloadType - -/** - * TODO - */ -public object Part : MultiPartDownloadType \ No newline at end of file diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt index 0c449eab606..dab45422913 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt @@ -5,18 +5,29 @@ package aws.sdk.kotlin.hll.s3transfermanager +import aws.sdk.kotlin.hll.s3transfermanager.model.MultiPartDownloadType +import aws.sdk.kotlin.hll.s3transfermanager.model.Part +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest.Companion.toCreateMultiPartUploadRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest.Companion.toPutObjectRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileResponse import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.PutObjectRequest import aws.sdk.kotlin.services.s3.withConfig +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope /** * High level utility for managing transfers to Amazon S3. */ public class S3TransferManager private constructor( public val client: S3Client, - public val partSize: Long?, - public val multipartUploadThreshold: Long?, - public val multipartDownloadType: MultiPartDownloadType?, - public val interceptors: MutableList?, + public val partSize: Long, + public val multipartUploadThreshold: Long, + public val multipartDownloadType: MultiPartDownloadType, + public val interceptors: MutableList, ) { public companion object { public operator fun invoke(block: Builder.() -> Unit): S3TransferManager = @@ -25,20 +36,100 @@ public class S3TransferManager private constructor( public class Builder { public var client: S3Client? = null - public var partSize: Long? = 8_000_000L - public var multipartUploadThreshold: Long? = 16_000_000L - public var multipartDownloadType: MultiPartDownloadType? = Part - public var interceptors: MutableList? = mutableListOf() + public var partSize: Long? = null + public var multipartUploadThreshold: Long? = null + public var multipartDownloadType: MultiPartDownloadType? = null + public var interceptors: MutableList? = null internal fun build(): S3TransferManager = S3TransferManager( client = client?.withConfig { interceptors += BusinessMetricInterceptor } ?: error("client must be set"), - partSize = partSize, - multipartUploadThreshold = multipartUploadThreshold, - multipartDownloadType = multipartDownloadType, - interceptors = interceptors + partSize = partSize ?: 8_000_000L, + multipartUploadThreshold = multipartUploadThreshold ?: 16_000_000L, + multipartDownloadType = multipartDownloadType ?: Part, + interceptors = interceptors ?: mutableListOf(), ) } - public fun x(): String = "" // TODO + private var context: TransferContext = TransferContext() + + // TODO: Try to find parts of the code you can commonize + /** + * TODO + */ + public suspend fun uploadFile(uploadFileRequest: UploadFileRequest): Deferred = coroutineScope { + async { + val multiPartUpload = uploadFileRequest.contentLength >= multipartUploadThreshold + + context.request = if (multiPartUpload) { + uploadFileRequest.toCreateMultiPartUploadRequest() + } else { + uploadFileRequest.toPutObjectRequest() + } + + operationHook(TransferInitiated) { + context.transferredBytes = 0L + context.transferableBytes = uploadFileRequest.contentLength + + if (multiPartUpload) { + context.response = client.createMultipartUpload(context.request as CreateMultipartUploadRequest) + } + } + + operationHook(BytesTransferred) { + if (multiPartUpload) { + // TODO: MPU logic + // TODO: Update bytes transferred + } else { + context.response = client.putObject(context.request as PutObjectRequest) + context.transferredBytes = context.transferableBytes + } + } + + operationHook(TransferCompleted) { + if (multiPartUpload) { + // TODO: MPU logic + // TODO: Update bytes transferred? + } + } + + UploadFileResponse.fromS3Response(context.response) + } + } + + private suspend fun operationHook(hook: TransferHook, block: suspend () -> Any) { + interceptors.forEach { interceptor -> + when (hook) { + is TransferInitiated -> { + interceptor.readBeforeTransferInitiated(context) + context = interceptor.modifyBeforeTransferInitiated(context) + block.invoke() + interceptor.readAfterTransferInitiated(context) + context = interceptor.modifyAfterTransferInitiated(context) + } + is BytesTransferred -> { + interceptor.readBeforeBytesTransferred(context) + context = interceptor.modifyBeforeBytesTransferred(context) + block.invoke() + interceptor.readAfterBytesTransferred(context) + context = interceptor.modifyAfterBytesTransferred(context) + } + is FileTransferred -> { + interceptor.readBeforeFileTransferred(context) + context = interceptor.modifyBeforeFileTransferred(context) + block.invoke() + interceptor.readAfterFileTransferred(context) + context = interceptor.modifyAfterFileTransferred(context) + } + is TransferCompleted -> { + interceptor.readBeforeTransferCompleted(context) + context = interceptor.modifyBeforeTransferCompleted(context) + block.invoke() + interceptor.readAfterTransferCompleted(context) + context = interceptor.modifyAfterTransferCompleted(context) + } + else -> error("TransferHook not implemented: ${hook::class.simpleName}") + } + } + } } diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt index 1c46d19bf8b..49dd97b5d18 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt @@ -1,5 +1,65 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + package aws.sdk.kotlin.hll.s3transfermanager -public class TransferInterceptor { - // TODO -} \ No newline at end of file +// TODO: Add documentation to each thing in the context +/** + * TODO + */ +public data class TransferContext( + // Req/Resp + var request: Any? = null, + var response: Any? = null, + + // Byte transfers + var transferableBytes: Long? = null, + var currentBytes: ByteArray? = null, + var transferredBytes: Long? = null, + + // File transfers + var transferableFiles: Long? = null, + var currentFile: String? = null, + var transferredFiles: Long? = null, +) + +// TODO: Add documentation to each hook +/** + * TODO + */ +public interface TransferInterceptor { + // Transfer initialization hooks + public fun readBeforeTransferInitiated(context: TransferContext) {} + public fun modifyBeforeTransferInitiated(context: TransferContext): TransferContext = context + public fun readAfterTransferInitiated(context: TransferContext) {} + public fun modifyAfterTransferInitiated(context: TransferContext): TransferContext = context + + // Byte transferring hooks + public fun readBeforeBytesTransferred(context: TransferContext) {} + public fun modifyBeforeBytesTransferred(context: TransferContext): TransferContext = context + public fun readAfterBytesTransferred(context: TransferContext) {} + public fun modifyAfterBytesTransferred(context: TransferContext): TransferContext = context + + // File transfer hooks + public fun readBeforeFileTransferred(context: TransferContext) {} + public fun modifyBeforeFileTransferred(context: TransferContext): TransferContext = context + public fun readAfterFileTransferred(context: TransferContext) {} + public fun modifyAfterFileTransferred(context: TransferContext): TransferContext = context + + // Transfer completion hooks + public fun readBeforeTransferCompleted(context: TransferContext) {} + public fun modifyBeforeTransferCompleted(context: TransferContext): TransferContext = context + public fun readAfterTransferCompleted(context: TransferContext) {} + public fun modifyAfterTransferCompleted(context: TransferContext): TransferContext = context +} + +/** + * TODO + */ +internal interface TransferHook +internal object TransferInitiated : TransferHook +internal object BytesTransferred : TransferHook +internal object FileTransferred : TransferHook +internal object TransferCompleted : TransferHook diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType.kt new file mode 100644 index 00000000000..a21c8fd8264 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType.kt @@ -0,0 +1,21 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.model + +/** + * TODO + */ +public sealed interface MultiPartDownloadType + +/** + * TODO + */ +public object Range : MultiPartDownloadType + +/** + * TODO + */ +public object Part : MultiPartDownloadType diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFile.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFile.kt new file mode 100644 index 00000000000..7be81bcb995 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFile.kt @@ -0,0 +1,339 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.model + +import aws.sdk.kotlin.services.s3.model.ChecksumAlgorithm +import aws.sdk.kotlin.services.s3.model.ChecksumType +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadResponse +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.ObjectCannedAcl +import aws.sdk.kotlin.services.s3.model.ObjectLockLegalHoldStatus +import aws.sdk.kotlin.services.s3.model.ObjectLockMode +import aws.sdk.kotlin.services.s3.model.PutObjectRequest +import aws.sdk.kotlin.services.s3.model.PutObjectResponse +import aws.sdk.kotlin.services.s3.model.RequestCharged +import aws.sdk.kotlin.services.s3.model.RequestPayer +import aws.sdk.kotlin.services.s3.model.ServerSideEncryption +import aws.sdk.kotlin.services.s3.model.StorageClass +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.time.Instant +import kotlin.Boolean + +// TODO: Add documentation to each thing ? +/** + * TODO + */ +public class UploadFileRequest private constructor( + public val acl: ObjectCannedAcl?, + public val body: ByteStream?, + public val bucket: String?, + public val bucketKeyEnabled: Boolean?, + public val cacheControl: String?, + public val checksumAlgorithm: ChecksumAlgorithm?, + public val checksumCrc32: String?, + public val checksumCrc32C: String?, + public val checksumCrc64Nvme: String?, + public val checksumSha1: String?, + public val checksumSha256: String?, + public val contentDisposition: String?, + public val contentEncoding: String?, + public val contentLanguage: String?, + public val contentLength: Long, + public val contentType: String?, + public val expectedBucketOwner: String?, + public val expires: Instant?, // TODO: Is this the right instant? + public val grantFullControl: String?, + public val grantRead: String?, + public val grantReadAcp: String?, + public val grantWriteAcp: String?, + public val ifMatch: String?, + public val ifNoneMatch: String?, + public val key: String?, + public val metadata: Map?, + public val objectLockLegalHoldStatus: ObjectLockLegalHoldStatus?, + public val objectLockMode: ObjectLockMode?, + public val objectLockRetainUntilDate: Instant?, + public val requestPayer: RequestPayer?, + public val serverSideEncryption: ServerSideEncryption?, + public val source: String?, + public val sseCustomerAlgorithm: String?, + public val sseCustomerKey: String?, + public val sseCustomerKeyMd5: String?, + public val ssekmsEncryptionContext: String?, + public val ssekmsKeyId: String?, + public val storageClass: StorageClass?, + public val tagging: String?, + public val websiteRedirectLocation: String?, +) { + public companion object { + public operator fun invoke(block: Builder.() -> Unit): UploadFileRequest = + Builder().apply(block).build() + + // TODO: Checkout the null values CAREFULLY + internal fun UploadFileRequest.toPutObjectRequest(): PutObjectRequest = + PutObjectRequest { + acl = this@toPutObjectRequest.acl + body = this@toPutObjectRequest.body + bucket = this@toPutObjectRequest.bucket + bucketKeyEnabled = this@toPutObjectRequest.bucketKeyEnabled + cacheControl = this@toPutObjectRequest.cacheControl + checksumAlgorithm = this@toPutObjectRequest.checksumAlgorithm + checksumCrc32 = this@toPutObjectRequest.checksumCrc32 + checksumCrc32C = this@toPutObjectRequest.checksumCrc32C + checksumCrc64Nvme = this@toPutObjectRequest.checksumCrc64Nvme + checksumSha1 = this@toPutObjectRequest.checksumSha1 + checksumSha256 = this@toPutObjectRequest.checksumSha256 + contentDisposition = this@toPutObjectRequest.contentDisposition + contentEncoding = this@toPutObjectRequest.contentEncoding + contentLanguage = this@toPutObjectRequest.contentLanguage + contentLength = this@toPutObjectRequest.contentLength + contentMd5 = null + contentType = this@toPutObjectRequest.contentType + expectedBucketOwner = this@toPutObjectRequest.expectedBucketOwner + expires = this@toPutObjectRequest.expires + grantFullControl = this@toPutObjectRequest.grantFullControl + grantRead = this@toPutObjectRequest.grantRead + grantReadAcp = this@toPutObjectRequest.grantReadAcp + grantWriteAcp = this@toPutObjectRequest.grantWriteAcp + ifMatch = this@toPutObjectRequest.ifMatch + ifNoneMatch = this@toPutObjectRequest.ifNoneMatch + key = this@toPutObjectRequest.key + metadata = this@toPutObjectRequest.metadata + objectLockLegalHoldStatus = this@toPutObjectRequest.objectLockLegalHoldStatus + objectLockMode = this@toPutObjectRequest.objectLockMode + objectLockRetainUntilDate = this@toPutObjectRequest.objectLockRetainUntilDate + requestPayer = this@toPutObjectRequest.requestPayer + serverSideEncryption = this@toPutObjectRequest.serverSideEncryption + sseCustomerAlgorithm = this@toPutObjectRequest.sseCustomerAlgorithm + sseCustomerKey = this@toPutObjectRequest.sseCustomerKey + sseCustomerKeyMd5 = this@toPutObjectRequest.sseCustomerKeyMd5 + ssekmsEncryptionContext = this@toPutObjectRequest.ssekmsEncryptionContext + ssekmsKeyId = this@toPutObjectRequest.ssekmsKeyId + storageClass = this@toPutObjectRequest.storageClass + tagging = this@toPutObjectRequest.tagging + websiteRedirectLocation = this@toPutObjectRequest.websiteRedirectLocation + writeOffsetBytes = null + } + + // TODO: Checkout the null values CAREFULLY + internal fun UploadFileRequest.toCreateMultiPartUploadRequest(): CreateMultipartUploadRequest = + CreateMultipartUploadRequest { + acl = this@toCreateMultiPartUploadRequest.acl + bucket = this@toCreateMultiPartUploadRequest.bucket + bucketKeyEnabled = this@toCreateMultiPartUploadRequest.bucketKeyEnabled + cacheControl = this@toCreateMultiPartUploadRequest.cacheControl + checksumAlgorithm = this@toCreateMultiPartUploadRequest.checksumAlgorithm + checksumType = null + contentDisposition = this@toCreateMultiPartUploadRequest.contentDisposition + contentEncoding = this@toCreateMultiPartUploadRequest.contentEncoding + contentLanguage = this@toCreateMultiPartUploadRequest.contentLanguage + contentType = this@toCreateMultiPartUploadRequest.contentType + expectedBucketOwner = this@toCreateMultiPartUploadRequest.expectedBucketOwner + expires = this@toCreateMultiPartUploadRequest.expires + grantFullControl = this@toCreateMultiPartUploadRequest.grantFullControl + grantRead = this@toCreateMultiPartUploadRequest.grantRead + grantReadAcp = this@toCreateMultiPartUploadRequest.grantReadAcp + grantWriteAcp = this@toCreateMultiPartUploadRequest.grantWriteAcp + key = this@toCreateMultiPartUploadRequest.key + metadata = this@toCreateMultiPartUploadRequest.metadata + objectLockLegalHoldStatus = this@toCreateMultiPartUploadRequest.objectLockLegalHoldStatus + objectLockMode = this@toCreateMultiPartUploadRequest.objectLockMode + objectLockRetainUntilDate = this@toCreateMultiPartUploadRequest.objectLockRetainUntilDate + requestPayer = this@toCreateMultiPartUploadRequest.requestPayer + serverSideEncryption = this@toCreateMultiPartUploadRequest.serverSideEncryption + sseCustomerAlgorithm = this@toCreateMultiPartUploadRequest.sseCustomerAlgorithm + sseCustomerKey = this@toCreateMultiPartUploadRequest.sseCustomerKey + sseCustomerKeyMd5 = this@toCreateMultiPartUploadRequest.sseCustomerKeyMd5 + ssekmsEncryptionContext = this@toCreateMultiPartUploadRequest.ssekmsEncryptionContext + ssekmsKeyId = this@toCreateMultiPartUploadRequest.ssekmsKeyId + storageClass = this@toCreateMultiPartUploadRequest.storageClass + tagging = this@toCreateMultiPartUploadRequest.tagging + websiteRedirectLocation = this@toCreateMultiPartUploadRequest.websiteRedirectLocation + } + } + + public class Builder { + public var acl: ObjectCannedAcl? = null + public var body: ByteStream? = null + public var bucket: String? = null + public var bucketKeyEnabled: Boolean? = null + public var cacheControl: String? = null + public var checksumAlgorithm: ChecksumAlgorithm? = null + public var checksumCrc32: String? = null + public var checksumCrc32C: String? = null + public var checksumCrc64Nvme: String? = null + public var checksumSha1: String? = null + public var checksumSha256: String? = null + public var contentDisposition: String? = null + public var contentEncoding: String? = null + public var contentLanguage: String? = null + public var contentLength: Long? = null + public var contentType: String? = null + public var expectedBucketOwner: String? = null + public var expires: Instant? = null // TODO: Is this the right instant? Write some tests to see if conversions work properly? + public var grantFullControl: String? = null + public var grantRead: String? = null + public var grantReadAcp: String? = null + public var grantWriteAcp: String? = null + public var ifMatch: String? = null + public var ifNoneMatch: String? = null + public var key: String? = null + public var metadata: Map? = null + public var objectLockLegalHoldStatus: ObjectLockLegalHoldStatus? = null + public var objectLockMode: ObjectLockMode? = null + public var objectLockRetainUntilDate: Instant? = null + public var requestPayer: RequestPayer? = null + public var source: String? = null + public var serverSideEncryption: ServerSideEncryption? = null + public var sseCustomerAlgorithm: String? = null + public var sseCustomerKey: String? = null + public var sseCustomerKeyMd5: String? = null + public var ssekmsEncryptionContext: String? = null + public var ssekmsKeyId: String? = null + public var storageClass: StorageClass? = null + public var tagging: String? = null + public var websiteRedirectLocation: String? = null + + internal fun build(): UploadFileRequest = + UploadFileRequest( + acl, + body, + bucket, + bucketKeyEnabled, + cacheControl, + checksumAlgorithm, + checksumCrc32, + checksumCrc32C, + checksumCrc64Nvme, + checksumSha1, + checksumSha256, + contentDisposition, + contentEncoding, + contentLanguage, + contentLength ?: error("contentLength must be set"), + contentType, + expectedBucketOwner, + expires, + grantFullControl, + grantRead, + grantReadAcp, + grantWriteAcp, + ifMatch, + ifNoneMatch, + key, + metadata, + objectLockLegalHoldStatus, + objectLockMode, + objectLockRetainUntilDate, + requestPayer, + serverSideEncryption, + source, + sseCustomerAlgorithm, + sseCustomerKey, + sseCustomerKeyMd5, + ssekmsEncryptionContext, + ssekmsKeyId, + storageClass, + tagging, + websiteRedirectLocation, + ) + } +} + +/** + * TODO + */ +public class UploadFileResponse( + public val bucketKeyEnabled: Boolean?, + public val checksumCrc32: String?, + public val checksumCrc32C: String?, + public val checksumCrc64Nvme: String?, + public val checksumSha1: String?, + public val checksumSha256: String?, + public val checksumType: ChecksumType?, + public val eTag: String?, + public val expiration: String?, + public val requestCharged: RequestCharged?, + public val serverSideEncryption: ServerSideEncryption?, + public val ssekmsKeyId: String?, + public val versionId: String?, +) { + public companion object { + public operator fun invoke(block: Builder.() -> Unit): UploadFileResponse = + Builder().apply(block).build() + + // TODO: Double check the conversion + internal fun fromS3Response(response: Any?): UploadFileResponse = + when (response) { + is PutObjectResponse -> + UploadFileResponse { + bucketKeyEnabled = response.bucketKeyEnabled + checksumCrc32 = response.checksumCrc32 + checksumCrc32C = response.checksumCrc32C + checksumCrc64Nvme = response.checksumCrc64Nvme + checksumSha1 = response.checksumSha1 + checksumSha256 = response.checksumSha256 + checksumType = response.checksumType + eTag = response.eTag + expiration = response.expiration + requestCharged = response.requestCharged + serverSideEncryption = response.serverSideEncryption + ssekmsKeyId = response.ssekmsKeyId + versionId = response.versionId + } + is CompleteMultipartUploadResponse -> + UploadFileResponse { + bucketKeyEnabled = response.bucketKeyEnabled + checksumCrc32 = response.checksumCrc32 + checksumCrc32C = response.checksumCrc32C + checksumCrc64Nvme = response.checksumCrc64Nvme + checksumSha1 = response.checksumSha1 + checksumSha256 = response.checksumSha256 + checksumType = response.checksumType + eTag = response.eTag + expiration = response.expiration + requestCharged = response.requestCharged + serverSideEncryption = response.serverSideEncryption + ssekmsKeyId = response.ssekmsKeyId + versionId = response.versionId + } + else -> error("Response must be a PutObjectResponse or CompleteMultipartUploadResponse") + } + } + + public class Builder { + public var bucketKeyEnabled: Boolean? = null + public var checksumCrc32: String? = null + public var checksumCrc32C: String? = null + public var checksumCrc64Nvme: String? = null + public var checksumSha1: String? = null + public var checksumSha256: String? = null + public var checksumType: ChecksumType? = null + public var eTag: String? = null + public var expiration: String? = null + public var requestCharged: RequestCharged? = null + public var serverSideEncryption: ServerSideEncryption? = null + public var ssekmsKeyId: String? = null + public var versionId: String? = null + + internal fun build(): UploadFileResponse = + UploadFileResponse( + bucketKeyEnabled, + checksumCrc32, + checksumCrc32C, + checksumCrc64Nvme, + checksumSha1, + checksumSha256, + checksumType, + eTag, + expiration, + requestCharged, + serverSideEncryption, + ssekmsKeyId, + versionId, + ) + } +} From c9486f186f9ab20364ca0f925aa568b9bf26fb9d Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 9 Oct 2025 13:38:57 -0400 Subject: [PATCH 03/11] self review --- .../sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt index 49dd97b5d18..566bde2d7f8 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt @@ -5,7 +5,7 @@ package aws.sdk.kotlin.hll.s3transfermanager -// TODO: Add documentation to each thing in the context +// TODO: Add PUBLIC documentation to each thing in the context /** * TODO */ @@ -25,7 +25,7 @@ public data class TransferContext( var transferredFiles: Long? = null, ) -// TODO: Add documentation to each hook +// TODO: Add PUBLIC documentation to each hook /** * TODO */ From 835866a9adf547346ac6cc63b60b1f0696c46540 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 16 Oct 2025 21:47:03 -0400 Subject: [PATCH 04/11] finish upload file --- gradle/libs.versions.toml | 6 + .../api/s3-transfer-manager.api | 12 +- hll/s3-transfer-manager/build.gradle.kts | 8 +- .../s3transfermanager/S3TransferManager.kt | 263 ++++++++++---- .../s3transfermanager/TransferInterceptor.kt | 13 +- .../model/MultiPartDownloadType.kt | 13 +- .../hll/s3transfermanager/model/UploadFile.kt | 339 ------------------ .../model/UploadFileRequest.kt | 151 ++++++++ .../model/UploadFileResponse.kt | 64 ++++ .../s3transfermanager/utils/Conversions.kt | 124 +++++++ .../hll/s3transfermanager/utils/UploadFile.kt | 140 ++++++++ .../BusinessMetricInterceptorTest.kt | 45 +++ .../TransferInterceptorTest.kt | 59 +++ .../hll/s3transfermanager/UploadFileTest.kt | 58 +++ 14 files changed, 866 insertions(+), 429 deletions(-) delete mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFile.kt create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest.kt create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse.kt create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Conversions.kt create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/UploadFile.kt create mode 100644 hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt create mode 100644 hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt create mode 100644 hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 277af76e376..a60d02ba0d5 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -28,6 +28,9 @@ mockk-version = "1.13.13" slf4j-version = "2.0.16" jsoup-version = "1.20.1" +# s3 transfer manager +s3-version = "1.5.62" + [libraries] aws-kotlin-repo-tools-build-support = { module="aws.sdk.kotlin.gradle:build-support", version.ref = "aws-kotlin-repo-tools-version" } @@ -92,6 +95,8 @@ smithy-kotlin-telemetry-provider-micrometer = { module = "aws.smithy.kotlin:tele smithy-kotlin-telemetry-provider-otel = { module = "aws.smithy.kotlin:telemetry-provider-otel", version.ref = "smithy-kotlin-runtime-version" } smithy-kotlin-test-suite = { module = "aws.smithy.kotlin:test-suite", version.ref = "smithy-kotlin-runtime-version" } smithy-kotlin-testing = { module = "aws.smithy.kotlin:testing", version.ref = "smithy-kotlin-runtime-version" } +smithy-kotlin-test-jvm = { module = "aws.smithy.kotlin:http-test-jvm", version.ref = "smithy-kotlin-runtime-version" } +smithy-kotlin-testing-jvm = { module = "aws.smithy.kotlin:testing-jvm", version.ref = "smithy-kotlin-runtime-version" } smithy-kotlin-codegen = { module = "software.amazon.smithy.kotlin:smithy-kotlin-codegen", version.ref = "smithy-kotlin-codegen-version" } smithy-kotlin-codegen-testutils = { module = "software.amazon.smithy.kotlin:smithy-kotlin-codegen-testutils", version.ref = "smithy-kotlin-codegen-version" } @@ -124,6 +129,7 @@ kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serializa mockk = { module = "io.mockk:mockk", version.ref = "mockk-version" } ddb-local = { module = "com.amazonaws:DynamoDBLocal", version.ref = "ddb-local-version" } +s3 = { module = "aws.sdk.kotlin:s3", version.ref = "s3-version" } [bundles] # bundle of smithy-kotlin dependencies all AWS service clients have diff --git a/hll/s3-transfer-manager/api/s3-transfer-manager.api b/hll/s3-transfer-manager/api/s3-transfer-manager.api index 23ab43edc64..ee14ee4d2dc 100644 --- a/hll/s3-transfer-manager/api/s3-transfer-manager.api +++ b/hll/s3-transfer-manager/api/s3-transfer-manager.api @@ -5,8 +5,9 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager { public final fun getInterceptors ()Ljava/util/List; public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType; public final fun getMultipartUploadThreshold ()J - public final fun getPartSize ()J + public final fun getTargePartSize ()J public final fun uploadFile (Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public final fun uploadFile (Lkotlin/jvm/functions/Function1;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; } public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Builder { @@ -14,13 +15,13 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Builde public final fun getClient ()Laws/sdk/kotlin/services/s3/S3Client; public final fun getInterceptors ()Ljava/util/List; public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType; - public final fun getMultipartUploadThreshold ()Ljava/lang/Long; - public final fun getPartSize ()Ljava/lang/Long; + public final fun getMultipartUploadThreshold ()J + public final fun getTargePartSize ()J public final fun setClient (Laws/sdk/kotlin/services/s3/S3Client;)V public final fun setInterceptors (Ljava/util/List;)V public final fun setMultipartDownloadType (Laws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType;)V - public final fun setMultipartUploadThreshold (Ljava/lang/Long;)V - public final fun setPartSize (Ljava/lang/Long;)V + public final fun setMultipartUploadThreshold (J)V + public final fun setTargePartSize (J)V } public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Companion { @@ -158,6 +159,7 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$Builder { public fun ()V + public final fun build ()Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest; public final fun getAcl ()Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl; public final fun getBody ()Laws/smithy/kotlin/runtime/content/ByteStream; public final fun getBucket ()Ljava/lang/String; diff --git a/hll/s3-transfer-manager/build.gradle.kts b/hll/s3-transfer-manager/build.gradle.kts index 0125d442603..dc3631f407d 100644 --- a/hll/s3-transfer-manager/build.gradle.kts +++ b/hll/s3-transfer-manager/build.gradle.kts @@ -12,7 +12,13 @@ kotlin { commonMain { dependencies { implementation(project(":aws-runtime:aws-http")) - implementation(project(":services:s3")) // TODO: Hardcode an S3 Client version to avoid breakages + implementation(libs.s3) + } + } + commonTest { + dependencies { + implementation(libs.smithy.kotlin.test.jvm) + implementation(libs.smithy.kotlin.testing.jvm) } } } diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt index dab45422913..05e846f69d9 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt @@ -8,13 +8,30 @@ package aws.sdk.kotlin.hll.s3transfermanager import aws.sdk.kotlin.hll.s3transfermanager.model.MultiPartDownloadType import aws.sdk.kotlin.hll.s3transfermanager.model.Part import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest -import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest.Companion.toCreateMultiPartUploadRequest -import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest.Companion.toPutObjectRequest import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileResponse +import aws.sdk.kotlin.hll.s3transfermanager.utils.buildCompleteMultipartUploadRequest +import aws.sdk.kotlin.hll.s3transfermanager.utils.buildUploadPartRequest +import aws.sdk.kotlin.hll.s3transfermanager.utils.ceilDiv +import aws.sdk.kotlin.hll.s3transfermanager.utils.getNextPart +import aws.sdk.kotlin.hll.s3transfermanager.utils.resolvePartSize +import aws.sdk.kotlin.hll.s3transfermanager.utils.toCreateMultiPartUploadRequest +import aws.sdk.kotlin.hll.s3transfermanager.utils.toPutObjectRequest +import aws.sdk.kotlin.hll.s3transfermanager.utils.toUploadFileResponse import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.abortMultipartUpload +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadResponse +import aws.sdk.kotlin.services.s3.model.CompletedPart import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadResponse import aws.sdk.kotlin.services.s3.model.PutObjectRequest +import aws.sdk.kotlin.services.s3.model.PutObjectResponse +import aws.sdk.kotlin.services.s3.model.UploadPartRequest +import aws.sdk.kotlin.services.s3.model.UploadPartResponse import aws.sdk.kotlin.services.s3.withConfig +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.io.SdkBuffer +import aws.smithy.kotlin.runtime.telemetry.logging.logger import kotlinx.coroutines.Deferred import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -24,11 +41,13 @@ import kotlinx.coroutines.coroutineScope */ public class S3TransferManager private constructor( public val client: S3Client, - public val partSize: Long, + public val targePartSize: Long, public val multipartUploadThreshold: Long, public val multipartDownloadType: MultiPartDownloadType, public val interceptors: MutableList, ) { + internal var context: TransferContext = TransferContext() + public companion object { public operator fun invoke(block: Builder.() -> Unit): S3TransferManager = Builder().apply(block).build() @@ -36,100 +55,220 @@ public class S3TransferManager private constructor( public class Builder { public var client: S3Client? = null - public var partSize: Long? = null - public var multipartUploadThreshold: Long? = null - public var multipartDownloadType: MultiPartDownloadType? = null - public var interceptors: MutableList? = null + public var targePartSize: Long = 8_000_000 + public var multipartUploadThreshold: Long = 16_000_000L + public var multipartDownloadType: MultiPartDownloadType = Part + public var interceptors: MutableList = mutableListOf() internal fun build(): S3TransferManager = S3TransferManager( client = client?.withConfig { interceptors += BusinessMetricInterceptor } ?: error("client must be set"), - partSize = partSize ?: 8_000_000L, - multipartUploadThreshold = multipartUploadThreshold ?: 16_000_000L, - multipartDownloadType = multipartDownloadType ?: Part, - interceptors = interceptors ?: mutableListOf(), + targePartSize = targePartSize, + multipartUploadThreshold = multipartUploadThreshold, + multipartDownloadType = multipartDownloadType, + interceptors = interceptors, ) } - private var context: TransferContext = TransferContext() + /** + * Executes a sequence of operations around a hook. + * + * The execution flow is as follows: + * 1. Runs all interceptors scheduled to execute **before** the hook. + * 2. Executes the main hook logic. + * 3. Runs all interceptors scheduled to execute **after** the hook. + */ + private suspend fun operationHook(hook: TransferHook, block: suspend () -> Any) { + when (hook) { + is TransferInitiated -> { + interceptors.forEach { it.readBeforeTransferInitiated(context) } + interceptors.forEach { context = it.modifyBeforeTransferInitiated(context) } + block.invoke() + interceptors.forEach { it.readAfterTransferInitiated(context) } + interceptors.forEach { context = it.modifyAfterTransferInitiated(context) } + } + is BytesTransferred -> { + interceptors.forEach { it.readBeforeBytesTransferred(context) } + interceptors.forEach { context = it.modifyBeforeBytesTransferred(context) } + block.invoke() + interceptors.forEach { it.readAfterBytesTransferred(context) } + interceptors.forEach { context = it.modifyAfterBytesTransferred(context) } + } + is FileTransferred -> { + interceptors.forEach { it.readBeforeFileTransferred(context) } + interceptors.forEach { context = it.modifyBeforeFileTransferred(context) } + block.invoke() + interceptors.forEach { it.readAfterFileTransferred(context) } + interceptors.forEach { context = it.modifyAfterFileTransferred(context) } + } + is TransferCompleted -> { + interceptors.forEach { it.readBeforeTransferCompleted(context) } + interceptors.forEach { context = it.modifyBeforeTransferCompleted(context) } + block.invoke() + interceptors.forEach { it.readAfterTransferCompleted(context) } + interceptors.forEach { context = it.modifyAfterTransferCompleted(context) } + } + else -> error("TransferHook not implemented: ${hook::class.simpleName}") + } + } - // TODO: Try to find parts of the code you can commonize /** - * TODO + * Uploads a byte stream to Amazon S3, automatically using multipart uploads + * for large objects as needed. + * + * This function handles the complexity of splitting the data into parts, + * uploading each part, and completing the multipart upload. For object smaller than [multipartUploadThreshold], + * a standard single-part upload is performed automatically. + * + * If the specified [targePartSize] for multipart uploads is too small to allow + * all parts to fit within S3's limit of 10,000 parts, the part size will be + * automatically increased so that exactly 10,000 parts are uploaded. */ public suspend fun uploadFile(uploadFileRequest: UploadFileRequest): Deferred = coroutineScope { - async { - val multiPartUpload = uploadFileRequest.contentLength >= multipartUploadThreshold + val multiPartUpload = uploadFileRequest.contentLength >= multipartUploadThreshold + val uploadedParts = mutableListOf() + var mpuUploadId = "null" + + val logger = coroutineContext.logger() + /* + Handles transfer initiated hook + */ + suspend fun transferInitiated(multiPartUpload: Boolean) { + context.transferredBytes = 0L + context.transferableBytes = uploadFileRequest.contentLength context.request = if (multiPartUpload) { uploadFileRequest.toCreateMultiPartUploadRequest() } else { uploadFileRequest.toPutObjectRequest() } - operationHook(TransferInitiated) { - context.transferredBytes = 0L - context.transferableBytes = uploadFileRequest.contentLength - if (multiPartUpload) { context.response = client.createMultipartUpload(context.request as CreateMultipartUploadRequest) + mpuUploadId = (context.response as CreateMultipartUploadResponse).uploadId ?: throw Exception("Missing upload id in create multipart upload response") } } + } - operationHook(BytesTransferred) { - if (multiPartUpload) { - // TODO: MPU logic - // TODO: Update bytes transferred - } else { + /* + Handles bytes transferred hook + */ + suspend fun transferBytes(multiPartUpload: Boolean) { + if (multiPartUpload) { + try { + val partSize = resolvePartSize(uploadFileRequest, this@S3TransferManager, logger) + val numberOfParts = ceilDiv(uploadFileRequest.contentLength, partSize) + val partSource = when (uploadFileRequest.body) { + is ByteStream.Buffer -> uploadFileRequest.body.bytes() + is ByteStream.ChannelStream -> uploadFileRequest.body.readFrom() + is ByteStream.SourceStream -> uploadFileRequest.body.readFrom() + else -> error("Unhandled body type: ${uploadFileRequest.body?.let { it::class.simpleName } ?: "null"}") + } + val partBuffer = SdkBuffer() + var currentPartNumber = 1L + + while (context.transferredBytes!! < context.transferableBytes!!) { + partBuffer.getNextPart(partSource, partSize, this@S3TransferManager) + if (currentPartNumber != numberOfParts) { + check(partBuffer.size == partSize) { + "Part #$currentPartNumber size mismatch detected. Expected $partSize, actual: ${partBuffer.size}" + } + } + + context.request = + buildUploadPartRequest( + uploadFileRequest, + partBuffer, + currentPartNumber, + mpuUploadId, + ) + + operationHook(BytesTransferred) { + context.response = client.uploadPart(context.request as UploadPartRequest) + context.transferredBytes = context.transferredBytes!! + partSize + } + + uploadedParts += CompletedPart { + partNumber = currentPartNumber.toInt() + eTag = (context.response as UploadPartResponse).eTag + } + currentPartNumber += 1 + } + + check(uploadedParts.size == numberOfParts.toInt()) { + "The number of uploaded parts does not match the expected count. Expected $numberOfParts, actual: ${uploadedParts.size}" + } + } catch (uploadPartThrowable: Throwable) { + try { + client.abortMultipartUpload { + bucket = uploadFileRequest.bucket + expectedBucketOwner = uploadFileRequest.expectedBucketOwner + key = uploadFileRequest.key + requestPayer = uploadFileRequest.requestPayer + uploadId = mpuUploadId + } + throw Exception("Multipart upload failed (ID: $mpuUploadId). One or more parts could not be uploaded", uploadPartThrowable) + } catch (abortThrowable: Throwable) { + throw Exception("Multipart upload failed (ID: $mpuUploadId). Unable to abort multipart upload.", abortThrowable) + } + } + } else { + operationHook(BytesTransferred) { context.response = client.putObject(context.request as PutObjectRequest) context.transferredBytes = context.transferableBytes } } + } + /* + Handles transfer completed hook + */ + suspend fun transferComplete(multiPartUpload: Boolean) { + if (multiPartUpload) { + context.request = buildCompleteMultipartUploadRequest(uploadFileRequest, mpuUploadId, uploadedParts) + } operationHook(TransferCompleted) { if (multiPartUpload) { - // TODO: MPU logic - // TODO: Update bytes transferred? + try { + context.response = client.completeMultipartUpload(context.request as CompleteMultipartUploadRequest) + } catch (t: Throwable) { + throw Exception("Unable to complete multipart upload with ID: $mpuUploadId", t) + } } } - - UploadFileResponse.fromS3Response(context.response) } - } - private suspend fun operationHook(hook: TransferHook, block: suspend () -> Any) { - interceptors.forEach { interceptor -> - when (hook) { - is TransferInitiated -> { - interceptor.readBeforeTransferInitiated(context) - context = interceptor.modifyBeforeTransferInitiated(context) - block.invoke() - interceptor.readAfterTransferInitiated(context) - context = interceptor.modifyAfterTransferInitiated(context) - } - is BytesTransferred -> { - interceptor.readBeforeBytesTransferred(context) - context = interceptor.modifyBeforeBytesTransferred(context) - block.invoke() - interceptor.readAfterBytesTransferred(context) - context = interceptor.modifyAfterBytesTransferred(context) - } - is FileTransferred -> { - interceptor.readBeforeFileTransferred(context) - context = interceptor.modifyBeforeFileTransferred(context) - block.invoke() - interceptor.readAfterFileTransferred(context) - context = interceptor.modifyAfterFileTransferred(context) - } - is TransferCompleted -> { - interceptor.readBeforeTransferCompleted(context) - context = interceptor.modifyBeforeTransferCompleted(context) - block.invoke() - interceptor.readAfterTransferCompleted(context) - context = interceptor.modifyAfterTransferCompleted(context) - } - else -> error("TransferHook not implemented: ${hook::class.simpleName}") + async { + checkNotNull(uploadFileRequest.body?.contentLength) { + "UploadFileRequest.body.contentLength must be set" + } + check(uploadFileRequest.body.contentLength == uploadFileRequest.contentLength) { + "contentLength mismatch. uploadFileRequest: ${uploadFileRequest.contentLength}, uploadFileRequest.body.contentLength: ${uploadFileRequest.body.contentLength}" + } + + transferInitiated(multiPartUpload) + transferBytes(multiPartUpload) + transferComplete(multiPartUpload) + + when (context.response) { + is PutObjectResponse -> (context.response as PutObjectResponse).toUploadFileResponse() + is CompleteMultipartUploadResponse -> (context.response as CompleteMultipartUploadResponse).toUploadFileResponse() + else -> error("Unexpected response: ${context.response?.let { it::class.simpleName } ?: "null"}") } } } + + /** + * Uploads a byte stream to Amazon S3, automatically using multipart uploads + * for large objects as needed. + * + * This function handles the complexity of splitting the data into parts, + * uploading each part, and completing the multipart upload. For object smaller than [multipartUploadThreshold], + * a standard single-part upload is performed automatically. + * + * If the specified [targePartSize] for multipart uploads is too small to allow + * all parts to fit within S3's limit of 10,000 parts, the part size will be + * automatically increased so that exactly 10,000 parts are uploaded. + */ + public suspend inline fun uploadFile(crossinline block: UploadFileRequest.Builder.() -> Unit): Deferred = uploadFile(UploadFileRequest.Builder().apply(block).build()) } diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt index 566bde2d7f8..b6403801db9 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt @@ -5,10 +5,8 @@ package aws.sdk.kotlin.hll.s3transfermanager -// TODO: Add PUBLIC documentation to each thing in the context -/** - * TODO - */ +// TODO: KDocs + public data class TransferContext( // Req/Resp var request: Any? = null, @@ -25,10 +23,6 @@ public data class TransferContext( var transferredFiles: Long? = null, ) -// TODO: Add PUBLIC documentation to each hook -/** - * TODO - */ public interface TransferInterceptor { // Transfer initialization hooks public fun readBeforeTransferInitiated(context: TransferContext) {} @@ -55,9 +49,6 @@ public interface TransferInterceptor { public fun modifyAfterTransferCompleted(context: TransferContext): TransferContext = context } -/** - * TODO - */ internal interface TransferHook internal object TransferInitiated : TransferHook internal object BytesTransferred : TransferHook diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType.kt index a21c8fd8264..3eeea64246d 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType.kt @@ -5,17 +5,8 @@ package aws.sdk.kotlin.hll.s3transfermanager.model -/** - * TODO - */ -public sealed interface MultiPartDownloadType +// TODO: KDocs -/** - * TODO - */ +public sealed interface MultiPartDownloadType public object Range : MultiPartDownloadType - -/** - * TODO - */ public object Part : MultiPartDownloadType diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFile.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFile.kt deleted file mode 100644 index 7be81bcb995..00000000000 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFile.kt +++ /dev/null @@ -1,339 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package aws.sdk.kotlin.hll.s3transfermanager.model - -import aws.sdk.kotlin.services.s3.model.ChecksumAlgorithm -import aws.sdk.kotlin.services.s3.model.ChecksumType -import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadResponse -import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest -import aws.sdk.kotlin.services.s3.model.ObjectCannedAcl -import aws.sdk.kotlin.services.s3.model.ObjectLockLegalHoldStatus -import aws.sdk.kotlin.services.s3.model.ObjectLockMode -import aws.sdk.kotlin.services.s3.model.PutObjectRequest -import aws.sdk.kotlin.services.s3.model.PutObjectResponse -import aws.sdk.kotlin.services.s3.model.RequestCharged -import aws.sdk.kotlin.services.s3.model.RequestPayer -import aws.sdk.kotlin.services.s3.model.ServerSideEncryption -import aws.sdk.kotlin.services.s3.model.StorageClass -import aws.smithy.kotlin.runtime.content.ByteStream -import aws.smithy.kotlin.runtime.time.Instant -import kotlin.Boolean - -// TODO: Add documentation to each thing ? -/** - * TODO - */ -public class UploadFileRequest private constructor( - public val acl: ObjectCannedAcl?, - public val body: ByteStream?, - public val bucket: String?, - public val bucketKeyEnabled: Boolean?, - public val cacheControl: String?, - public val checksumAlgorithm: ChecksumAlgorithm?, - public val checksumCrc32: String?, - public val checksumCrc32C: String?, - public val checksumCrc64Nvme: String?, - public val checksumSha1: String?, - public val checksumSha256: String?, - public val contentDisposition: String?, - public val contentEncoding: String?, - public val contentLanguage: String?, - public val contentLength: Long, - public val contentType: String?, - public val expectedBucketOwner: String?, - public val expires: Instant?, // TODO: Is this the right instant? - public val grantFullControl: String?, - public val grantRead: String?, - public val grantReadAcp: String?, - public val grantWriteAcp: String?, - public val ifMatch: String?, - public val ifNoneMatch: String?, - public val key: String?, - public val metadata: Map?, - public val objectLockLegalHoldStatus: ObjectLockLegalHoldStatus?, - public val objectLockMode: ObjectLockMode?, - public val objectLockRetainUntilDate: Instant?, - public val requestPayer: RequestPayer?, - public val serverSideEncryption: ServerSideEncryption?, - public val source: String?, - public val sseCustomerAlgorithm: String?, - public val sseCustomerKey: String?, - public val sseCustomerKeyMd5: String?, - public val ssekmsEncryptionContext: String?, - public val ssekmsKeyId: String?, - public val storageClass: StorageClass?, - public val tagging: String?, - public val websiteRedirectLocation: String?, -) { - public companion object { - public operator fun invoke(block: Builder.() -> Unit): UploadFileRequest = - Builder().apply(block).build() - - // TODO: Checkout the null values CAREFULLY - internal fun UploadFileRequest.toPutObjectRequest(): PutObjectRequest = - PutObjectRequest { - acl = this@toPutObjectRequest.acl - body = this@toPutObjectRequest.body - bucket = this@toPutObjectRequest.bucket - bucketKeyEnabled = this@toPutObjectRequest.bucketKeyEnabled - cacheControl = this@toPutObjectRequest.cacheControl - checksumAlgorithm = this@toPutObjectRequest.checksumAlgorithm - checksumCrc32 = this@toPutObjectRequest.checksumCrc32 - checksumCrc32C = this@toPutObjectRequest.checksumCrc32C - checksumCrc64Nvme = this@toPutObjectRequest.checksumCrc64Nvme - checksumSha1 = this@toPutObjectRequest.checksumSha1 - checksumSha256 = this@toPutObjectRequest.checksumSha256 - contentDisposition = this@toPutObjectRequest.contentDisposition - contentEncoding = this@toPutObjectRequest.contentEncoding - contentLanguage = this@toPutObjectRequest.contentLanguage - contentLength = this@toPutObjectRequest.contentLength - contentMd5 = null - contentType = this@toPutObjectRequest.contentType - expectedBucketOwner = this@toPutObjectRequest.expectedBucketOwner - expires = this@toPutObjectRequest.expires - grantFullControl = this@toPutObjectRequest.grantFullControl - grantRead = this@toPutObjectRequest.grantRead - grantReadAcp = this@toPutObjectRequest.grantReadAcp - grantWriteAcp = this@toPutObjectRequest.grantWriteAcp - ifMatch = this@toPutObjectRequest.ifMatch - ifNoneMatch = this@toPutObjectRequest.ifNoneMatch - key = this@toPutObjectRequest.key - metadata = this@toPutObjectRequest.metadata - objectLockLegalHoldStatus = this@toPutObjectRequest.objectLockLegalHoldStatus - objectLockMode = this@toPutObjectRequest.objectLockMode - objectLockRetainUntilDate = this@toPutObjectRequest.objectLockRetainUntilDate - requestPayer = this@toPutObjectRequest.requestPayer - serverSideEncryption = this@toPutObjectRequest.serverSideEncryption - sseCustomerAlgorithm = this@toPutObjectRequest.sseCustomerAlgorithm - sseCustomerKey = this@toPutObjectRequest.sseCustomerKey - sseCustomerKeyMd5 = this@toPutObjectRequest.sseCustomerKeyMd5 - ssekmsEncryptionContext = this@toPutObjectRequest.ssekmsEncryptionContext - ssekmsKeyId = this@toPutObjectRequest.ssekmsKeyId - storageClass = this@toPutObjectRequest.storageClass - tagging = this@toPutObjectRequest.tagging - websiteRedirectLocation = this@toPutObjectRequest.websiteRedirectLocation - writeOffsetBytes = null - } - - // TODO: Checkout the null values CAREFULLY - internal fun UploadFileRequest.toCreateMultiPartUploadRequest(): CreateMultipartUploadRequest = - CreateMultipartUploadRequest { - acl = this@toCreateMultiPartUploadRequest.acl - bucket = this@toCreateMultiPartUploadRequest.bucket - bucketKeyEnabled = this@toCreateMultiPartUploadRequest.bucketKeyEnabled - cacheControl = this@toCreateMultiPartUploadRequest.cacheControl - checksumAlgorithm = this@toCreateMultiPartUploadRequest.checksumAlgorithm - checksumType = null - contentDisposition = this@toCreateMultiPartUploadRequest.contentDisposition - contentEncoding = this@toCreateMultiPartUploadRequest.contentEncoding - contentLanguage = this@toCreateMultiPartUploadRequest.contentLanguage - contentType = this@toCreateMultiPartUploadRequest.contentType - expectedBucketOwner = this@toCreateMultiPartUploadRequest.expectedBucketOwner - expires = this@toCreateMultiPartUploadRequest.expires - grantFullControl = this@toCreateMultiPartUploadRequest.grantFullControl - grantRead = this@toCreateMultiPartUploadRequest.grantRead - grantReadAcp = this@toCreateMultiPartUploadRequest.grantReadAcp - grantWriteAcp = this@toCreateMultiPartUploadRequest.grantWriteAcp - key = this@toCreateMultiPartUploadRequest.key - metadata = this@toCreateMultiPartUploadRequest.metadata - objectLockLegalHoldStatus = this@toCreateMultiPartUploadRequest.objectLockLegalHoldStatus - objectLockMode = this@toCreateMultiPartUploadRequest.objectLockMode - objectLockRetainUntilDate = this@toCreateMultiPartUploadRequest.objectLockRetainUntilDate - requestPayer = this@toCreateMultiPartUploadRequest.requestPayer - serverSideEncryption = this@toCreateMultiPartUploadRequest.serverSideEncryption - sseCustomerAlgorithm = this@toCreateMultiPartUploadRequest.sseCustomerAlgorithm - sseCustomerKey = this@toCreateMultiPartUploadRequest.sseCustomerKey - sseCustomerKeyMd5 = this@toCreateMultiPartUploadRequest.sseCustomerKeyMd5 - ssekmsEncryptionContext = this@toCreateMultiPartUploadRequest.ssekmsEncryptionContext - ssekmsKeyId = this@toCreateMultiPartUploadRequest.ssekmsKeyId - storageClass = this@toCreateMultiPartUploadRequest.storageClass - tagging = this@toCreateMultiPartUploadRequest.tagging - websiteRedirectLocation = this@toCreateMultiPartUploadRequest.websiteRedirectLocation - } - } - - public class Builder { - public var acl: ObjectCannedAcl? = null - public var body: ByteStream? = null - public var bucket: String? = null - public var bucketKeyEnabled: Boolean? = null - public var cacheControl: String? = null - public var checksumAlgorithm: ChecksumAlgorithm? = null - public var checksumCrc32: String? = null - public var checksumCrc32C: String? = null - public var checksumCrc64Nvme: String? = null - public var checksumSha1: String? = null - public var checksumSha256: String? = null - public var contentDisposition: String? = null - public var contentEncoding: String? = null - public var contentLanguage: String? = null - public var contentLength: Long? = null - public var contentType: String? = null - public var expectedBucketOwner: String? = null - public var expires: Instant? = null // TODO: Is this the right instant? Write some tests to see if conversions work properly? - public var grantFullControl: String? = null - public var grantRead: String? = null - public var grantReadAcp: String? = null - public var grantWriteAcp: String? = null - public var ifMatch: String? = null - public var ifNoneMatch: String? = null - public var key: String? = null - public var metadata: Map? = null - public var objectLockLegalHoldStatus: ObjectLockLegalHoldStatus? = null - public var objectLockMode: ObjectLockMode? = null - public var objectLockRetainUntilDate: Instant? = null - public var requestPayer: RequestPayer? = null - public var source: String? = null - public var serverSideEncryption: ServerSideEncryption? = null - public var sseCustomerAlgorithm: String? = null - public var sseCustomerKey: String? = null - public var sseCustomerKeyMd5: String? = null - public var ssekmsEncryptionContext: String? = null - public var ssekmsKeyId: String? = null - public var storageClass: StorageClass? = null - public var tagging: String? = null - public var websiteRedirectLocation: String? = null - - internal fun build(): UploadFileRequest = - UploadFileRequest( - acl, - body, - bucket, - bucketKeyEnabled, - cacheControl, - checksumAlgorithm, - checksumCrc32, - checksumCrc32C, - checksumCrc64Nvme, - checksumSha1, - checksumSha256, - contentDisposition, - contentEncoding, - contentLanguage, - contentLength ?: error("contentLength must be set"), - contentType, - expectedBucketOwner, - expires, - grantFullControl, - grantRead, - grantReadAcp, - grantWriteAcp, - ifMatch, - ifNoneMatch, - key, - metadata, - objectLockLegalHoldStatus, - objectLockMode, - objectLockRetainUntilDate, - requestPayer, - serverSideEncryption, - source, - sseCustomerAlgorithm, - sseCustomerKey, - sseCustomerKeyMd5, - ssekmsEncryptionContext, - ssekmsKeyId, - storageClass, - tagging, - websiteRedirectLocation, - ) - } -} - -/** - * TODO - */ -public class UploadFileResponse( - public val bucketKeyEnabled: Boolean?, - public val checksumCrc32: String?, - public val checksumCrc32C: String?, - public val checksumCrc64Nvme: String?, - public val checksumSha1: String?, - public val checksumSha256: String?, - public val checksumType: ChecksumType?, - public val eTag: String?, - public val expiration: String?, - public val requestCharged: RequestCharged?, - public val serverSideEncryption: ServerSideEncryption?, - public val ssekmsKeyId: String?, - public val versionId: String?, -) { - public companion object { - public operator fun invoke(block: Builder.() -> Unit): UploadFileResponse = - Builder().apply(block).build() - - // TODO: Double check the conversion - internal fun fromS3Response(response: Any?): UploadFileResponse = - when (response) { - is PutObjectResponse -> - UploadFileResponse { - bucketKeyEnabled = response.bucketKeyEnabled - checksumCrc32 = response.checksumCrc32 - checksumCrc32C = response.checksumCrc32C - checksumCrc64Nvme = response.checksumCrc64Nvme - checksumSha1 = response.checksumSha1 - checksumSha256 = response.checksumSha256 - checksumType = response.checksumType - eTag = response.eTag - expiration = response.expiration - requestCharged = response.requestCharged - serverSideEncryption = response.serverSideEncryption - ssekmsKeyId = response.ssekmsKeyId - versionId = response.versionId - } - is CompleteMultipartUploadResponse -> - UploadFileResponse { - bucketKeyEnabled = response.bucketKeyEnabled - checksumCrc32 = response.checksumCrc32 - checksumCrc32C = response.checksumCrc32C - checksumCrc64Nvme = response.checksumCrc64Nvme - checksumSha1 = response.checksumSha1 - checksumSha256 = response.checksumSha256 - checksumType = response.checksumType - eTag = response.eTag - expiration = response.expiration - requestCharged = response.requestCharged - serverSideEncryption = response.serverSideEncryption - ssekmsKeyId = response.ssekmsKeyId - versionId = response.versionId - } - else -> error("Response must be a PutObjectResponse or CompleteMultipartUploadResponse") - } - } - - public class Builder { - public var bucketKeyEnabled: Boolean? = null - public var checksumCrc32: String? = null - public var checksumCrc32C: String? = null - public var checksumCrc64Nvme: String? = null - public var checksumSha1: String? = null - public var checksumSha256: String? = null - public var checksumType: ChecksumType? = null - public var eTag: String? = null - public var expiration: String? = null - public var requestCharged: RequestCharged? = null - public var serverSideEncryption: ServerSideEncryption? = null - public var ssekmsKeyId: String? = null - public var versionId: String? = null - - internal fun build(): UploadFileResponse = - UploadFileResponse( - bucketKeyEnabled, - checksumCrc32, - checksumCrc32C, - checksumCrc64Nvme, - checksumSha1, - checksumSha256, - checksumType, - eTag, - expiration, - requestCharged, - serverSideEncryption, - ssekmsKeyId, - versionId, - ) - } -} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest.kt new file mode 100644 index 00000000000..742387a9202 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest.kt @@ -0,0 +1,151 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.model + +import aws.sdk.kotlin.services.s3.model.ChecksumAlgorithm +import aws.sdk.kotlin.services.s3.model.ObjectCannedAcl +import aws.sdk.kotlin.services.s3.model.ObjectLockLegalHoldStatus +import aws.sdk.kotlin.services.s3.model.ObjectLockMode +import aws.sdk.kotlin.services.s3.model.RequestPayer +import aws.sdk.kotlin.services.s3.model.ServerSideEncryption +import aws.sdk.kotlin.services.s3.model.StorageClass +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.time.Instant + +public class UploadFileRequest private constructor( + public val acl: ObjectCannedAcl?, + public val body: ByteStream?, + public val bucket: String?, + public val bucketKeyEnabled: Boolean?, + public val cacheControl: String?, + public val checksumAlgorithm: ChecksumAlgorithm?, + public val checksumCrc32: String?, + public val checksumCrc32C: String?, + public val checksumCrc64Nvme: String?, + public val checksumSha1: String?, + public val checksumSha256: String?, + public val contentDisposition: String?, + public val contentEncoding: String?, + public val contentLanguage: String?, + public val contentLength: Long, + public val contentType: String?, + public val expectedBucketOwner: String?, + public val expires: Instant?, + public val grantFullControl: String?, + public val grantRead: String?, + public val grantReadAcp: String?, + public val grantWriteAcp: String?, + public val ifMatch: String?, + public val ifNoneMatch: String?, + public val key: String?, + public val metadata: Map?, + public val objectLockLegalHoldStatus: ObjectLockLegalHoldStatus?, + public val objectLockMode: ObjectLockMode?, + public val objectLockRetainUntilDate: Instant?, + public val requestPayer: RequestPayer?, + public val serverSideEncryption: ServerSideEncryption?, + public val source: String?, + public val sseCustomerAlgorithm: String?, + public val sseCustomerKey: String?, + public val sseCustomerKeyMd5: String?, + public val ssekmsEncryptionContext: String?, + public val ssekmsKeyId: String?, + public val storageClass: StorageClass?, + public val tagging: String?, + public val websiteRedirectLocation: String?, +) { + public companion object { + public operator fun invoke(block: Builder.() -> Unit): UploadFileRequest = + Builder().apply(block).build() + } + + public class Builder { + public var acl: ObjectCannedAcl? = null + public var body: ByteStream? = null + public var bucket: String? = null + public var bucketKeyEnabled: Boolean? = null + public var cacheControl: String? = null + public var checksumAlgorithm: ChecksumAlgorithm? = null + public var checksumCrc32: String? = null + public var checksumCrc32C: String? = null + public var checksumCrc64Nvme: String? = null + public var checksumSha1: String? = null + public var checksumSha256: String? = null + public var contentDisposition: String? = null + public var contentEncoding: String? = null + public var contentLanguage: String? = null + public var contentLength: Long? = null + public var contentType: String? = null + public var expectedBucketOwner: String? = null + public var expires: Instant? = null + public var grantFullControl: String? = null + public var grantRead: String? = null + public var grantReadAcp: String? = null + public var grantWriteAcp: String? = null + public var ifMatch: String? = null + public var ifNoneMatch: String? = null + public var key: String? = null + public var metadata: Map? = null + public var objectLockLegalHoldStatus: ObjectLockLegalHoldStatus? = null + public var objectLockMode: ObjectLockMode? = null + public var objectLockRetainUntilDate: Instant? = null + public var requestPayer: RequestPayer? = null + public var source: String? = null + public var serverSideEncryption: ServerSideEncryption? = null + public var sseCustomerAlgorithm: String? = null + public var sseCustomerKey: String? = null + public var sseCustomerKeyMd5: String? = null + public var ssekmsEncryptionContext: String? = null + public var ssekmsKeyId: String? = null + public var storageClass: StorageClass? = null + public var tagging: String? = null + public var websiteRedirectLocation: String? = null + + public fun build(): UploadFileRequest = + UploadFileRequest( + acl, + body, + bucket, + bucketKeyEnabled, + cacheControl, + checksumAlgorithm, + checksumCrc32, + checksumCrc32C, + checksumCrc64Nvme, + checksumSha1, + checksumSha256, + contentDisposition, + contentEncoding, + contentLanguage, + contentLength ?: error("contentLength must be set"), + contentType, + expectedBucketOwner, + expires, + grantFullControl, + grantRead, + grantReadAcp, + grantWriteAcp, + ifMatch, + ifNoneMatch, + key, + metadata, + objectLockLegalHoldStatus, + objectLockMode, + objectLockRetainUntilDate, + requestPayer, + serverSideEncryption, + source, + sseCustomerAlgorithm, + sseCustomerKey, + sseCustomerKeyMd5, + ssekmsEncryptionContext, + ssekmsKeyId, + storageClass, + tagging, + websiteRedirectLocation, + ) + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse.kt new file mode 100644 index 00000000000..0b7c60bce86 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse.kt @@ -0,0 +1,64 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.model + +import aws.sdk.kotlin.services.s3.model.ChecksumType +import aws.sdk.kotlin.services.s3.model.RequestCharged +import aws.sdk.kotlin.services.s3.model.ServerSideEncryption + +public class UploadFileResponse( + public val bucketKeyEnabled: Boolean?, + public val checksumCrc32: String?, + public val checksumCrc32C: String?, + public val checksumCrc64Nvme: String?, + public val checksumSha1: String?, + public val checksumSha256: String?, + public val checksumType: ChecksumType?, + public val eTag: String?, + public val expiration: String?, + public val requestCharged: RequestCharged?, + public val serverSideEncryption: ServerSideEncryption?, + public val ssekmsKeyId: String?, + public val versionId: String?, +) { + public companion object { + public operator fun invoke(block: Builder.() -> Unit): UploadFileResponse = + Builder().apply(block).build() + } + + public class Builder { + public var bucketKeyEnabled: Boolean? = null + public var checksumCrc32: String? = null + public var checksumCrc32C: String? = null + public var checksumCrc64Nvme: String? = null + public var checksumSha1: String? = null + public var checksumSha256: String? = null + public var checksumType: ChecksumType? = null + public var eTag: String? = null + public var expiration: String? = null + public var requestCharged: RequestCharged? = null + public var serverSideEncryption: ServerSideEncryption? = null + public var ssekmsKeyId: String? = null + public var versionId: String? = null + + internal fun build(): UploadFileResponse = + UploadFileResponse( + bucketKeyEnabled, + checksumCrc32, + checksumCrc32C, + checksumCrc64Nvme, + checksumSha1, + checksumSha256, + checksumType, + eTag, + expiration, + requestCharged, + serverSideEncryption, + ssekmsKeyId, + versionId, + ) + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Conversions.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Conversions.kt new file mode 100644 index 00000000000..7b0984d90db --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Conversions.kt @@ -0,0 +1,124 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.utils + +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileResponse +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadResponse +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.PutObjectRequest +import aws.sdk.kotlin.services.s3.model.PutObjectResponse + +internal fun PutObjectResponse.toUploadFileResponse(): UploadFileResponse = + UploadFileResponse { + bucketKeyEnabled = this@toUploadFileResponse.bucketKeyEnabled + checksumCrc32 = this@toUploadFileResponse.checksumCrc32 + checksumCrc32C = this@toUploadFileResponse.checksumCrc32C + checksumCrc64Nvme = this@toUploadFileResponse.checksumCrc64Nvme + checksumSha1 = this@toUploadFileResponse.checksumSha1 + checksumSha256 = this@toUploadFileResponse.checksumSha256 + checksumType = this@toUploadFileResponse.checksumType + eTag = this@toUploadFileResponse.eTag + expiration = this@toUploadFileResponse.expiration + requestCharged = this@toUploadFileResponse.requestCharged + serverSideEncryption = this@toUploadFileResponse.serverSideEncryption + ssekmsKeyId = this@toUploadFileResponse.ssekmsKeyId + versionId = this@toUploadFileResponse.versionId + } + +internal fun CompleteMultipartUploadResponse.toUploadFileResponse(): UploadFileResponse = + UploadFileResponse { + bucketKeyEnabled = this@toUploadFileResponse.bucketKeyEnabled + checksumCrc32 = this@toUploadFileResponse.checksumCrc32 + checksumCrc32C = this@toUploadFileResponse.checksumCrc32C + checksumCrc64Nvme = this@toUploadFileResponse.checksumCrc64Nvme + checksumSha1 = this@toUploadFileResponse.checksumSha1 + checksumSha256 = this@toUploadFileResponse.checksumSha256 + checksumType = this@toUploadFileResponse.checksumType + eTag = this@toUploadFileResponse.eTag + expiration = this@toUploadFileResponse.expiration + requestCharged = this@toUploadFileResponse.requestCharged + serverSideEncryption = this@toUploadFileResponse.serverSideEncryption + ssekmsKeyId = this@toUploadFileResponse.ssekmsKeyId + versionId = this@toUploadFileResponse.versionId + } + +internal fun UploadFileRequest.toPutObjectRequest(): PutObjectRequest = + PutObjectRequest { + acl = this@toPutObjectRequest.acl + body = this@toPutObjectRequest.body + bucket = this@toPutObjectRequest.bucket + bucketKeyEnabled = this@toPutObjectRequest.bucketKeyEnabled + cacheControl = this@toPutObjectRequest.cacheControl + checksumAlgorithm = this@toPutObjectRequest.checksumAlgorithm + checksumCrc32 = this@toPutObjectRequest.checksumCrc32 + checksumCrc32C = this@toPutObjectRequest.checksumCrc32C + checksumCrc64Nvme = this@toPutObjectRequest.checksumCrc64Nvme + checksumSha1 = this@toPutObjectRequest.checksumSha1 + checksumSha256 = this@toPutObjectRequest.checksumSha256 + contentDisposition = this@toPutObjectRequest.contentDisposition + contentEncoding = this@toPutObjectRequest.contentEncoding + contentLanguage = this@toPutObjectRequest.contentLanguage + contentLength = this@toPutObjectRequest.contentLength + contentType = this@toPutObjectRequest.contentType + expectedBucketOwner = this@toPutObjectRequest.expectedBucketOwner + expires = this@toPutObjectRequest.expires + grantFullControl = this@toPutObjectRequest.grantFullControl + grantRead = this@toPutObjectRequest.grantRead + grantReadAcp = this@toPutObjectRequest.grantReadAcp + grantWriteAcp = this@toPutObjectRequest.grantWriteAcp + ifMatch = this@toPutObjectRequest.ifMatch + ifNoneMatch = this@toPutObjectRequest.ifNoneMatch + key = this@toPutObjectRequest.key + metadata = this@toPutObjectRequest.metadata + objectLockLegalHoldStatus = this@toPutObjectRequest.objectLockLegalHoldStatus + objectLockMode = this@toPutObjectRequest.objectLockMode + objectLockRetainUntilDate = this@toPutObjectRequest.objectLockRetainUntilDate + requestPayer = this@toPutObjectRequest.requestPayer + serverSideEncryption = this@toPutObjectRequest.serverSideEncryption + sseCustomerAlgorithm = this@toPutObjectRequest.sseCustomerAlgorithm + sseCustomerKey = this@toPutObjectRequest.sseCustomerKey + sseCustomerKeyMd5 = this@toPutObjectRequest.sseCustomerKeyMd5 + ssekmsEncryptionContext = this@toPutObjectRequest.ssekmsEncryptionContext + ssekmsKeyId = this@toPutObjectRequest.ssekmsKeyId + storageClass = this@toPutObjectRequest.storageClass + tagging = this@toPutObjectRequest.tagging + websiteRedirectLocation = this@toPutObjectRequest.websiteRedirectLocation + } + +internal fun UploadFileRequest.toCreateMultiPartUploadRequest(): CreateMultipartUploadRequest = + CreateMultipartUploadRequest { + acl = this@toCreateMultiPartUploadRequest.acl + bucket = this@toCreateMultiPartUploadRequest.bucket + bucketKeyEnabled = this@toCreateMultiPartUploadRequest.bucketKeyEnabled + cacheControl = this@toCreateMultiPartUploadRequest.cacheControl + checksumAlgorithm = this@toCreateMultiPartUploadRequest.checksumAlgorithm + contentDisposition = this@toCreateMultiPartUploadRequest.contentDisposition + contentEncoding = this@toCreateMultiPartUploadRequest.contentEncoding + contentLanguage = this@toCreateMultiPartUploadRequest.contentLanguage + contentType = this@toCreateMultiPartUploadRequest.contentType + expectedBucketOwner = this@toCreateMultiPartUploadRequest.expectedBucketOwner + expires = this@toCreateMultiPartUploadRequest.expires + grantFullControl = this@toCreateMultiPartUploadRequest.grantFullControl + grantRead = this@toCreateMultiPartUploadRequest.grantRead + grantReadAcp = this@toCreateMultiPartUploadRequest.grantReadAcp + grantWriteAcp = this@toCreateMultiPartUploadRequest.grantWriteAcp + key = this@toCreateMultiPartUploadRequest.key + metadata = this@toCreateMultiPartUploadRequest.metadata + objectLockLegalHoldStatus = this@toCreateMultiPartUploadRequest.objectLockLegalHoldStatus + objectLockMode = this@toCreateMultiPartUploadRequest.objectLockMode + objectLockRetainUntilDate = this@toCreateMultiPartUploadRequest.objectLockRetainUntilDate + requestPayer = this@toCreateMultiPartUploadRequest.requestPayer + serverSideEncryption = this@toCreateMultiPartUploadRequest.serverSideEncryption + sseCustomerAlgorithm = this@toCreateMultiPartUploadRequest.sseCustomerAlgorithm + sseCustomerKey = this@toCreateMultiPartUploadRequest.sseCustomerKey + sseCustomerKeyMd5 = this@toCreateMultiPartUploadRequest.sseCustomerKeyMd5 + ssekmsEncryptionContext = this@toCreateMultiPartUploadRequest.ssekmsEncryptionContext + ssekmsKeyId = this@toCreateMultiPartUploadRequest.ssekmsKeyId + storageClass = this@toCreateMultiPartUploadRequest.storageClass + tagging = this@toCreateMultiPartUploadRequest.tagging + websiteRedirectLocation = this@toCreateMultiPartUploadRequest.websiteRedirectLocation + } diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/UploadFile.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/UploadFile.kt new file mode 100644 index 00000000000..0079ff18dd2 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/UploadFile.kt @@ -0,0 +1,140 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.utils + +import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.CompletedPart +import aws.sdk.kotlin.services.s3.model.UploadPartRequest +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.content.fromInputStream +import aws.smithy.kotlin.runtime.io.SdkBuffer +import aws.smithy.kotlin.runtime.io.SdkByteReadChannel +import aws.smithy.kotlin.runtime.io.SdkSource +import aws.smithy.kotlin.runtime.telemetry.logging.Logger + +// S3 imposed limit for parts in a multipart upload +private const val MAX_NUMBER_PARTS = 10_000L + +/** + * Determines the actual part size to use for a multipart S3 upload. + * + * This function calculates the part size based on the total size + * of the file and the requested part size. If the requested part size is + * too small to allow the upload to fit within S3's 10,000-part limit, the + * part size will be automatically increased so that exactly 10,000 parts + * are uploaded. + */ +internal fun resolvePartSize(uploadFileRequest: UploadFileRequest, tm: S3TransferManager, logger: Logger): Long { + val targetNumberOfParts = uploadFileRequest.contentLength / tm.targePartSize + return if (targetNumberOfParts > MAX_NUMBER_PARTS) { + ceilDiv(uploadFileRequest.contentLength, MAX_NUMBER_PARTS).also { + logger.debug { "Target part size is too small to meet the 10,000 S3 part limit. Increasing part size to $it" } + } + } else { + tm.targePartSize + } +} + +/** + * Retrieves the next part of a multipart upload from the given part source. + */ +internal suspend fun SdkBuffer.getNextPart(partSource: Any, partSize: Long, tm: S3TransferManager) { + when (partSource) { + is ByteArray -> { + this.write( + partSource.sliceArray( + tm.context.transferredBytes!!.toInt().. { + var readBytes = 0L + while (readBytes < partSize) { + readBytes += partSource.read(this, partSize - readBytes) + } + } + is SdkSource -> { + var readBytes = 0L + while (readBytes < partSize) { + readBytes += partSource.read(this, partSize - readBytes) + } + } + } +} + +/** + * Builds a low-level S3 upload part request from a high-level upload file request + * and data from the S3 Transfer Manager. + */ +internal fun buildUploadPartRequest( + uploadFileRequest: UploadFileRequest, + currentPart: SdkBuffer, + currentPartNumber: Long, + mpuUploadId: String, +): UploadPartRequest = + UploadPartRequest { + // From high-level request + bucket = uploadFileRequest.bucket + checksumAlgorithm = uploadFileRequest.checksumAlgorithm + expectedBucketOwner = uploadFileRequest.expectedBucketOwner + key = uploadFileRequest.key + requestPayer = uploadFileRequest.requestPayer + sseCustomerAlgorithm = uploadFileRequest.sseCustomerAlgorithm + sseCustomerKey = uploadFileRequest.sseCustomerKey + sseCustomerKeyMd5 = uploadFileRequest.sseCustomerKeyMd5 + + // From transfer manager + uploadId = mpuUploadId + body = ByteStream.fromInputStream(currentPart.inputStream(), currentPart.size) + partNumber = currentPartNumber.toInt() + } + +/** + * Builds a low-level S3 complete multipart upload request from a high-level upload file request + * and data from the S3 Transfer Manager. + */ +internal fun buildCompleteMultipartUploadRequest( + uploadFileRequest: UploadFileRequest, + mpuUploadId: String, + uploadedParts: List, +): CompleteMultipartUploadRequest = + CompleteMultipartUploadRequest { + // From high-level request + bucket = uploadFileRequest.bucket + checksumCrc32 = uploadFileRequest.checksumCrc32 + checksumCrc32C = uploadFileRequest.checksumCrc32C + checksumCrc64Nvme = uploadFileRequest.checksumCrc64Nvme + checksumSha1 = uploadFileRequest.checksumSha1 + checksumSha256 = uploadFileRequest.checksumSha256 + expectedBucketOwner = uploadFileRequest.expectedBucketOwner + ifMatch = uploadFileRequest.ifMatch + ifNoneMatch = uploadFileRequest.ifNoneMatch + key = uploadFileRequest.key + requestPayer = uploadFileRequest.requestPayer + sseCustomerAlgorithm = uploadFileRequest.sseCustomerAlgorithm + sseCustomerKey = uploadFileRequest.sseCustomerKey + sseCustomerKeyMd5 = uploadFileRequest.sseCustomerKeyMd5 + + // From transfer manager + uploadId = mpuUploadId + multipartUpload { + parts = uploadedParts + } + } + +/** + * Returns the ceiling of the division + * + * This means the result is rounded up to the nearest integer if the dividend is not + * evenly divisible by the divisor + */ +internal fun ceilDiv(dividend: Long, divisor: Long): Long { + val div = dividend / divisor + val remainder = dividend % divisor + return if (remainder != 0L) div + 1 else div +} diff --git a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt new file mode 100644 index 00000000000..2e9e768b224 --- /dev/null +++ b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt @@ -0,0 +1,45 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager + +import aws.sdk.kotlin.runtime.http.interceptors.businessmetrics.AwsBusinessMetric +import aws.sdk.kotlin.services.s3.S3Client +import aws.smithy.kotlin.runtime.businessmetrics.containsBusinessMetric +import aws.smithy.kotlin.runtime.client.ProtocolResponseInterceptorContext +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.httptest.TestEngine +import kotlinx.coroutines.runBlocking +import kotlin.test.Test + +class BusinessMetricInterceptorTest { + @Test + fun businessMetricExists(): Unit = runBlocking { + val message = "Hello World" + val testInterceptor = object : HttpInterceptor { + override fun readAfterTransmit(context: ProtocolResponseInterceptorContext) { + assert(context.executionContext.containsBusinessMetric(AwsBusinessMetric.S3_TRANSFER)) + } + } + + S3Client { + region = "us-west-2" + httpClient = TestEngine() + interceptors += testInterceptor + }.use { s3Client -> + S3TransferManager { + client = s3Client + }.uploadFile { + bucket = "b" + key = "k" + body = ByteStream.fromString(message) + contentLength = message.length.toLong() + } + } + } +} diff --git a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt new file mode 100644 index 00000000000..58ae4d541ff --- /dev/null +++ b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt @@ -0,0 +1,59 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager + +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.PutObjectRequest +import aws.sdk.kotlin.services.s3.model.PutObjectResponse +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.httptest.TestEngine +import kotlinx.coroutines.runBlocking +import kotlin.test.Test + +class TransferInterceptorTest { + @Test + fun interceptorsCanReadAndModify(): Unit = runBlocking { + val message = "Hello World" + + S3Client { + region = "us-west-2" + httpClient = TestEngine() + }.use { s3Client -> + S3TransferManager { + client = s3Client + interceptors += object : TransferInterceptor { + // Test reads + override fun readBeforeTransferInitiated(context: TransferContext) { + assert(context.transferredBytes == 0L) + assert(context.request is PutObjectRequest) + } + override fun readBeforeTransferCompleted(context: TransferContext) { + assert(context.transferredBytes == message.length.toLong()) + assert(context.response is PutObjectResponse) + } + + // Test modifications + override fun modifyBeforeTransferCompleted(context: TransferContext): TransferContext { + val newContext = context.copy() + newContext.request = CompleteMultipartUploadRequest {} + newContext.transferredBytes = message.length.toLong() * 10 + return newContext + } + override fun readAfterTransferCompleted(context: TransferContext) { + assert(context.request is CompleteMultipartUploadRequest) + assert(context.transferredBytes == message.length.toLong() * 10) + } + } + }.uploadFile { + bucket = "b" + key = "k" + body = ByteStream.fromString(message) + contentLength = message.length.toLong() + } + } + } +} diff --git a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt new file mode 100644 index 00000000000..3d7979b2d3c --- /dev/null +++ b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt @@ -0,0 +1,58 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager + +import aws.sdk.kotlin.services.s3.S3Client +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.content.fromInputStream +import aws.smithy.kotlin.runtime.testing.RandomTempFile +import kotlinx.coroutines.runBlocking +import kotlin.test.Ignore +import kotlin.test.Test + +// TODO: Setup e2e test environment - can't run these every build and in CI +class UploadFileTest { + @Ignore + @Test + fun singleObjectUpload(): Unit = runBlocking { + val message = "Hello World" + + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager { + client = s3Client + }.uploadFile { + bucket = "aoperez" + key = "k" + body = ByteStream.fromString(message) + contentLength = message.length.toLong() + } + } + } + + @Ignore + @Test + fun multiplePartUpload(): Unit = runBlocking { + val messageLength = 10L * 1024L * 1024L // 10 MB + val file = RandomTempFile(messageLength) + + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager { + client = s3Client + multipartUploadThreshold = 1 + targePartSize = 5L * 1024L * 1024L // 5 MB + }.uploadFile { + bucket = "aoperez" + key = "mpuK" + body = ByteStream.fromInputStream(file.inputStream(), messageLength) + contentLength = messageLength + } + } + } +} From 8edf60463a5911c5fc396f1b865bbe4093a50a36 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 17 Oct 2025 10:10:23 -0400 Subject: [PATCH 05/11] provide static dummy credentials to tests --- .../hll/s3transfermanager/BusinessMetricInterceptorTest.kt | 4 ++++ .../kotlin/hll/s3transfermanager/TransferInterceptorTest.kt | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt index 2e9e768b224..724e3200183 100644 --- a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt +++ b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt @@ -5,8 +5,11 @@ package aws.sdk.kotlin.hll.s3transfermanager +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.runtime.http.interceptors.businessmetrics.AwsBusinessMetric import aws.sdk.kotlin.services.s3.S3Client +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials.Companion.invoke import aws.smithy.kotlin.runtime.businessmetrics.containsBusinessMetric import aws.smithy.kotlin.runtime.client.ProtocolResponseInterceptorContext import aws.smithy.kotlin.runtime.content.ByteStream @@ -31,6 +34,7 @@ class BusinessMetricInterceptorTest { region = "us-west-2" httpClient = TestEngine() interceptors += testInterceptor + credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) }.use { s3Client -> S3TransferManager { client = s3Client diff --git a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt index 58ae4d541ff..6a94df2b79b 100644 --- a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt +++ b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt @@ -5,10 +5,13 @@ package aws.sdk.kotlin.hll.s3transfermanager +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest import aws.sdk.kotlin.services.s3.model.PutObjectRequest import aws.sdk.kotlin.services.s3.model.PutObjectResponse +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials.Companion.invoke import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.httptest.TestEngine import kotlinx.coroutines.runBlocking @@ -22,6 +25,7 @@ class TransferInterceptorTest { S3Client { region = "us-west-2" httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) }.use { s3Client -> S3TransferManager { client = s3Client From 413787354855c2beefe219f2d8f61cbb656290bc Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 17 Oct 2025 10:35:25 -0400 Subject: [PATCH 06/11] add s3 as a test dependency? --- hll/s3-transfer-manager/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/hll/s3-transfer-manager/build.gradle.kts b/hll/s3-transfer-manager/build.gradle.kts index dc3631f407d..c2b799f6be9 100644 --- a/hll/s3-transfer-manager/build.gradle.kts +++ b/hll/s3-transfer-manager/build.gradle.kts @@ -19,6 +19,7 @@ kotlin { dependencies { implementation(libs.smithy.kotlin.test.jvm) implementation(libs.smithy.kotlin.testing.jvm) + implementation(libs.s3) } } } From d84516d31822c6877724d5e96057e4c9e7142e27 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 17 Oct 2025 10:46:14 -0400 Subject: [PATCH 07/11] refactor tests to be JVM only --- hll/s3-transfer-manager/build.gradle.kts | 3 +-- .../hll/s3transfermanager/BusinessMetricInterceptorTest.kt | 2 +- .../kotlin/hll/s3transfermanager/TransferInterceptorTest.kt | 5 ++++- .../aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt | 4 ++-- 4 files changed, 8 insertions(+), 6 deletions(-) rename hll/s3-transfer-manager/{common => jvm}/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt (97%) rename hll/s3-transfer-manager/{common => jvm}/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt (97%) rename hll/s3-transfer-manager/{common => jvm}/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt (95%) diff --git a/hll/s3-transfer-manager/build.gradle.kts b/hll/s3-transfer-manager/build.gradle.kts index c2b799f6be9..6a6315e5707 100644 --- a/hll/s3-transfer-manager/build.gradle.kts +++ b/hll/s3-transfer-manager/build.gradle.kts @@ -15,11 +15,10 @@ kotlin { implementation(libs.s3) } } - commonTest { + jvmTest { dependencies { implementation(libs.smithy.kotlin.test.jvm) implementation(libs.smithy.kotlin.testing.jvm) - implementation(libs.s3) } } } diff --git a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt similarity index 97% rename from hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt rename to hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt index 724e3200183..1d26b4c6fbd 100644 --- a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt +++ b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt @@ -36,7 +36,7 @@ class BusinessMetricInterceptorTest { interceptors += testInterceptor credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) }.use { s3Client -> - S3TransferManager { + S3TransferManager.Companion { client = s3Client }.uploadFile { bucket = "b" diff --git a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt similarity index 97% rename from hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt rename to hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt index 6a94df2b79b..c2e8713dba9 100644 --- a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt +++ b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt @@ -15,6 +15,7 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials.Companion.invok import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.httptest.TestEngine import kotlinx.coroutines.runBlocking +import kotlin.collections.plusAssign import kotlin.test.Test class TransferInterceptorTest { @@ -27,7 +28,7 @@ class TransferInterceptorTest { httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) }.use { s3Client -> - S3TransferManager { + S3TransferManager.Companion { client = s3Client interceptors += object : TransferInterceptor { // Test reads @@ -35,6 +36,7 @@ class TransferInterceptorTest { assert(context.transferredBytes == 0L) assert(context.request is PutObjectRequest) } + override fun readBeforeTransferCompleted(context: TransferContext) { assert(context.transferredBytes == message.length.toLong()) assert(context.response is PutObjectResponse) @@ -47,6 +49,7 @@ class TransferInterceptorTest { newContext.transferredBytes = message.length.toLong() * 10 return newContext } + override fun readAfterTransferCompleted(context: TransferContext) { assert(context.request is CompleteMultipartUploadRequest) assert(context.transferredBytes == message.length.toLong() * 10) diff --git a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt similarity index 95% rename from hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt rename to hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt index 3d7979b2d3c..fdfc545a732 100644 --- a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt +++ b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt @@ -23,7 +23,7 @@ class UploadFileTest { S3Client { region = "us-west-2" }.use { s3Client -> - S3TransferManager { + S3TransferManager.Companion { client = s3Client }.uploadFile { bucket = "aoperez" @@ -43,7 +43,7 @@ class UploadFileTest { S3Client { region = "us-west-2" }.use { s3Client -> - S3TransferManager { + S3TransferManager.Companion { client = s3Client multipartUploadThreshold = 1 targePartSize = 5L * 1024L * 1024L // 5 MB From 9d5ec07074e07b8f05785adff0658ce6446dd807 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Fri, 17 Oct 2025 11:04:23 -0400 Subject: [PATCH 08/11] Depend on missing 'DefaultAwsSigner' --- hll/s3-transfer-manager/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/hll/s3-transfer-manager/build.gradle.kts b/hll/s3-transfer-manager/build.gradle.kts index 6a6315e5707..6528e0bd37a 100644 --- a/hll/s3-transfer-manager/build.gradle.kts +++ b/hll/s3-transfer-manager/build.gradle.kts @@ -19,6 +19,7 @@ kotlin { dependencies { implementation(libs.smithy.kotlin.test.jvm) implementation(libs.smithy.kotlin.testing.jvm) + implementation(libs.smithy.kotlin.aws.signing.common) } } } From fb2417f5e2389cb528245e6f21b4163316ca7864 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Tue, 21 Oct 2025 10:00:13 -0400 Subject: [PATCH 09/11] feedback --- aws-runtime/aws-http/api/aws-http.api | 2 + .../AwsBusinessMetricsUtils.kt | 2 + gradle/libs.versions.toml | 4 -- .../api/s3-transfer-manager.api | 59 ++++++++-------- hll/s3-transfer-manager/build.gradle.kts | 13 +++- .../s3transfermanager/S3TransferManager.kt | 67 +++++++++---------- ...ansferManagerBusinessMetricInterceptor.kt} | 2 +- .../s3transfermanager/TransferInterceptor.kt | 4 +- ...wnloadType.kt => MultipartDownloadType.kt} | 6 +- .../model/UploadFileRequest.kt | 12 +--- .../model/UploadFileResponse.kt | 16 ++++- .../s3transfermanager/utils/Conversions.kt | 13 ++-- .../hll/s3transfermanager/utils/Exceptions.kt | 8 +++ .../hll/s3transfermanager/utils/UploadFile.kt | 8 +-- ...> S3TransferManagerBusinessMetricsTest.kt} | 8 +-- .../TransferInterceptorTest.kt | 11 ++- .../hll/s3transfermanager/UploadFileTest.kt | 10 ++- settings.gradle.kts | 7 +- 18 files changed, 136 insertions(+), 116 deletions(-) rename hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/{BusinessMetricInterceptor.kt => S3TransferManagerBusinessMetricInterceptor.kt} (90%) rename hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/{MultiPartDownloadType.kt => MultipartDownloadType.kt} (58%) create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Exceptions.kt rename hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/{BusinessMetricInterceptorTest.kt => S3TransferManagerBusinessMetricsTest.kt} (86%) diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index 3bbefbda4c8..332992d53e3 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -172,6 +172,8 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsB public static final field DDB_MAPPER Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; public static final field S3_EXPRESS_BUCKET Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; public static final field S3_TRANSFER Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; + public static final field S3_TRANSFER_DOWNLOAD_DIRECTORY Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; + public static final field S3_TRANSFER_UPLOAD_DIRECTORY Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; public static fun getEntries ()Lkotlin/enums/EnumEntries; public fun getIdentifier ()Ljava/lang/String; public fun toString ()Ljava/lang/String; diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt index f806b908149..bcee46a0020 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt @@ -63,6 +63,8 @@ public enum class AwsBusinessMetric(public override val identifier: String) : Bu S3_EXPRESS_BUCKET("J"), DDB_MAPPER("d"), S3_TRANSFER("G"), + S3_TRANSFER_UPLOAD_DIRECTORY("9"), + S3_TRANSFER_DOWNLOAD_DIRECTORY("+"), ; @InternalApi diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 22152e6b54d..93fa5b883f2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -28,9 +28,6 @@ mockk-version = "1.13.13" slf4j-version = "2.0.16" jsoup-version = "1.20.1" -# s3 transfer manager -s3-version = "1.5.62" - [libraries] aws-kotlin-repo-tools-build-support = { module="aws.sdk.kotlin.gradle:build-support", version.ref = "aws-kotlin-repo-tools-version" } @@ -129,7 +126,6 @@ kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serializa mockk = { module = "io.mockk:mockk", version.ref = "mockk-version" } ddb-local = { module = "com.amazonaws:DynamoDBLocal", version.ref = "ddb-local-version" } -s3 = { module = "aws.sdk.kotlin:s3", version.ref = "s3-version" } [bundles] # bundle of smithy-kotlin dependencies all AWS service clients have diff --git a/hll/s3-transfer-manager/api/s3-transfer-manager.api b/hll/s3-transfer-manager/api/s3-transfer-manager.api index ee14ee4d2dc..fe4b3332c5c 100644 --- a/hll/s3-transfer-manager/api/s3-transfer-manager.api +++ b/hll/s3-transfer-manager/api/s3-transfer-manager.api @@ -1,11 +1,11 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager { public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Companion; - public synthetic fun (Laws/sdk/kotlin/services/s3/S3Client;JJLaws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType;Ljava/util/List;Lkotlin/jvm/internal/DefaultConstructorMarker;)V + public synthetic fun (Laws/sdk/kotlin/services/s3/S3Client;JJLaws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType;Ljava/util/List;Lkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun getClient ()Laws/sdk/kotlin/services/s3/S3Client; public final fun getInterceptors ()Ljava/util/List; - public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType; - public final fun getMultipartUploadThreshold ()J - public final fun getTargePartSize ()J + public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType; + public final fun getMultipartUploadThresholdBytes ()J + public final fun getPartSizeBytes ()J public final fun uploadFile (Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public final fun uploadFile (Lkotlin/jvm/functions/Function1;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; } @@ -14,14 +14,14 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Builde public fun ()V public final fun getClient ()Laws/sdk/kotlin/services/s3/S3Client; public final fun getInterceptors ()Ljava/util/List; - public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType; - public final fun getMultipartUploadThreshold ()J - public final fun getTargePartSize ()J + public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType; + public final fun getMultipartUploadThresholdBytes ()J + public final fun getPartSizeBytes ()J public final fun setClient (Laws/sdk/kotlin/services/s3/S3Client;)V public final fun setInterceptors (Ljava/util/List;)V - public final fun setMultipartDownloadType (Laws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType;)V - public final fun setMultipartUploadThreshold (J)V - public final fun setTargePartSize (J)V + public final fun setMultipartDownloadType (Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType;)V + public final fun setMultipartUploadThresholdBytes (J)V + public final fun setPartSizeBytes (J)V } public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Companion { @@ -30,20 +30,20 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Compan public final class aws/sdk/kotlin/hll/s3transfermanager/TransferContext { public fun ()V - public fun (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;[BLjava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;)V - public synthetic fun (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;[BLjava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;)V + public synthetic fun (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun component1 ()Ljava/lang/Object; public final fun component2 ()Ljava/lang/Object; public final fun component3 ()Ljava/lang/Long; - public final fun component4 ()[B + public final fun component4 ()Laws/smithy/kotlin/runtime/content/ByteStream; public final fun component5 ()Ljava/lang/Long; public final fun component6 ()Ljava/lang/Long; public final fun component7 ()Ljava/lang/String; public final fun component8 ()Ljava/lang/Long; - public final fun copy (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;[BLjava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public static synthetic fun copy$default (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;[BLjava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;ILjava/lang/Object;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public final fun copy (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static synthetic fun copy$default (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;ILjava/lang/Object;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; public fun equals (Ljava/lang/Object;)Z - public final fun getCurrentBytes ()[B + public final fun getCurrentBytes ()Laws/smithy/kotlin/runtime/content/ByteStream; public final fun getCurrentFile ()Ljava/lang/String; public final fun getRequest ()Ljava/lang/Object; public final fun getResponse ()Ljava/lang/Object; @@ -52,7 +52,7 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/TransferContext { public final fun getTransferredBytes ()Ljava/lang/Long; public final fun getTransferredFiles ()Ljava/lang/Long; public fun hashCode ()I - public final fun setCurrentBytes ([B)V + public final fun setCurrentBytes (Laws/smithy/kotlin/runtime/content/ByteStream;)V public final fun setCurrentFile (Ljava/lang/String;)V public final fun setRequest (Ljava/lang/Object;)V public final fun setResponse (Ljava/lang/Object;)V @@ -101,20 +101,20 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor$Defa public static fun readBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V } -public abstract interface class aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType { +public abstract interface class aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType { } -public final class aws/sdk/kotlin/hll/s3transfermanager/model/Part : aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType { +public final class aws/sdk/kotlin/hll/s3transfermanager/model/Part : aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType { public static final field INSTANCE Laws/sdk/kotlin/hll/s3transfermanager/model/Part; } -public final class aws/sdk/kotlin/hll/s3transfermanager/model/Range : aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType { +public final class aws/sdk/kotlin/hll/s3transfermanager/model/Range : aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType { public static final field INSTANCE Laws/sdk/kotlin/hll/s3transfermanager/model/Range; } public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest { public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$Companion; - public synthetic fun (Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/String;Ljava/lang/Boolean;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;JLjava/lang/String;Ljava/lang/String;Laws/smithy/kotlin/runtime/time/Instant;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus;Laws/sdk/kotlin/services/s3/model/ObjectLockMode;Laws/smithy/kotlin/runtime/time/Instant;Laws/sdk/kotlin/services/s3/model/RequestPayer;Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/StorageClass;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/internal/DefaultConstructorMarker;)V + public synthetic fun (Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/String;Ljava/lang/Boolean;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/smithy/kotlin/runtime/time/Instant;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus;Laws/sdk/kotlin/services/s3/model/ObjectLockMode;Laws/smithy/kotlin/runtime/time/Instant;Laws/sdk/kotlin/services/s3/model/RequestPayer;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;Laws/sdk/kotlin/services/s3/model/StorageClass;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun getAcl ()Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl; public final fun getBody ()Laws/smithy/kotlin/runtime/content/ByteStream; public final fun getBucket ()Ljava/lang/String; @@ -129,7 +129,6 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest public final fun getContentDisposition ()Ljava/lang/String; public final fun getContentEncoding ()Ljava/lang/String; public final fun getContentLanguage ()Ljava/lang/String; - public final fun getContentLength ()J public final fun getContentType ()Ljava/lang/String; public final fun getExpectedBucketOwner ()Ljava/lang/String; public final fun getExpires ()Laws/smithy/kotlin/runtime/time/Instant; @@ -146,7 +145,6 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest public final fun getObjectLockRetainUntilDate ()Laws/smithy/kotlin/runtime/time/Instant; public final fun getRequestPayer ()Laws/sdk/kotlin/services/s3/model/RequestPayer; public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; - public final fun getSource ()Ljava/lang/String; public final fun getSseCustomerAlgorithm ()Ljava/lang/String; public final fun getSseCustomerKey ()Ljava/lang/String; public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; @@ -174,7 +172,6 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$ public final fun getContentDisposition ()Ljava/lang/String; public final fun getContentEncoding ()Ljava/lang/String; public final fun getContentLanguage ()Ljava/lang/String; - public final fun getContentLength ()Ljava/lang/Long; public final fun getContentType ()Ljava/lang/String; public final fun getExpectedBucketOwner ()Ljava/lang/String; public final fun getExpires ()Laws/smithy/kotlin/runtime/time/Instant; @@ -191,7 +188,6 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$ public final fun getObjectLockRetainUntilDate ()Laws/smithy/kotlin/runtime/time/Instant; public final fun getRequestPayer ()Laws/sdk/kotlin/services/s3/model/RequestPayer; public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; - public final fun getSource ()Ljava/lang/String; public final fun getSseCustomerAlgorithm ()Ljava/lang/String; public final fun getSseCustomerKey ()Ljava/lang/String; public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; @@ -214,7 +210,6 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$ public final fun setContentDisposition (Ljava/lang/String;)V public final fun setContentEncoding (Ljava/lang/String;)V public final fun setContentLanguage (Ljava/lang/String;)V - public final fun setContentLength (Ljava/lang/Long;)V public final fun setContentType (Ljava/lang/String;)V public final fun setExpectedBucketOwner (Ljava/lang/String;)V public final fun setExpires (Laws/smithy/kotlin/runtime/time/Instant;)V @@ -231,7 +226,6 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$ public final fun setObjectLockRetainUntilDate (Laws/smithy/kotlin/runtime/time/Instant;)V public final fun setRequestPayer (Laws/sdk/kotlin/services/s3/model/RequestPayer;)V public final fun setServerSideEncryption (Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;)V - public final fun setSource (Ljava/lang/String;)V public final fun setSseCustomerAlgorithm (Ljava/lang/String;)V public final fun setSseCustomerKey (Ljava/lang/String;)V public final fun setSseCustomerKeyMd5 (Ljava/lang/String;)V @@ -248,7 +242,7 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$ public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse { public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse$Companion; - public fun (Ljava/lang/Boolean;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ChecksumType;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/RequestCharged;Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;Ljava/lang/String;Ljava/lang/String;)V + public fun (Ljava/lang/Boolean;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ChecksumType;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/RequestCharged;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;Ljava/lang/String;)V public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; public final fun getChecksumCrc32 ()Ljava/lang/String; public final fun getChecksumCrc32C ()Ljava/lang/String; @@ -260,6 +254,9 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse public final fun getExpiration ()Ljava/lang/String; public final fun getRequestCharged ()Laws/sdk/kotlin/services/s3/model/RequestCharged; public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; public final fun getSsekmsKeyId ()Ljava/lang/String; public final fun getVersionId ()Ljava/lang/String; } @@ -277,6 +274,9 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse public final fun getExpiration ()Ljava/lang/String; public final fun getRequestCharged ()Laws/sdk/kotlin/services/s3/model/RequestCharged; public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; public final fun getSsekmsKeyId ()Ljava/lang/String; public final fun getVersionId ()Ljava/lang/String; public final fun setBucketKeyEnabled (Ljava/lang/Boolean;)V @@ -290,6 +290,9 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse public final fun setExpiration (Ljava/lang/String;)V public final fun setRequestCharged (Laws/sdk/kotlin/services/s3/model/RequestCharged;)V public final fun setServerSideEncryption (Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;)V + public final fun setSseCustomerAlgorithm (Ljava/lang/String;)V + public final fun setSseCustomerKeyMd5 (Ljava/lang/String;)V + public final fun setSsekmsEncryptionContext (Ljava/lang/String;)V public final fun setSsekmsKeyId (Ljava/lang/String;)V public final fun setVersionId (Ljava/lang/String;)V } diff --git a/hll/s3-transfer-manager/build.gradle.kts b/hll/s3-transfer-manager/build.gradle.kts index 6528e0bd37a..bb636ac3db6 100644 --- a/hll/s3-transfer-manager/build.gradle.kts +++ b/hll/s3-transfer-manager/build.gradle.kts @@ -1,10 +1,18 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.gradle.kotlin.dsl.dependencies +import org.gradle.kotlin.dsl.sourceSets + /* * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ description = "S3 Transfer Manager for the AWS SDK for Kotlin" -extra["displayName"] = "AWS :: SDK :: Kotlin :: HLL :: S3TransferManager" +extra["displayName"] = "AWS :: SDK :: Kotlin :: HLL :: S3 Transfer Manager" extra["moduleName"] = "aws.sdk.kotlin.hll.s3transfermanager" kotlin { @@ -12,14 +20,13 @@ kotlin { commonMain { dependencies { implementation(project(":aws-runtime:aws-http")) - implementation(libs.s3) + implementation(project(":services:s3")) } } jvmTest { dependencies { implementation(libs.smithy.kotlin.test.jvm) implementation(libs.smithy.kotlin.testing.jvm) - implementation(libs.smithy.kotlin.aws.signing.common) } } } diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt index 05e846f69d9..edc27f86449 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt @@ -5,10 +5,11 @@ package aws.sdk.kotlin.hll.s3transfermanager -import aws.sdk.kotlin.hll.s3transfermanager.model.MultiPartDownloadType +import aws.sdk.kotlin.hll.s3transfermanager.model.MultipartDownloadType import aws.sdk.kotlin.hll.s3transfermanager.model.Part import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileResponse +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException import aws.sdk.kotlin.hll.s3transfermanager.utils.buildCompleteMultipartUploadRequest import aws.sdk.kotlin.hll.s3transfermanager.utils.buildUploadPartRequest import aws.sdk.kotlin.hll.s3transfermanager.utils.ceilDiv @@ -41,9 +42,9 @@ import kotlinx.coroutines.coroutineScope */ public class S3TransferManager private constructor( public val client: S3Client, - public val targePartSize: Long, - public val multipartUploadThreshold: Long, - public val multipartDownloadType: MultiPartDownloadType, + public val partSizeBytes: Long, + public val multipartUploadThresholdBytes: Long, + public val multipartDownloadType: MultipartDownloadType, public val interceptors: MutableList, ) { internal var context: TransferContext = TransferContext() @@ -55,16 +56,16 @@ public class S3TransferManager private constructor( public class Builder { public var client: S3Client? = null - public var targePartSize: Long = 8_000_000 - public var multipartUploadThreshold: Long = 16_000_000L - public var multipartDownloadType: MultiPartDownloadType = Part + public var partSizeBytes: Long = 8_000_000 + public var multipartUploadThresholdBytes: Long = 16_000_000L + public var multipartDownloadType: MultipartDownloadType = Part public var interceptors: MutableList = mutableListOf() internal fun build(): S3TransferManager = S3TransferManager( - client = client?.withConfig { interceptors += BusinessMetricInterceptor } ?: error("client must be set"), - targePartSize = targePartSize, - multipartUploadThreshold = multipartUploadThreshold, + client = client?.withConfig { interceptors += S3TransferManagerBusinessMetricInterceptor } ?: error("client must be set"), + partSizeBytes = partSizeBytes, + multipartUploadThresholdBytes = multipartUploadThresholdBytes, multipartDownloadType = multipartDownloadType, interceptors = interceptors, ) @@ -117,17 +118,18 @@ public class S3TransferManager private constructor( * for large objects as needed. * * This function handles the complexity of splitting the data into parts, - * uploading each part, and completing the multipart upload. For object smaller than [multipartUploadThreshold], + * uploading each part, and completing the multipart upload. For object smaller than [multipartUploadThresholdBytes], * a standard single-part upload is performed automatically. * - * If the specified [targePartSize] for multipart uploads is too small to allow + * If the specified [partSizeBytes] for multipart uploads is too small to allow * all parts to fit within S3's limit of 10,000 parts, the part size will be * automatically increased so that exactly 10,000 parts are uploaded. */ public suspend fun uploadFile(uploadFileRequest: UploadFileRequest): Deferred = coroutineScope { - val multiPartUpload = uploadFileRequest.contentLength >= multipartUploadThreshold + val contentLength = uploadFileRequest.body?.contentLength ?: throw S3TransferManagerException("UploadFileRequest.body.contentLength must be set") + val multiPartUpload = contentLength >= multipartUploadThresholdBytes val uploadedParts = mutableListOf() - var mpuUploadId = "null" + lateinit var mpuUploadId: String val logger = coroutineContext.logger() @@ -136,7 +138,7 @@ public class S3TransferManager private constructor( */ suspend fun transferInitiated(multiPartUpload: Boolean) { context.transferredBytes = 0L - context.transferableBytes = uploadFileRequest.contentLength + context.transferableBytes = contentLength context.request = if (multiPartUpload) { uploadFileRequest.toCreateMultiPartUploadRequest() } else { @@ -145,7 +147,7 @@ public class S3TransferManager private constructor( operationHook(TransferInitiated) { if (multiPartUpload) { context.response = client.createMultipartUpload(context.request as CreateMultipartUploadRequest) - mpuUploadId = (context.response as CreateMultipartUploadResponse).uploadId ?: throw Exception("Missing upload id in create multipart upload response") + mpuUploadId = (context.response as CreateMultipartUploadResponse).uploadId ?: throw S3TransferManagerException("Missing upload id in create multipart upload response") } } } @@ -156,13 +158,13 @@ public class S3TransferManager private constructor( suspend fun transferBytes(multiPartUpload: Boolean) { if (multiPartUpload) { try { - val partSize = resolvePartSize(uploadFileRequest, this@S3TransferManager, logger) - val numberOfParts = ceilDiv(uploadFileRequest.contentLength, partSize) + val partSize = resolvePartSize(contentLength, this@S3TransferManager, logger) + val numberOfParts = ceilDiv(contentLength, partSize) val partSource = when (uploadFileRequest.body) { is ByteStream.Buffer -> uploadFileRequest.body.bytes() is ByteStream.ChannelStream -> uploadFileRequest.body.readFrom() is ByteStream.SourceStream -> uploadFileRequest.body.readFrom() - else -> error("Unhandled body type: ${uploadFileRequest.body?.let { it::class.simpleName } ?: "null"}") + else -> throw S3TransferManagerException("Unhandled body type: ${uploadFileRequest.body?.let { it::class.simpleName } ?: "null"}") } val partBuffer = SdkBuffer() var currentPartNumber = 1L @@ -170,8 +172,8 @@ public class S3TransferManager private constructor( while (context.transferredBytes!! < context.transferableBytes!!) { partBuffer.getNextPart(partSource, partSize, this@S3TransferManager) if (currentPartNumber != numberOfParts) { - check(partBuffer.size == partSize) { - "Part #$currentPartNumber size mismatch detected. Expected $partSize, actual: ${partBuffer.size}" + if (partBuffer.size != partSize) { + throw S3TransferManagerException("Part #$currentPartNumber size mismatch detected. Expected $partSize, actual: ${partBuffer.size}") } } @@ -195,8 +197,8 @@ public class S3TransferManager private constructor( currentPartNumber += 1 } - check(uploadedParts.size == numberOfParts.toInt()) { - "The number of uploaded parts does not match the expected count. Expected $numberOfParts, actual: ${uploadedParts.size}" + if (uploadedParts.size != numberOfParts.toInt()) { + throw S3TransferManagerException("The number of uploaded parts does not match the expected count. Expected $numberOfParts, actual: ${uploadedParts.size}") } } catch (uploadPartThrowable: Throwable) { try { @@ -207,9 +209,9 @@ public class S3TransferManager private constructor( requestPayer = uploadFileRequest.requestPayer uploadId = mpuUploadId } - throw Exception("Multipart upload failed (ID: $mpuUploadId). One or more parts could not be uploaded", uploadPartThrowable) + throw S3TransferManagerException("Multipart upload failed (ID: $mpuUploadId). One or more parts could not be uploaded", uploadPartThrowable) } catch (abortThrowable: Throwable) { - throw Exception("Multipart upload failed (ID: $mpuUploadId). Unable to abort multipart upload.", abortThrowable) + throw S3TransferManagerException("Multipart upload failed (ID: $mpuUploadId). Unable to abort multipart upload.", abortThrowable) } } } else { @@ -232,20 +234,13 @@ public class S3TransferManager private constructor( try { context.response = client.completeMultipartUpload(context.request as CompleteMultipartUploadRequest) } catch (t: Throwable) { - throw Exception("Unable to complete multipart upload with ID: $mpuUploadId", t) + throw S3TransferManagerException("Unable to complete multipart upload with ID: $mpuUploadId", t) } } } } async { - checkNotNull(uploadFileRequest.body?.contentLength) { - "UploadFileRequest.body.contentLength must be set" - } - check(uploadFileRequest.body.contentLength == uploadFileRequest.contentLength) { - "contentLength mismatch. uploadFileRequest: ${uploadFileRequest.contentLength}, uploadFileRequest.body.contentLength: ${uploadFileRequest.body.contentLength}" - } - transferInitiated(multiPartUpload) transferBytes(multiPartUpload) transferComplete(multiPartUpload) @@ -253,7 +248,7 @@ public class S3TransferManager private constructor( when (context.response) { is PutObjectResponse -> (context.response as PutObjectResponse).toUploadFileResponse() is CompleteMultipartUploadResponse -> (context.response as CompleteMultipartUploadResponse).toUploadFileResponse() - else -> error("Unexpected response: ${context.response?.let { it::class.simpleName } ?: "null"}") + else -> throw S3TransferManagerException("Unexpected response: ${context.response?.let { it::class.simpleName } ?: "null"}") } } } @@ -263,10 +258,10 @@ public class S3TransferManager private constructor( * for large objects as needed. * * This function handles the complexity of splitting the data into parts, - * uploading each part, and completing the multipart upload. For object smaller than [multipartUploadThreshold], + * uploading each part, and completing the multipart upload. For object smaller than [multipartUploadThresholdBytes], * a standard single-part upload is performed automatically. * - * If the specified [targePartSize] for multipart uploads is too small to allow + * If the specified [partSizeBytes] for multipart uploads is too small to allow * all parts to fit within S3's limit of 10,000 parts, the part size will be * automatically increased so that exactly 10,000 parts are uploaded. */ diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricInterceptor.kt similarity index 90% rename from hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptor.kt rename to hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricInterceptor.kt index f9f2228364b..16331343cbb 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptor.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricInterceptor.kt @@ -13,7 +13,7 @@ import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor /** * An interceptor that emits the S3 Transfer Manager business metric */ -internal object BusinessMetricInterceptor : HttpInterceptor { +internal object S3TransferManagerBusinessMetricInterceptor : HttpInterceptor { override suspend fun modifyBeforeSerialization(context: RequestInterceptorContext): Any { context.executionContext.emitBusinessMetric(AwsBusinessMetric.S3_TRANSFER) return context.request diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt index b6403801db9..fca50b465fa 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt @@ -5,6 +5,8 @@ package aws.sdk.kotlin.hll.s3transfermanager +import aws.smithy.kotlin.runtime.content.ByteStream + // TODO: KDocs public data class TransferContext( @@ -14,7 +16,7 @@ public data class TransferContext( // Byte transfers var transferableBytes: Long? = null, - var currentBytes: ByteArray? = null, + var currentBytes: ByteStream? = null, var transferredBytes: Long? = null, // File transfers diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt similarity index 58% rename from hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType.kt rename to hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt index 3eeea64246d..4b872c292da 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt @@ -7,6 +7,6 @@ package aws.sdk.kotlin.hll.s3transfermanager.model // TODO: KDocs -public sealed interface MultiPartDownloadType -public object Range : MultiPartDownloadType -public object Part : MultiPartDownloadType +public sealed interface MultipartDownloadType +public object Range : MultipartDownloadType +public object Part : MultipartDownloadType diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest.kt index 742387a9202..041f781dab8 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest.kt @@ -30,7 +30,6 @@ public class UploadFileRequest private constructor( public val contentDisposition: String?, public val contentEncoding: String?, public val contentLanguage: String?, - public val contentLength: Long, public val contentType: String?, public val expectedBucketOwner: String?, public val expires: Instant?, @@ -46,13 +45,12 @@ public class UploadFileRequest private constructor( public val objectLockMode: ObjectLockMode?, public val objectLockRetainUntilDate: Instant?, public val requestPayer: RequestPayer?, - public val serverSideEncryption: ServerSideEncryption?, - public val source: String?, public val sseCustomerAlgorithm: String?, public val sseCustomerKey: String?, public val sseCustomerKeyMd5: String?, public val ssekmsEncryptionContext: String?, public val ssekmsKeyId: String?, + public val serverSideEncryption: ServerSideEncryption?, public val storageClass: StorageClass?, public val tagging: String?, public val websiteRedirectLocation: String?, @@ -77,7 +75,6 @@ public class UploadFileRequest private constructor( public var contentDisposition: String? = null public var contentEncoding: String? = null public var contentLanguage: String? = null - public var contentLength: Long? = null public var contentType: String? = null public var expectedBucketOwner: String? = null public var expires: Instant? = null @@ -93,13 +90,12 @@ public class UploadFileRequest private constructor( public var objectLockMode: ObjectLockMode? = null public var objectLockRetainUntilDate: Instant? = null public var requestPayer: RequestPayer? = null - public var source: String? = null - public var serverSideEncryption: ServerSideEncryption? = null public var sseCustomerAlgorithm: String? = null public var sseCustomerKey: String? = null public var sseCustomerKeyMd5: String? = null public var ssekmsEncryptionContext: String? = null public var ssekmsKeyId: String? = null + public var serverSideEncryption: ServerSideEncryption? = null public var storageClass: StorageClass? = null public var tagging: String? = null public var websiteRedirectLocation: String? = null @@ -120,7 +116,6 @@ public class UploadFileRequest private constructor( contentDisposition, contentEncoding, contentLanguage, - contentLength ?: error("contentLength must be set"), contentType, expectedBucketOwner, expires, @@ -136,13 +131,12 @@ public class UploadFileRequest private constructor( objectLockMode, objectLockRetainUntilDate, requestPayer, - serverSideEncryption, - source, sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMd5, ssekmsEncryptionContext, ssekmsKeyId, + serverSideEncryption, storageClass, tagging, websiteRedirectLocation, diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse.kt index 0b7c60bce86..0d51e82e6c7 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse.kt @@ -8,6 +8,7 @@ package aws.sdk.kotlin.hll.s3transfermanager.model import aws.sdk.kotlin.services.s3.model.ChecksumType import aws.sdk.kotlin.services.s3.model.RequestCharged import aws.sdk.kotlin.services.s3.model.ServerSideEncryption +import kotlin.String public class UploadFileResponse( public val bucketKeyEnabled: Boolean?, @@ -20,8 +21,11 @@ public class UploadFileResponse( public val eTag: String?, public val expiration: String?, public val requestCharged: RequestCharged?, - public val serverSideEncryption: ServerSideEncryption?, + public val sseCustomerAlgorithm: String?, + public val sseCustomerKeyMd5: String?, + public val ssekmsEncryptionContext: String?, public val ssekmsKeyId: String?, + public val serverSideEncryption: ServerSideEncryption?, public val versionId: String?, ) { public companion object { @@ -40,8 +44,11 @@ public class UploadFileResponse( public var eTag: String? = null public var expiration: String? = null public var requestCharged: RequestCharged? = null - public var serverSideEncryption: ServerSideEncryption? = null + public var sseCustomerAlgorithm: String? = null + public var sseCustomerKeyMd5: String? = null + public var ssekmsEncryptionContext: String? = null public var ssekmsKeyId: String? = null + public var serverSideEncryption: ServerSideEncryption? = null public var versionId: String? = null internal fun build(): UploadFileResponse = @@ -56,8 +63,11 @@ public class UploadFileResponse( eTag, expiration, requestCharged, - serverSideEncryption, + sseCustomerAlgorithm, + sseCustomerKeyMd5, + ssekmsEncryptionContext, ssekmsKeyId, + serverSideEncryption, versionId, ) } diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Conversions.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Conversions.kt index 7b0984d90db..14b6b442466 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Conversions.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Conversions.kt @@ -24,8 +24,11 @@ internal fun PutObjectResponse.toUploadFileResponse(): UploadFileResponse = eTag = this@toUploadFileResponse.eTag expiration = this@toUploadFileResponse.expiration requestCharged = this@toUploadFileResponse.requestCharged - serverSideEncryption = this@toUploadFileResponse.serverSideEncryption + sseCustomerAlgorithm = this@toUploadFileResponse.sseCustomerAlgorithm + sseCustomerKeyMd5 = this@toUploadFileResponse.sseCustomerKeyMd5 + ssekmsEncryptionContext = this@toUploadFileResponse.ssekmsEncryptionContext ssekmsKeyId = this@toUploadFileResponse.ssekmsKeyId + serverSideEncryption = this@toUploadFileResponse.serverSideEncryption versionId = this@toUploadFileResponse.versionId } @@ -41,8 +44,8 @@ internal fun CompleteMultipartUploadResponse.toUploadFileResponse(): UploadFileR eTag = this@toUploadFileResponse.eTag expiration = this@toUploadFileResponse.expiration requestCharged = this@toUploadFileResponse.requestCharged - serverSideEncryption = this@toUploadFileResponse.serverSideEncryption ssekmsKeyId = this@toUploadFileResponse.ssekmsKeyId + serverSideEncryption = this@toUploadFileResponse.serverSideEncryption versionId = this@toUploadFileResponse.versionId } @@ -62,7 +65,7 @@ internal fun UploadFileRequest.toPutObjectRequest(): PutObjectRequest = contentDisposition = this@toPutObjectRequest.contentDisposition contentEncoding = this@toPutObjectRequest.contentEncoding contentLanguage = this@toPutObjectRequest.contentLanguage - contentLength = this@toPutObjectRequest.contentLength + contentLength = this@toPutObjectRequest.body?.contentLength contentType = this@toPutObjectRequest.contentType expectedBucketOwner = this@toPutObjectRequest.expectedBucketOwner expires = this@toPutObjectRequest.expires @@ -78,12 +81,12 @@ internal fun UploadFileRequest.toPutObjectRequest(): PutObjectRequest = objectLockMode = this@toPutObjectRequest.objectLockMode objectLockRetainUntilDate = this@toPutObjectRequest.objectLockRetainUntilDate requestPayer = this@toPutObjectRequest.requestPayer - serverSideEncryption = this@toPutObjectRequest.serverSideEncryption sseCustomerAlgorithm = this@toPutObjectRequest.sseCustomerAlgorithm sseCustomerKey = this@toPutObjectRequest.sseCustomerKey sseCustomerKeyMd5 = this@toPutObjectRequest.sseCustomerKeyMd5 ssekmsEncryptionContext = this@toPutObjectRequest.ssekmsEncryptionContext ssekmsKeyId = this@toPutObjectRequest.ssekmsKeyId + serverSideEncryption = this@toPutObjectRequest.serverSideEncryption storageClass = this@toPutObjectRequest.storageClass tagging = this@toPutObjectRequest.tagging websiteRedirectLocation = this@toPutObjectRequest.websiteRedirectLocation @@ -112,12 +115,12 @@ internal fun UploadFileRequest.toCreateMultiPartUploadRequest(): CreateMultipart objectLockMode = this@toCreateMultiPartUploadRequest.objectLockMode objectLockRetainUntilDate = this@toCreateMultiPartUploadRequest.objectLockRetainUntilDate requestPayer = this@toCreateMultiPartUploadRequest.requestPayer - serverSideEncryption = this@toCreateMultiPartUploadRequest.serverSideEncryption sseCustomerAlgorithm = this@toCreateMultiPartUploadRequest.sseCustomerAlgorithm sseCustomerKey = this@toCreateMultiPartUploadRequest.sseCustomerKey sseCustomerKeyMd5 = this@toCreateMultiPartUploadRequest.sseCustomerKeyMd5 ssekmsEncryptionContext = this@toCreateMultiPartUploadRequest.ssekmsEncryptionContext ssekmsKeyId = this@toCreateMultiPartUploadRequest.ssekmsKeyId + serverSideEncryption = this@toCreateMultiPartUploadRequest.serverSideEncryption storageClass = this@toCreateMultiPartUploadRequest.storageClass tagging = this@toCreateMultiPartUploadRequest.tagging websiteRedirectLocation = this@toCreateMultiPartUploadRequest.websiteRedirectLocation diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Exceptions.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Exceptions.kt new file mode 100644 index 00000000000..b4eded45c30 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Exceptions.kt @@ -0,0 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.utils + +internal class S3TransferManagerException(message: String, cause: Throwable? = null) : Exception(message, cause) diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/UploadFile.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/UploadFile.kt index 0079ff18dd2..bf4bf649115 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/UploadFile.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/UploadFile.kt @@ -29,14 +29,14 @@ private const val MAX_NUMBER_PARTS = 10_000L * part size will be automatically increased so that exactly 10,000 parts * are uploaded. */ -internal fun resolvePartSize(uploadFileRequest: UploadFileRequest, tm: S3TransferManager, logger: Logger): Long { - val targetNumberOfParts = uploadFileRequest.contentLength / tm.targePartSize +internal fun resolvePartSize(contentLength: Long, tm: S3TransferManager, logger: Logger): Long { + val targetNumberOfParts = contentLength / tm.partSizeBytes return if (targetNumberOfParts > MAX_NUMBER_PARTS) { - ceilDiv(uploadFileRequest.contentLength, MAX_NUMBER_PARTS).also { + ceilDiv(contentLength, MAX_NUMBER_PARTS).also { logger.debug { "Target part size is too small to meet the 10,000 S3 part limit. Increasing part size to $it" } } } else { - tm.targePartSize + tm.partSizeBytes } } diff --git a/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricsTest.kt similarity index 86% rename from hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt rename to hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricsTest.kt index 1d26b4c6fbd..ed3c5720b3f 100644 --- a/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptorTest.kt +++ b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricsTest.kt @@ -9,7 +9,6 @@ import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.runtime.http.interceptors.businessmetrics.AwsBusinessMetric import aws.sdk.kotlin.services.s3.S3Client import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials.Companion.invoke import aws.smithy.kotlin.runtime.businessmetrics.containsBusinessMetric import aws.smithy.kotlin.runtime.client.ProtocolResponseInterceptorContext import aws.smithy.kotlin.runtime.content.ByteStream @@ -20,9 +19,9 @@ import aws.smithy.kotlin.runtime.httptest.TestEngine import kotlinx.coroutines.runBlocking import kotlin.test.Test -class BusinessMetricInterceptorTest { +class S3TransferManagerBusinessMetricsTest { @Test - fun businessMetricExists(): Unit = runBlocking { + fun s3Transfer(): Unit = runBlocking { val message = "Hello World" val testInterceptor = object : HttpInterceptor { override fun readAfterTransmit(context: ProtocolResponseInterceptorContext) { @@ -36,13 +35,12 @@ class BusinessMetricInterceptorTest { interceptors += testInterceptor credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) }.use { s3Client -> - S3TransferManager.Companion { + S3TransferManager { client = s3Client }.uploadFile { bucket = "b" key = "k" body = ByteStream.fromString(message) - contentLength = message.length.toLong() } } } diff --git a/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt index c2e8713dba9..8199295bedd 100644 --- a/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt +++ b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt @@ -11,7 +11,6 @@ import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest import aws.sdk.kotlin.services.s3.model.PutObjectRequest import aws.sdk.kotlin.services.s3.model.PutObjectResponse import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials.Companion.invoke import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.httptest.TestEngine import kotlinx.coroutines.runBlocking @@ -28,7 +27,7 @@ class TransferInterceptorTest { httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) }.use { s3Client -> - S3TransferManager.Companion { + S3TransferManager { client = s3Client interceptors += object : TransferInterceptor { // Test reads @@ -44,10 +43,9 @@ class TransferInterceptorTest { // Test modifications override fun modifyBeforeTransferCompleted(context: TransferContext): TransferContext { - val newContext = context.copy() - newContext.request = CompleteMultipartUploadRequest {} - newContext.transferredBytes = message.length.toLong() * 10 - return newContext + context.request = CompleteMultipartUploadRequest {} + context.transferredBytes = message.length.toLong() * 10 + return context } override fun readAfterTransferCompleted(context: TransferContext) { @@ -59,7 +57,6 @@ class TransferInterceptorTest { bucket = "b" key = "k" body = ByteStream.fromString(message) - contentLength = message.length.toLong() } } } diff --git a/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt index fdfc545a732..893b9ab5bd7 100644 --- a/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt +++ b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt @@ -23,13 +23,12 @@ class UploadFileTest { S3Client { region = "us-west-2" }.use { s3Client -> - S3TransferManager.Companion { + S3TransferManager { client = s3Client }.uploadFile { bucket = "aoperez" key = "k" body = ByteStream.fromString(message) - contentLength = message.length.toLong() } } } @@ -43,15 +42,14 @@ class UploadFileTest { S3Client { region = "us-west-2" }.use { s3Client -> - S3TransferManager.Companion { + S3TransferManager { client = s3Client - multipartUploadThreshold = 1 - targePartSize = 5L * 1024L * 1024L // 5 MB + multipartUploadThresholdBytes = 1 + partSizeBytes = 5L * 1024L * 1024L // 5 MB }.uploadFile { bucket = "aoperez" key = "mpuK" body = ByteStream.fromInputStream(file.inputStream(), messageLength) - contentLength = messageLength } } } diff --git a/settings.gradle.kts b/settings.gradle.kts index bd31e6e53c4..665e55cfb82 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -57,7 +57,6 @@ include(":aws-runtime:aws-http") include(":hll") include(":hll:hll-codegen") include(":hll:hll-mapping-core") -include(":hll:s3-transfer-manager") include(":services") include(":tests") include(":tests:codegen") @@ -92,6 +91,12 @@ if ("dynamodb".isBootstrappedService) { logger.warn(":services:dynamodb is not bootstrapped, skipping :hll:dynamodb-mapper and subprojects") } +if ("s3".isBootstrappedService) { + include(":hll:s3-transfer-manager") +} else { + logger.warn(":services:s3 is not bootstrapped, skipping :hll:s3-transfer-manager and subprojects") +} + // Service benchmarks project val benchmarkServices = listOf( // keep this list in sync with tests/benchmarks/service-benchmarks/build.gradle.kts From a193a2f1b54f84eae4460aa4fadaacbc7e724e17 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Wed, 29 Oct 2025 22:56:24 -0400 Subject: [PATCH 10/11] feedback v2 --- hll/build.gradle.kts | 2 + .../operations/rendering/OperationRenderer.kt | 2 +- hll/hll-codegen/build.gradle.kts | 20 ++ .../sdk/kotlin/hll/codegen/model/Member.kt | 2 + .../build.gradle.kts | 51 ++++ .../S3TransferManagerSymbolProcessor.kt | 59 +++++ ...3TransferManagerSymbolProcessorProvider.kt | 15 ++ .../codegen/mappings/MappingTypes.kt | 42 +++ .../codegen/mappings/Mappings.kt | 12 + .../codegen/mappings/uploadfile/Converters.kt | 247 ++++++++++++++++++ .../codegen/mappings/uploadfile/IO.kt | 80 ++++++ .../codegen/renderers/ConversionRenderer.kt | 43 +++ .../codegen/renderers/IORenderer.kt | 77 ++++++ .../S3TransferManagerCodegenException.kt | 14 + .../s3transfermanager/codegen/utils/Utils.kt | 55 ++++ ...ols.ksp.processing.SymbolProcessorProvider | 1 + .../api/s3-transfer-manager.api | 135 +++++----- hll/s3-transfer-manager/build.gradle.kts | 84 +++++- .../s3transfermanager/S3TransferManager.kt | 247 +++--------------- .../s3transfermanager/TransferInterceptor.kt | 179 ++++++++++--- .../model/MultipartDownloadType.kt | 15 +- .../model/UploadFileRequest.kt | 145 ---------- .../model/UploadFileResponse.kt | 74 ------ .../operations/uploadfile/HelperFunctions.kt | 116 ++++++++ .../operations/uploadfile/UploadFile.kt | 78 ++++++ .../uploadfile/hooks/CompleteTransfer.kt | 49 ++++ .../uploadfile/hooks/InitiateTransfer.kt | 47 ++++ .../uploadfile/hooks/TransferBytes.kt | 226 ++++++++++++++++ .../s3transfermanager/utils/Conversions.kt | 127 --------- ...ransferManagerBusinessMetricInterceptor.kt | 2 +- ...tions.kt => S3TransferManagerException.kt} | 6 + .../hll/s3transfermanager/utils/UploadFile.kt | 140 ---------- .../operations/uploadfile/UploadFileTest.kt | 82 ++++++ ...rManagerBusinessMetricsInterceptorTest.kt} | 12 +- .../utils}/TransferInterceptorTest.kt | 60 ++++- .../hll/s3transfermanager/UploadFileTest.kt | 56 ---- settings.gradle.kts | 1 + 37 files changed, 1714 insertions(+), 889 deletions(-) create mode 100644 hll/s3-transfer-manager-codegen/build.gradle.kts create mode 100644 hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessor.kt create mode 100644 hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessorProvider.kt create mode 100644 hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/MappingTypes.kt create mode 100644 hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/Mappings.kt create mode 100644 hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadfile/Converters.kt create mode 100644 hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadfile/IO.kt create mode 100644 hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/ConversionRenderer.kt create mode 100644 hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/IORenderer.kt create mode 100644 hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/S3TransferManagerCodegenException.kt create mode 100644 hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/Utils.kt create mode 100644 hll/s3-transfer-manager-codegen/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider delete mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest.kt delete mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse.kt create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/HelperFunctions.kt create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/UploadFile.kt create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/CompleteTransfer.kt create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/InitiateTransfer.kt create mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/TransferBytes.kt delete mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Conversions.kt rename hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/{ => utils}/S3TransferManagerBusinessMetricInterceptor.kt (93%) rename hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/{Exceptions.kt => S3TransferManagerException.kt} (59%) delete mode 100644 hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/UploadFile.kt create mode 100644 hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/UploadFileTest.kt rename hll/s3-transfer-manager/{jvm/test/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricsTest.kt => common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricsInterceptorTest.kt} (84%) rename hll/s3-transfer-manager/{jvm/test/aws/sdk/kotlin/hll/s3transfermanager => common/test/aws/sdk/kotlin/hll/s3transfermanager/utils}/TransferInterceptorTest.kt (51%) delete mode 100644 hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt diff --git a/hll/build.gradle.kts b/hll/build.gradle.kts index a2e924f4cba..ad8c73d6cb8 100644 --- a/hll/build.gradle.kts +++ b/hll/build.gradle.kts @@ -112,6 +112,8 @@ val projectsToIgnore = listOf( "dynamodb-mapper-ops-codegen", "dynamodb-mapper-schema-codegen", "dynamodb-mapper-schema-generator-plugin-test", + + "s3-transfer-manager-codegen", // TODO: Disable publishing ? ).filter { it in subprojects.map { it.name }.toSet() } // Some projects may not be in the build depending on bootstrapping apiValidation { diff --git a/hll/dynamodb-mapper/dynamodb-mapper-ops-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/operations/rendering/OperationRenderer.kt b/hll/dynamodb-mapper/dynamodb-mapper-ops-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/operations/rendering/OperationRenderer.kt index 900aee48df4..7196565fac8 100644 --- a/hll/dynamodb-mapper/dynamodb-mapper-ops-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/operations/rendering/OperationRenderer.kt +++ b/hll/dynamodb-mapper/dynamodb-mapper-ops-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/operations/rendering/OperationRenderer.kt @@ -97,7 +97,7 @@ internal class OperationRenderer( DataTypeGenerator(ctx, this, operation.request).generate() blankLine() - imports += ImportDirective(operation.request.lowLevel.type, operation.request.lowLevelName) + imports += ImportDirective(operation.request.lowLevel.type, operation.request.lowLevelName) // TODO: Bookmarking so I can implement this myself openBlock("private fun #T.convert(", operation.request.type) requestMembers(MemberCodegenBehavior.Hoist) { write("#L: #T, ", name, type) } diff --git a/hll/hll-codegen/build.gradle.kts b/hll/hll-codegen/build.gradle.kts index 19c65cade4b..d01abfa1d68 100644 --- a/hll/hll-codegen/build.gradle.kts +++ b/hll/hll-codegen/build.gradle.kts @@ -1,3 +1,12 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.gradle.kotlin.dsl.withType +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + /* * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 @@ -48,3 +57,14 @@ publishing { } } } + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} + +tasks.withType { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_1_8) + } +} diff --git a/hll/hll-codegen/src/main/kotlin/aws/sdk/kotlin/hll/codegen/model/Member.kt b/hll/hll-codegen/src/main/kotlin/aws/sdk/kotlin/hll/codegen/model/Member.kt index 54d62727b03..bbde36912be 100644 --- a/hll/hll-codegen/src/main/kotlin/aws/sdk/kotlin/hll/codegen/model/Member.kt +++ b/hll/hll-codegen/src/main/kotlin/aws/sdk/kotlin/hll/codegen/model/Member.kt @@ -23,6 +23,7 @@ public data class Member( val type: Type, val mutable: Boolean = false, val attributes: Attributes = emptyAttributes(), + val kDocs: String? = null, ) { @InternalSdkApi public companion object { @@ -34,6 +35,7 @@ public data class Member( name = prop.simpleName.getShortName(), type = Type.from(prop.type), mutable = prop.isMutable, + kDocs = prop.docString, ) return ModelParsingPlugin.transform(member, ModelParsingPlugin::postProcessMember) diff --git a/hll/s3-transfer-manager-codegen/build.gradle.kts b/hll/s3-transfer-manager-codegen/build.gradle.kts new file mode 100644 index 00000000000..cdbbc37a6e9 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/build.gradle.kts @@ -0,0 +1,51 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.gradle.kotlin.dsl.withType +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +description = "S3 Transfer Manager Code Generation" +extra["displayName"] = "AWS :: SDK :: Kotlin :: HLL :: S3 Transfer Manager Codegen" +extra["moduleName"] = "aws.sdk.kotlin.hll.s3transfermanager.codegen" + +plugins { + id(libs.plugins.kotlin.jvm.get().pluginId) +} + +dependencies { + implementation(libs.ksp.api) + implementation(project(":hll:hll-codegen")) + implementation(project(":services:s3")) +} + +kotlin { + explicitApi() + sourceSets.all { + listOf( + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", + "kotlin.RequiresOptIn", + ).forEach(languageSettings::optIn) + } +} + +tasks.withType { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_1_8) + freeCompilerArgs.add("-Xjdk-release=1.8") + freeCompilerArgs.add("-opt-in=kotlin.RequiresOptIn") + } +} + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessor.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessor.kt new file mode 100644 index 00000000000..36ea5e6c6ae --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessor.kt @@ -0,0 +1,59 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen + +import aws.sdk.kotlin.hll.codegen.core.CodeGeneratorFactory +import aws.sdk.kotlin.hll.codegen.ksp.processors.HllKspProcessor +import aws.sdk.kotlin.hll.codegen.rendering.RenderContext +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.conversionMappings +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.ioMappings +import aws.sdk.kotlin.hll.s3transfermanager.codegen.renderers.ConversionRenderer +import aws.sdk.kotlin.hll.s3transfermanager.codegen.renderers.IORenderer +import com.google.devtools.ksp.processing.Resolver +import com.google.devtools.ksp.processing.SymbolProcessorEnvironment +import com.google.devtools.ksp.symbol.KSAnnotated + +internal class S3TransferManagerSymbolProcessor(environment: SymbolProcessorEnvironment) : HllKspProcessor(environment) { + val rendererName = "s3-transfer-manager-code-generator" + val codeGenerator = environment.codeGenerator + val logger = environment.logger + + override fun processImpl(resolver: Resolver): List { + val ioMappingsContext = + RenderContext( + logger, + CodeGeneratorFactory(codeGenerator, logger), + "aws.sdk.kotlin.hll.s3transfermanager.model", + rendererName, + ) + + ioMappings.forEach { mapping -> + IORenderer( + ioMappingsContext, + mapping.className, + mapping, + resolver, + ).render() + } + + val conversionMappingsContext = + RenderContext( + logger, + CodeGeneratorFactory(codeGenerator, logger), + "aws.sdk.kotlin.hll.s3transfermanager.model.utils", + rendererName, + ) + + ConversionRenderer( + conversionMappingsContext, + "Converters", // TODO: Will this override the file after each conversion ? + conversionMappings, + resolver, + ).render() + + return listOf() + } +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessorProvider.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessorProvider.kt new file mode 100644 index 00000000000..3deb764223a --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessorProvider.kt @@ -0,0 +1,15 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen + +import com.google.devtools.ksp.processing.SymbolProcessor +import com.google.devtools.ksp.processing.SymbolProcessorEnvironment +import com.google.devtools.ksp.processing.SymbolProcessorProvider + +public class S3TransferManagerSymbolProcessorProvider : SymbolProcessorProvider { + override fun create(environment: SymbolProcessorEnvironment): SymbolProcessor = + S3TransferManagerSymbolProcessor(environment) +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/MappingTypes.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/MappingTypes.kt new file mode 100644 index 00000000000..e248f58362f --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/MappingTypes.kt @@ -0,0 +1,42 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings + +import aws.sdk.kotlin.hll.codegen.model.TypeRef + +/** + * Converts one type to another + */ +internal data class ConversionMapping( + val source: TypeRef, + val destination: TypeRef, + val members: Set, + val additionalImports: List = emptyList(), + val additionalParameters: List = emptyList(), + val additionalLogic: String = "", +) + +/** + * High level S3 TM request/response from low level S3 operation members + */ +internal data class IOMapping( + val type: MappingType, + val className: String, + val sourceOperation: String, + val members: Set, +) + +internal enum class MappingType { + /** + * Maps high level operation request members to low level request members + */ + REQUEST, + + /** + * Maps high level operation response members to low level response members + */ + RESPONSE, +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/Mappings.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/Mappings.kt new file mode 100644 index 00000000000..384779082f1 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/Mappings.kt @@ -0,0 +1,12 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings + +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.uploadfile.uploadFileConversions +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.uploadfile.uploadFileIOMappings + +internal val ioMappings = uploadFileIOMappings +internal val conversionMappings = uploadFileConversions diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadfile/Converters.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadfile/Converters.kt new file mode 100644 index 00000000000..4db601d3f3b --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadfile/Converters.kt @@ -0,0 +1,247 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.uploadfile + +import aws.sdk.kotlin.hll.codegen.model.TypeRef +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.ConversionMapping + +internal val uploadFileConversions = listOf( + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "PutObjectResponse", + ), + destination = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadFileResponse", + ), + setOf( + "bucketKeyEnabled", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "checksumType", + "eTag", + "expiration", + "requestCharged", + "sseCustomerAlgorithm", + "sseCustomerKeyMd5", + "ssekmsEncryptionContext", + "ssekmsKeyId", + "serverSideEncryption", + "versionId", + ), + ), + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "CompleteMultipartUploadResponse", + ), + destination = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadFileResponse", + ), + setOf( + "bucketKeyEnabled", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "checksumType", + "eTag", + "expiration", + "requestCharged", + "ssekmsKeyId", + "serverSideEncryption", + "versionId", + ), + ), + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadFileRequest", + ), + destination = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "PutObjectRequest", + ), + setOf( + "acl", + "body", + "bucket", + "bucketKeyEnabled", + "cacheControl", + "checksumAlgorithm", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "contentDisposition", + "contentEncoding", + "contentLanguage", + "contentType", + "expectedBucketOwner", + "expires", + "grantFullControl", + "grantRead", + "grantReadAcp", + "grantWriteAcp", + "ifMatch", + "ifNoneMatch", + "key", + "metadata", + "objectLockLegalHoldStatus", + "objectLockMode", + "objectLockRetainUntilDate", + "requestPayer", + "sseCustomerAlgorithm", + "sseCustomerKey", + "sseCustomerKeyMd5", + "ssekmsEncryptionContext", + "ssekmsKeyId", + "serverSideEncryption", + "storageClass", + "tagging", + "websiteRedirectLocation", + ), + additionalLogic = "contentLength = this@toPutObjectRequest.body?.contentLength", + ), + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadFileRequest", + ), + destination = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "CreateMultipartUploadRequest", + ), + setOf( + "acl", + "bucket", + "bucketKeyEnabled", + "cacheControl", + "checksumAlgorithm", + "contentDisposition", + "contentEncoding", + "contentLanguage", + "contentType", + "expectedBucketOwner", + "expires", + "grantFullControl", + "grantRead", + "grantReadAcp", + "grantWriteAcp", + "key", + "metadata", + "objectLockLegalHoldStatus", + "objectLockMode", + "objectLockRetainUntilDate", + "requestPayer", + "sseCustomerAlgorithm", + "sseCustomerKey", + "sseCustomerKeyMd5", + "ssekmsEncryptionContext", + "ssekmsKeyId", + "serverSideEncryption", + "storageClass", + "tagging", + "websiteRedirectLocation", + ), + ), + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadFileRequest", + ), + destination = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "UploadPartRequest", + ), + setOf( + "bucket", + "checksumAlgorithm", + "expectedBucketOwner", + "key", + "requestPayer", + "sseCustomerAlgorithm", + "sseCustomerKey", + "sseCustomerKeyMd5", + ), + listOf( + TypeRef( + "aws.smithy.kotlin.runtime.io", + "SdkBuffer", + ), + TypeRef( + "aws.smithy.kotlin.runtime.io", + "SdkSource", + ), + TypeRef( + "aws.smithy.kotlin.runtime.content", + "ByteStream", + ), + ), + listOf( + "currentPart: SdkBuffer", + "currentPartNumber: Int", + "mpuUploadId: String", + ), + """ + uploadId = mpuUploadId + body = object : ByteStream.SourceStream() { + override fun readFrom(): SdkSource = currentPart + override val contentLength: Long = currentPart.size + } + partNumber = currentPartNumber + """.trimIndent(), + ), + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadFileRequest", + ), + destination = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "CompleteMultipartUploadRequest", + ), + setOf( + "bucket", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "expectedBucketOwner", + "ifMatch", + "ifNoneMatch", + "key", + "requestPayer", + "sseCustomerAlgorithm", + "sseCustomerKey", + "sseCustomerKeyMd5", + ), + listOf( + TypeRef( + "aws.sdk.kotlin.services.s3.model", + "CompletedPart", + ), + ), + listOf( + "mpuUploadId: String", + "uploadedParts: List", + ), + """ + uploadId = mpuUploadId + multipartUpload { + parts = uploadedParts + } + """.trimIndent(), + ), +) diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadfile/IO.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadfile/IO.kt new file mode 100644 index 00000000000..c8efa2b93e9 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadfile/IO.kt @@ -0,0 +1,80 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.uploadfile + +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.IOMapping +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.MappingType + +internal val uploadFileIOMappings = listOf( + IOMapping( + MappingType.REQUEST, + "UploadFileRequest", + "putObject", + setOf( + "acl", + "body", + "bucket", + "bucketKeyEnabled", + "cacheControl", + "checksumAlgorithm", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "contentDisposition", + "contentEncoding", + "contentLanguage", + "contentType", + "expectedBucketOwner", + "expires", + "grantFullControl", + "grantRead", + "grantReadAcp", + "grantWriteAcp", + "ifMatch", + "ifNoneMatch", + "key", + "metadata", + "objectLockLegalHoldStatus", + "objectLockMode", + "objectLockRetainUntilDate", + "requestPayer", + "sseCustomerAlgorithm", + "sseCustomerKey", + "sseCustomerKeyMd5", + "ssekmsEncryptionContext", + "ssekmsKeyId", + "serverSideEncryption", + "storageClass", + "tagging", + "websiteRedirectLocation", + ), + ), + IOMapping( + MappingType.RESPONSE, + "UploadFileResponse", + "putObject", + setOf( + "bucketKeyEnabled", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "checksumType", + "eTag", + "expiration", + "requestCharged", + "sseCustomerAlgorithm", + "sseCustomerKeyMd5", + "ssekmsEncryptionContext", + "ssekmsKeyId", + "serverSideEncryption", + "versionId", + ), + ), +) diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/ConversionRenderer.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/ConversionRenderer.kt new file mode 100644 index 00000000000..dbea9b462b3 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/ConversionRenderer.kt @@ -0,0 +1,43 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.renderers + +import aws.sdk.kotlin.hll.codegen.core.ImportDirective +import aws.sdk.kotlin.hll.codegen.rendering.RenderContext +import aws.sdk.kotlin.hll.codegen.rendering.RendererBase +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.ConversionMapping +import com.google.devtools.ksp.processing.Resolver + +internal class ConversionRenderer( + ctx: RenderContext, + fileName: String, + val conversions: List, + val resolver: Resolver, +) : RendererBase(ctx, fileName) { + override fun generate() { + conversions.forEach { conversion -> + val functionName = "to${conversion.destination.shortName}" + + imports += ImportDirective(conversion.source) + imports += ImportDirective(conversion.destination) + + conversion.additionalImports.forEach { + imports += ImportDirective(it) + } + + withBlock( + "internal fun ${conversion.source.shortName}.$functionName(${conversion.additionalParameters.joinToString(", ")}): ${conversion.destination.shortName} = ${conversion.destination.shortName} {", + "}", + ) { + conversion.members.forEach { member -> + write("$member = this@$functionName.$member") + } + write(conversion.additionalLogic) + } + blankLine() + } + } +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/IORenderer.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/IORenderer.kt new file mode 100644 index 00000000000..08e019f2455 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/IORenderer.kt @@ -0,0 +1,77 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.renderers + +import aws.sdk.kotlin.hll.codegen.core.ImportDirective +import aws.sdk.kotlin.hll.codegen.model.TypeRef +import aws.sdk.kotlin.hll.codegen.rendering.RenderContext +import aws.sdk.kotlin.hll.codegen.rendering.RendererBase +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.IOMapping +import aws.sdk.kotlin.hll.s3transfermanager.codegen.utils.operationMembers +import aws.sdk.kotlin.hll.s3transfermanager.codegen.utils.renderMember +import com.google.devtools.ksp.processing.Resolver + +/** + * Renders request and response types + */ +internal class IORenderer( + ctx: RenderContext, + val className: String, + val mapping: IOMapping, + val resolver: Resolver, +) : RendererBase(ctx, className) { + override fun generate() { + val members = resolver + .operationMembers( + mapping.sourceOperation, + mapping.type, + mapping.members, + ) + + withBlock( + "public class $className private constructor(builder: Builder) {", + "}", + ) { + members.forEach { member -> + val memberType = member.type as TypeRef + + imports += ImportDirective(memberType) // Type: SomeType + memberType.genericArgs.forEach { genericArg -> + imports += ImportDirective(genericArg as TypeRef) // Type: Map + } + + member.kDocs?.let { write(it) } // FIXME: KSP isn't detecting KDocs + write( + "public val ${member.name}: ${member.type.renderMember()}? = builder.${member.name}", + ) + } + blankLine() + + withBlock( + "public companion object {", + "}", + ) { + write("public operator fun invoke(block: Builder.() -> Unit): $className = Builder().apply(block).build()") + } + blankLine() + + withBlock( + "public class Builder {", + "}", + ) { + members.forEach { member -> + write( + "public var ${member.name}: ${(member.type as TypeRef).renderMember()}? = null", + ) + } + blankLine() + + write("@PublishedApi") + write("internal fun build(): $className = $className(this)") + } + } + } +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/S3TransferManagerCodegenException.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/S3TransferManagerCodegenException.kt new file mode 100644 index 00000000000..a4d46df39b9 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/S3TransferManagerCodegenException.kt @@ -0,0 +1,14 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.utils + +/** + * Exception thrown when an error occurs during S3 transfer codegen. + * + * @param message Description of the error. + * @param cause The underlying cause of the exception, if any. + */ +internal class S3TransferManagerCodegenException(message: String, cause: Throwable? = null) : Exception(message, cause) diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/Utils.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/Utils.kt new file mode 100644 index 00000000000..aa574a93a37 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/Utils.kt @@ -0,0 +1,55 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.utils + +import aws.sdk.kotlin.hll.codegen.model.Member +import aws.sdk.kotlin.hll.codegen.model.Operation +import aws.sdk.kotlin.hll.codegen.model.Type +import aws.sdk.kotlin.hll.codegen.model.TypeRef +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.MappingType +import aws.sdk.kotlin.services.s3.S3Client +import com.google.devtools.ksp.getClassDeclarationByName +import com.google.devtools.ksp.getDeclaredFunctions +import com.google.devtools.ksp.processing.Resolver + +internal fun Resolver.operationMembers( + operationName: String, + type: MappingType, + relevantMembers: Set, +): List = + Operation.from( + this + .getClassDeclarationByName()!! + .getDeclaredFunctions() + .find { it.simpleName.getShortName().equals(operationName, ignoreCase = true) } + ?: throw S3TransferManagerCodegenException("Operation $operationName not found"), + ) + .let { + if (type == MappingType.REQUEST) { + it.request + } else { + it.response + } + } + .members + .filter { member -> + relevantMembers.any { it.equals(member.name, ignoreCase = true) } + } + +internal fun Type.renderMember(): String { + val code = StringBuilder() + code.append(this.shortName) // Map + + (this as TypeRef).genericArgs.let { args -> + if (args.isNotEmpty()) { + code.append( + args.joinToString(", ", "<", ">") { it.shortName }, // Map + ) + } + } + + return code.toString() +} diff --git a/hll/s3-transfer-manager-codegen/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider b/hll/s3-transfer-manager-codegen/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider new file mode 100644 index 00000000000..6526cc79fb7 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider @@ -0,0 +1 @@ +aws.sdk.kotlin.hll.s3transfermanager.codegen.S3TransferManagerSymbolProcessorProvider diff --git a/hll/s3-transfer-manager/api/s3-transfer-manager.api b/hll/s3-transfer-manager/api/s3-transfer-manager.api index fe4b3332c5c..cb8f989238b 100644 --- a/hll/s3-transfer-manager/api/s3-transfer-manager.api +++ b/hll/s3-transfer-manager/api/s3-transfer-manager.api @@ -1,8 +1,10 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager { public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Companion; - public synthetic fun (Laws/sdk/kotlin/services/s3/S3Client;JJLaws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType;Ljava/util/List;Lkotlin/jvm/internal/DefaultConstructorMarker;)V + public synthetic fun (Laws/sdk/kotlin/services/s3/S3Client;Laws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Builder;Lkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun getClient ()Laws/sdk/kotlin/services/s3/S3Client; public final fun getInterceptors ()Ljava/util/List; + public final fun getMaxConcurrentPartUploads ()I + public final fun getMaxInMemoryParts ()I public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType; public final fun getMultipartUploadThresholdBytes ()J public final fun getPartSizeBytes ()J @@ -12,93 +14,79 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager { public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Builder { public fun ()V - public final fun getClient ()Laws/sdk/kotlin/services/s3/S3Client; public final fun getInterceptors ()Ljava/util/List; + public final fun getMaxConcurrentPartUploads ()I + public final fun getMaxInMemoryParts ()I public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType; public final fun getMultipartUploadThresholdBytes ()J public final fun getPartSizeBytes ()J - public final fun setClient (Laws/sdk/kotlin/services/s3/S3Client;)V public final fun setInterceptors (Ljava/util/List;)V + public final fun setMaxConcurrentPartUploads (I)V + public final fun setMaxInMemoryParts (I)V public final fun setMultipartDownloadType (Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType;)V public final fun setMultipartUploadThresholdBytes (J)V public final fun setPartSizeBytes (J)V } public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Companion { - public final fun invoke (Lkotlin/jvm/functions/Function1;)Laws/sdk/kotlin/hll/s3transfermanager/S3TransferManager; -} - -public final class aws/sdk/kotlin/hll/s3transfermanager/TransferContext { - public fun ()V - public fun (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;)V - public synthetic fun (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;ILkotlin/jvm/internal/DefaultConstructorMarker;)V - public final fun component1 ()Ljava/lang/Object; - public final fun component2 ()Ljava/lang/Object; - public final fun component3 ()Ljava/lang/Long; - public final fun component4 ()Laws/smithy/kotlin/runtime/content/ByteStream; - public final fun component5 ()Ljava/lang/Long; - public final fun component6 ()Ljava/lang/Long; - public final fun component7 ()Ljava/lang/String; - public final fun component8 ()Ljava/lang/Long; - public final fun copy (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public static synthetic fun copy$default (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;ILjava/lang/Object;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public fun equals (Ljava/lang/Object;)Z - public final fun getCurrentBytes ()Laws/smithy/kotlin/runtime/content/ByteStream; - public final fun getCurrentFile ()Ljava/lang/String; - public final fun getRequest ()Ljava/lang/Object; - public final fun getResponse ()Ljava/lang/Object; - public final fun getTransferableBytes ()Ljava/lang/Long; - public final fun getTransferableFiles ()Ljava/lang/Long; - public final fun getTransferredBytes ()Ljava/lang/Long; - public final fun getTransferredFiles ()Ljava/lang/Long; - public fun hashCode ()I - public final fun setCurrentBytes (Laws/smithy/kotlin/runtime/content/ByteStream;)V - public final fun setCurrentFile (Ljava/lang/String;)V - public final fun setRequest (Ljava/lang/Object;)V - public final fun setResponse (Ljava/lang/Object;)V - public final fun setTransferableBytes (Ljava/lang/Long;)V - public final fun setTransferableFiles (Ljava/lang/Long;)V - public final fun setTransferredBytes (Ljava/lang/Long;)V - public final fun setTransferredFiles (Ljava/lang/Long;)V - public fun toString ()Ljava/lang/String; + public final fun invoke (Laws/sdk/kotlin/services/s3/S3Client;Lkotlin/jvm/functions/Function1;)Laws/sdk/kotlin/hll/s3transfermanager/S3TransferManager; } public abstract interface class aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor { - public fun modifyAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public fun modifyAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public fun modifyAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public fun modifyAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public fun modifyBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public fun modifyBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public fun modifyBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public fun modifyBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public fun readAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V - public fun readAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V - public fun readAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V - public fun readAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V - public fun readBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V - public fun readBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V - public fun readBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V - public fun readBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public fun modifyAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun modifyAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun modifyAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun modifyAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun modifyBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun modifyBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun modifyBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun modifyBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V } public final class aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor$DefaultImpls { - public static fun modifyAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public static fun modifyAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public static fun modifyAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public static fun modifyAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public static fun modifyBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public static fun modifyBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public static fun modifyBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public static fun modifyBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; - public static fun readAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V - public static fun readAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V - public static fun readAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V - public static fun readAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V - public static fun readBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V - public static fun readBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V - public static fun readBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V - public static fun readBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public static fun modifyAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun modifyAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun modifyAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun modifyAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun modifyBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun modifyBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun modifyBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun modifyBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V +} + +public abstract interface class aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext { + public abstract fun getCurrentBytes ()Laws/smithy/kotlin/runtime/content/ByteStream; + public abstract fun getCurrentFile ()Ljava/lang/String; + public abstract fun getRequest ()Ljava/lang/Object; + public abstract fun getResponse ()Ljava/lang/Object; + public abstract fun getTransferableBytes ()Ljava/lang/Long; + public abstract fun getTransferableFiles ()Ljava/lang/Long; + public abstract fun getTransferredBytes ()Ljava/lang/Long; + public abstract fun getTransferredFiles ()Ljava/lang/Long; + public abstract fun setCurrentBytes (Laws/smithy/kotlin/runtime/content/ByteStream;)V + public abstract fun setCurrentFile (Ljava/lang/String;)V + public abstract fun setRequest (Ljava/lang/Object;)V + public abstract fun setResponse (Ljava/lang/Object;)V + public abstract fun setTransferableBytes (Ljava/lang/Long;)V + public abstract fun setTransferableFiles (Ljava/lang/Long;)V + public abstract fun setTransferredBytes (Ljava/lang/Long;)V + public abstract fun setTransferredFiles (Ljava/lang/Long;)V } public abstract interface class aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType { @@ -114,7 +102,7 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/model/Range : aws/sdk/ko public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest { public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$Companion; - public synthetic fun (Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/String;Ljava/lang/Boolean;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/smithy/kotlin/runtime/time/Instant;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus;Laws/sdk/kotlin/services/s3/model/ObjectLockMode;Laws/smithy/kotlin/runtime/time/Instant;Laws/sdk/kotlin/services/s3/model/RequestPayer;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;Laws/sdk/kotlin/services/s3/model/StorageClass;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/internal/DefaultConstructorMarker;)V + public synthetic fun (Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$Builder;Lkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun getAcl ()Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl; public final fun getBody ()Laws/smithy/kotlin/runtime/content/ByteStream; public final fun getBucket ()Ljava/lang/String; @@ -242,7 +230,7 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$ public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse { public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse$Companion; - public fun (Ljava/lang/Boolean;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ChecksumType;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/RequestCharged;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;Ljava/lang/String;)V + public synthetic fun (Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse$Builder;Lkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; public final fun getChecksumCrc32 ()Ljava/lang/String; public final fun getChecksumCrc32C ()Ljava/lang/String; @@ -263,6 +251,7 @@ public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse$Builder { public fun ()V + public final fun build ()Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse; public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; public final fun getChecksumCrc32 ()Ljava/lang/String; public final fun getChecksumCrc32C ()Ljava/lang/String; diff --git a/hll/s3-transfer-manager/build.gradle.kts b/hll/s3-transfer-manager/build.gradle.kts index bb636ac3db6..903865efed7 100644 --- a/hll/s3-transfer-manager/build.gradle.kts +++ b/hll/s3-transfer-manager/build.gradle.kts @@ -3,18 +3,25 @@ * SPDX-License-Identifier: Apache-2.0 */ +import aws.sdk.kotlin.gradle.kmp.NATIVE_ENABLED +import com.google.devtools.ksp.gradle.KspTaskJvm +import com.google.devtools.ksp.gradle.KspTaskMetadata import org.gradle.kotlin.dsl.dependencies +import org.gradle.kotlin.dsl.project import org.gradle.kotlin.dsl.sourceSets - -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ +import org.gradle.kotlin.dsl.withType +import org.jetbrains.kotlin.gradle.tasks.KotlinCompilationTask +import java.nio.file.Files +import java.nio.file.StandardCopyOption description = "S3 Transfer Manager for the AWS SDK for Kotlin" extra["displayName"] = "AWS :: SDK :: Kotlin :: HLL :: S3 Transfer Manager" extra["moduleName"] = "aws.sdk.kotlin.hll.s3transfermanager" +plugins { + alias(libs.plugins.ksp) +} + kotlin { sourceSets { commonMain { @@ -23,7 +30,7 @@ kotlin { implementation(project(":services:s3")) } } - jvmTest { + commonTest { dependencies { implementation(libs.smithy.kotlin.test.jvm) implementation(libs.smithy.kotlin.testing.jvm) @@ -31,3 +38,68 @@ kotlin { } } } + +ksp { + dependencies { + ksp(project(":hll:s3-transfer-manager-codegen")) + } +} + +// This is copied from :hll:dynamodb-mapper:dynamodb-mapper. TODO: Commonize +if (project.NATIVE_ENABLED) { + // Configure KSP for multiplatform: https://kotlinlang.org/docs/ksp-multiplatform.html + // https://github.com/google/ksp/issues/963#issuecomment-1894144639 + // https://github.com/google/ksp/issues/965 + kotlin.sourceSets.commonMain { + tasks.withType { + // Wire up the generated source to the commonMain source set + kotlin.srcDir(destinationDirectory) + } + } +} else { + // FIXME This is a dirty hack for JVM-only builds which KSP doesn't consider to be "multiplatform". Explanation of + // hack follows in narrative, minimally-opinionated comments. + + // Then we need to move the generated source from jvm to common + val moveGenSrc by tasks.registering { + // Can't move src until the src is generated + dependsOn(tasks.named("kspKotlinJvm")) + + // Detecting these paths programmatically is complex; just hardcode them + val srcDir = file("build/generated/ksp/jvm/jvmMain") + val destDir = file("build/generated/ksp/common/commonMain") + + inputs.dir(srcDir) + outputs.dirs(srcDir, destDir) + + doLast { + if (destDir.exists()) { + // Clean out the existing destination, otherwise move fails + require(destDir.deleteRecursively()) { "Failed to delete $destDir before moving from $srcDir" } + } else { + // Create the destination directories, otherwise move fails + require(destDir.mkdirs()) { "Failed to create path $destDir" } + } + + Files.move(srcDir.toPath(), destDir.toPath(), StandardCopyOption.REPLACE_EXISTING) + } + } + + listOf("jvmSourcesJar", "metadataSourcesJar", "jvmProcessResources").forEach { + tasks.named(it) { + dependsOn(moveGenSrc) + } + } + + tasks.withType> { + if (this !is KspTaskJvm) { + // Ensure that any **non-KSP** compile tasks depend on the generated src move + dependsOn(moveGenSrc) + } + } + + // Finally, wire up the generated source to the commonMain source set + kotlin.sourceSets.commonMain { + kotlin.srcDir("build/generated/ksp/common/commonMain/kotlin") + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt index edc27f86449..a2911a750ce 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt @@ -9,108 +9,39 @@ import aws.sdk.kotlin.hll.s3transfermanager.model.MultipartDownloadType import aws.sdk.kotlin.hll.s3transfermanager.model.Part import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileResponse -import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException -import aws.sdk.kotlin.hll.s3transfermanager.utils.buildCompleteMultipartUploadRequest -import aws.sdk.kotlin.hll.s3transfermanager.utils.buildUploadPartRequest -import aws.sdk.kotlin.hll.s3transfermanager.utils.ceilDiv -import aws.sdk.kotlin.hll.s3transfermanager.utils.getNextPart -import aws.sdk.kotlin.hll.s3transfermanager.utils.resolvePartSize -import aws.sdk.kotlin.hll.s3transfermanager.utils.toCreateMultiPartUploadRequest -import aws.sdk.kotlin.hll.s3transfermanager.utils.toPutObjectRequest -import aws.sdk.kotlin.hll.s3transfermanager.utils.toUploadFileResponse +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.uploadFileImplementation +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerBusinessMetricInterceptor import aws.sdk.kotlin.services.s3.S3Client -import aws.sdk.kotlin.services.s3.abortMultipartUpload -import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest -import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadResponse -import aws.sdk.kotlin.services.s3.model.CompletedPart -import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest -import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadResponse -import aws.sdk.kotlin.services.s3.model.PutObjectRequest -import aws.sdk.kotlin.services.s3.model.PutObjectResponse -import aws.sdk.kotlin.services.s3.model.UploadPartRequest -import aws.sdk.kotlin.services.s3.model.UploadPartResponse import aws.sdk.kotlin.services.s3.withConfig -import aws.smithy.kotlin.runtime.content.ByteStream -import aws.smithy.kotlin.runtime.io.SdkBuffer -import aws.smithy.kotlin.runtime.telemetry.logging.logger -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope /** * High level utility for managing transfers to Amazon S3. */ -public class S3TransferManager private constructor( - public val client: S3Client, - public val partSizeBytes: Long, - public val multipartUploadThresholdBytes: Long, - public val multipartDownloadType: MultipartDownloadType, - public val interceptors: MutableList, -) { - internal var context: TransferContext = TransferContext() +public class S3TransferManager private constructor(s3Client: S3Client, builder: Builder) { + public val client: S3Client = s3Client.withConfig { interceptors += S3TransferManagerBusinessMetricInterceptor } + public val partSizeBytes: Long = builder.partSizeBytes + public val multipartUploadThresholdBytes: Long = builder.multipartUploadThresholdBytes + public val multipartDownloadType: MultipartDownloadType = builder.multipartDownloadType + public val interceptors: MutableList = builder.interceptors + public val maxInMemoryParts: Int = builder.maxInMemoryParts + public val maxConcurrentPartUploads: Int = builder.maxConcurrentPartUploads public companion object { - public operator fun invoke(block: Builder.() -> Unit): S3TransferManager = - Builder().apply(block).build() + public operator fun invoke(client: S3Client, block: Builder.() -> Unit): S3TransferManager = + Builder().apply(block).build(client) } public class Builder { - public var client: S3Client? = null + // TODO: K-docs for each one public var partSizeBytes: Long = 8_000_000 public var multipartUploadThresholdBytes: Long = 16_000_000L public var multipartDownloadType: MultipartDownloadType = Part public var interceptors: MutableList = mutableListOf() + public var maxInMemoryParts: Int = 5 + public var maxConcurrentPartUploads: Int = 5 - internal fun build(): S3TransferManager = - S3TransferManager( - client = client?.withConfig { interceptors += S3TransferManagerBusinessMetricInterceptor } ?: error("client must be set"), - partSizeBytes = partSizeBytes, - multipartUploadThresholdBytes = multipartUploadThresholdBytes, - multipartDownloadType = multipartDownloadType, - interceptors = interceptors, - ) - } - - /** - * Executes a sequence of operations around a hook. - * - * The execution flow is as follows: - * 1. Runs all interceptors scheduled to execute **before** the hook. - * 2. Executes the main hook logic. - * 3. Runs all interceptors scheduled to execute **after** the hook. - */ - private suspend fun operationHook(hook: TransferHook, block: suspend () -> Any) { - when (hook) { - is TransferInitiated -> { - interceptors.forEach { it.readBeforeTransferInitiated(context) } - interceptors.forEach { context = it.modifyBeforeTransferInitiated(context) } - block.invoke() - interceptors.forEach { it.readAfterTransferInitiated(context) } - interceptors.forEach { context = it.modifyAfterTransferInitiated(context) } - } - is BytesTransferred -> { - interceptors.forEach { it.readBeforeBytesTransferred(context) } - interceptors.forEach { context = it.modifyBeforeBytesTransferred(context) } - block.invoke() - interceptors.forEach { it.readAfterBytesTransferred(context) } - interceptors.forEach { context = it.modifyAfterBytesTransferred(context) } - } - is FileTransferred -> { - interceptors.forEach { it.readBeforeFileTransferred(context) } - interceptors.forEach { context = it.modifyBeforeFileTransferred(context) } - block.invoke() - interceptors.forEach { it.readAfterFileTransferred(context) } - interceptors.forEach { context = it.modifyAfterFileTransferred(context) } - } - is TransferCompleted -> { - interceptors.forEach { it.readBeforeTransferCompleted(context) } - interceptors.forEach { context = it.modifyBeforeTransferCompleted(context) } - block.invoke() - interceptors.forEach { it.readAfterTransferCompleted(context) } - interceptors.forEach { context = it.modifyAfterTransferCompleted(context) } - } - else -> error("TransferHook not implemented: ${hook::class.simpleName}") - } + internal fun build(client: S3Client): S3TransferManager = + S3TransferManager(client, this) } /** @@ -125,133 +56,18 @@ public class S3TransferManager private constructor( * all parts to fit within S3's limit of 10,000 parts, the part size will be * automatically increased so that exactly 10,000 parts are uploaded. */ - public suspend fun uploadFile(uploadFileRequest: UploadFileRequest): Deferred = coroutineScope { - val contentLength = uploadFileRequest.body?.contentLength ?: throw S3TransferManagerException("UploadFileRequest.body.contentLength must be set") - val multiPartUpload = contentLength >= multipartUploadThresholdBytes - val uploadedParts = mutableListOf() - lateinit var mpuUploadId: String - - val logger = coroutineContext.logger() - - /* - Handles transfer initiated hook - */ - suspend fun transferInitiated(multiPartUpload: Boolean) { - context.transferredBytes = 0L - context.transferableBytes = contentLength - context.request = if (multiPartUpload) { - uploadFileRequest.toCreateMultiPartUploadRequest() - } else { - uploadFileRequest.toPutObjectRequest() - } - operationHook(TransferInitiated) { - if (multiPartUpload) { - context.response = client.createMultipartUpload(context.request as CreateMultipartUploadRequest) - mpuUploadId = (context.response as CreateMultipartUploadResponse).uploadId ?: throw S3TransferManagerException("Missing upload id in create multipart upload response") - } - } - } - - /* - Handles bytes transferred hook - */ - suspend fun transferBytes(multiPartUpload: Boolean) { - if (multiPartUpload) { - try { - val partSize = resolvePartSize(contentLength, this@S3TransferManager, logger) - val numberOfParts = ceilDiv(contentLength, partSize) - val partSource = when (uploadFileRequest.body) { - is ByteStream.Buffer -> uploadFileRequest.body.bytes() - is ByteStream.ChannelStream -> uploadFileRequest.body.readFrom() - is ByteStream.SourceStream -> uploadFileRequest.body.readFrom() - else -> throw S3TransferManagerException("Unhandled body type: ${uploadFileRequest.body?.let { it::class.simpleName } ?: "null"}") - } - val partBuffer = SdkBuffer() - var currentPartNumber = 1L - - while (context.transferredBytes!! < context.transferableBytes!!) { - partBuffer.getNextPart(partSource, partSize, this@S3TransferManager) - if (currentPartNumber != numberOfParts) { - if (partBuffer.size != partSize) { - throw S3TransferManagerException("Part #$currentPartNumber size mismatch detected. Expected $partSize, actual: ${partBuffer.size}") - } - } - - context.request = - buildUploadPartRequest( - uploadFileRequest, - partBuffer, - currentPartNumber, - mpuUploadId, - ) - - operationHook(BytesTransferred) { - context.response = client.uploadPart(context.request as UploadPartRequest) - context.transferredBytes = context.transferredBytes!! + partSize - } - - uploadedParts += CompletedPart { - partNumber = currentPartNumber.toInt() - eTag = (context.response as UploadPartResponse).eTag - } - currentPartNumber += 1 - } - - if (uploadedParts.size != numberOfParts.toInt()) { - throw S3TransferManagerException("The number of uploaded parts does not match the expected count. Expected $numberOfParts, actual: ${uploadedParts.size}") - } - } catch (uploadPartThrowable: Throwable) { - try { - client.abortMultipartUpload { - bucket = uploadFileRequest.bucket - expectedBucketOwner = uploadFileRequest.expectedBucketOwner - key = uploadFileRequest.key - requestPayer = uploadFileRequest.requestPayer - uploadId = mpuUploadId - } - throw S3TransferManagerException("Multipart upload failed (ID: $mpuUploadId). One or more parts could not be uploaded", uploadPartThrowable) - } catch (abortThrowable: Throwable) { - throw S3TransferManagerException("Multipart upload failed (ID: $mpuUploadId). Unable to abort multipart upload.", abortThrowable) - } - } - } else { - operationHook(BytesTransferred) { - context.response = client.putObject(context.request as PutObjectRequest) - context.transferredBytes = context.transferableBytes - } - } - } - - /* - Handles transfer completed hook - */ - suspend fun transferComplete(multiPartUpload: Boolean) { - if (multiPartUpload) { - context.request = buildCompleteMultipartUploadRequest(uploadFileRequest, mpuUploadId, uploadedParts) - } - operationHook(TransferCompleted) { - if (multiPartUpload) { - try { - context.response = client.completeMultipartUpload(context.request as CompleteMultipartUploadRequest) - } catch (t: Throwable) { - throw S3TransferManagerException("Unable to complete multipart upload with ID: $mpuUploadId", t) - } - } - } - } - - async { - transferInitiated(multiPartUpload) - transferBytes(multiPartUpload) - transferComplete(multiPartUpload) - - when (context.response) { - is PutObjectResponse -> (context.response as PutObjectResponse).toUploadFileResponse() - is CompleteMultipartUploadResponse -> (context.response as CompleteMultipartUploadResponse).toUploadFileResponse() - else -> throw S3TransferManagerException("Unexpected response: ${context.response?.let { it::class.simpleName } ?: "null"}") - } - } - } + public suspend fun uploadFile( + uploadFileRequest: UploadFileRequest, + ): UploadFileResponse = + uploadFileImplementation( + uploadFileRequest, + client, + multipartUploadThresholdBytes, + partSizeBytes, + interceptors, + maxInMemoryParts, + maxConcurrentPartUploads, + ) /** * Uploads a byte stream to Amazon S3, automatically using multipart uploads @@ -265,5 +81,8 @@ public class S3TransferManager private constructor( * all parts to fit within S3's limit of 10,000 parts, the part size will be * automatically increased so that exactly 10,000 parts are uploaded. */ - public suspend inline fun uploadFile(crossinline block: UploadFileRequest.Builder.() -> Unit): Deferred = uploadFile(UploadFileRequest.Builder().apply(block).build()) + public suspend inline fun uploadFile( + crossinline block: UploadFileRequest.Builder.() -> Unit, + ): UploadFileResponse = + uploadFile(UploadFileRequest.Builder().apply(block).build()) } diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt index fca50b465fa..5a5b6a86c7a 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt @@ -7,52 +7,163 @@ package aws.sdk.kotlin.hll.s3transfermanager import aws.smithy.kotlin.runtime.content.ByteStream -// TODO: KDocs - -public data class TransferContext( - // Req/Resp - var request: Any? = null, - var response: Any? = null, - - // Byte transfers - var transferableBytes: Long? = null, - var currentBytes: ByteStream? = null, - var transferredBytes: Long? = null, - - // File transfers - var transferableFiles: Long? = null, - var currentFile: String? = null, - var transferredFiles: Long? = null, -) - +/** + * A transfer interceptor allows peeking into the progress + * and context of an [S3TransferManager] transfer at a certain point of time using hooks. + * Also allows modifying a transfer in progress using [TransferInterceptorContext] parameters such as [TransferInterceptorContext.response]. + * + * Terminology: + * Hook - A specific execution point of an S3 transfer manager transfer. Exposed via methods in the [TransferInterceptor]. + * Transfer context - See: [TransferInterceptorContext] + * Transfer initiated - The point in time a transfer is initiated. For example, in multipart uploads this is when a [aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest] is sent to S3. + * Bytes transferred - Any time bytes are transferred to S3 for either an upload or download + * File transferred - Any time files are transferred to S3 for either an upload or download + * Transfer completed - The point in time a transfer is completed. For example in multipart uploads this is when a [aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest] is sent to S3. + */ public interface TransferInterceptor { // Transfer initialization hooks - public fun readBeforeTransferInitiated(context: TransferContext) {} - public fun modifyBeforeTransferInitiated(context: TransferContext): TransferContext = context - public fun readAfterTransferInitiated(context: TransferContext) {} - public fun modifyAfterTransferInitiated(context: TransferContext): TransferContext = context + public fun readBeforeTransferInitiated(context: TransferInterceptorContext) {} + public fun modifyBeforeTransferInitiated(context: TransferInterceptorContext) {} + public fun readAfterTransferInitiated(context: TransferInterceptorContext) {} + public fun modifyAfterTransferInitiated(context: TransferInterceptorContext) {} // Byte transferring hooks - public fun readBeforeBytesTransferred(context: TransferContext) {} - public fun modifyBeforeBytesTransferred(context: TransferContext): TransferContext = context - public fun readAfterBytesTransferred(context: TransferContext) {} - public fun modifyAfterBytesTransferred(context: TransferContext): TransferContext = context + public fun readBeforeBytesTransferred(context: TransferInterceptorContext) {} + public fun modifyBeforeBytesTransferred(context: TransferInterceptorContext) {} + public fun readAfterBytesTransferred(context: TransferInterceptorContext) {} + public fun modifyAfterBytesTransferred(context: TransferInterceptorContext) {} // File transfer hooks - public fun readBeforeFileTransferred(context: TransferContext) {} - public fun modifyBeforeFileTransferred(context: TransferContext): TransferContext = context - public fun readAfterFileTransferred(context: TransferContext) {} - public fun modifyAfterFileTransferred(context: TransferContext): TransferContext = context + public fun readBeforeFileTransferred(context: TransferInterceptorContext) {} + public fun modifyBeforeFileTransferred(context: TransferInterceptorContext) {} + public fun readAfterFileTransferred(context: TransferInterceptorContext) {} + public fun modifyAfterFileTransferred(context: TransferInterceptorContext) {} // Transfer completion hooks - public fun readBeforeTransferCompleted(context: TransferContext) {} - public fun modifyBeforeTransferCompleted(context: TransferContext): TransferContext = context - public fun readAfterTransferCompleted(context: TransferContext) {} - public fun modifyAfterTransferCompleted(context: TransferContext): TransferContext = context + public fun readBeforeTransferCompleted(context: TransferInterceptorContext) {} + public fun modifyBeforeTransferCompleted(context: TransferInterceptorContext) {} + public fun readAfterTransferCompleted(context: TransferInterceptorContext) {} + public fun modifyAfterTransferCompleted(context: TransferInterceptorContext) {} +} + +/** + * Executes a sequence of operations around a hook. + * + * The execution flow is as follows: + * 1. Runs all interceptors scheduled to execute **before** the hook. + * 2. Executes the main hook logic. + * 3. Runs all interceptors scheduled to execute **after** the hook. + */ +internal suspend fun operationHook( + hook: TransferHook, + context: TransferContext, + interceptors: List, + block: suspend () -> Unit, +) { + when (hook) { + is TransferInitiated -> { + interceptors.forEachCatching { readBeforeTransferInitiated(context) } + interceptors.forEachCatching { modifyBeforeTransferInitiated(context) } + block.invoke() + interceptors.forEachCatching { readAfterTransferInitiated(context) } + interceptors.forEachCatching { modifyAfterTransferInitiated(context) } + } + is BytesTransferred -> { + interceptors.forEachCatching { readBeforeBytesTransferred(context) } + interceptors.forEachCatching { modifyBeforeBytesTransferred(context) } + block.invoke() + interceptors.forEachCatching { readAfterBytesTransferred(context) } + interceptors.forEachCatching { modifyAfterBytesTransferred(context) } + } + is FileTransferred -> { + interceptors.forEachCatching { readBeforeFileTransferred(context) } + interceptors.forEachCatching { modifyBeforeFileTransferred(context) } + block.invoke() + interceptors.forEachCatching { readAfterFileTransferred(context) } + interceptors.forEachCatching { modifyAfterFileTransferred(context) } + } + is TransferCompleted -> { + interceptors.forEachCatching { readBeforeTransferCompleted(context) } + interceptors.forEachCatching { modifyBeforeTransferCompleted(context) } + block.invoke() + interceptors.forEachCatching { readAfterTransferCompleted(context) } + interceptors.forEachCatching { modifyAfterTransferCompleted(context) } + } + else -> { + error("TransferHook not implemented: ${hook::class.simpleName}") + } + } +} + +/** + * Executes an action for each [TransferInterceptor]. + * Collects all exceptions, if any, and finally throws the first one with the others suppressed. + */ +private fun List.forEachCatching( + action: TransferInterceptor.() -> Unit, +) { + var exception: Exception? = null + + this.forEach { + try { + it.action() + } catch (e: Exception) { + if (exception == null) { + exception = e + } else { + exception.addSuppressed(e) + } + } + } + + exception?.let { throw it } } +/** + * Describes a type of hook that is used during an [S3TransferManager] transfer + */ internal interface TransferHook internal object TransferInitiated : TransferHook internal object BytesTransferred : TransferHook internal object FileTransferred : TransferHook internal object TransferCompleted : TransferHook + +/** + * The context around an [S3TransferManager] transfer. + * Used to track transfer progress or to modify in progress transfers, such as low level requests/responses from S3. + */ +public interface TransferInterceptorContext { + // Req/Resp + public var request: Any? + public var response: Any? + + // Byte transfers + public var transferableBytes: Long? + public var currentBytes: ByteStream? + public var transferredBytes: Long? + + // File transfers + public var transferableFiles: Long? + public var currentFile: String? + public var transferredFiles: Long? +} + +/** + * Concrete implementation of [TransferInterceptorContext]. + * Used internally by the [S3TransferManager]. + */ +internal data class TransferContext( + // Req/Resp + override var request: Any? = null, + override var response: Any? = null, + + // Byte transfers + override var transferableBytes: Long? = null, + override var currentBytes: ByteStream? = null, + override var transferredBytes: Long? = null, + + // File transfers + override var transferableFiles: Long? = null, + override var currentFile: String? = null, + override var transferredFiles: Long? = null, +) : TransferInterceptorContext diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt index 4b872c292da..a58911474fb 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt @@ -5,8 +5,19 @@ package aws.sdk.kotlin.hll.s3transfermanager.model -// TODO: KDocs - +/** + * Defines the strategy used for multipart downloads in [aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager]. + * + * A multipart download can either be performed by specifying byte ranges or by requesting individual parts. + */ public sealed interface MultipartDownloadType + +/** + * Download specific byte ranges from an object. + */ public object Range : MultipartDownloadType + +/** + * Download individual parts of an object as defined by the multipart upload structure. + */ public object Part : MultipartDownloadType diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest.kt deleted file mode 100644 index 041f781dab8..00000000000 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest.kt +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package aws.sdk.kotlin.hll.s3transfermanager.model - -import aws.sdk.kotlin.services.s3.model.ChecksumAlgorithm -import aws.sdk.kotlin.services.s3.model.ObjectCannedAcl -import aws.sdk.kotlin.services.s3.model.ObjectLockLegalHoldStatus -import aws.sdk.kotlin.services.s3.model.ObjectLockMode -import aws.sdk.kotlin.services.s3.model.RequestPayer -import aws.sdk.kotlin.services.s3.model.ServerSideEncryption -import aws.sdk.kotlin.services.s3.model.StorageClass -import aws.smithy.kotlin.runtime.content.ByteStream -import aws.smithy.kotlin.runtime.time.Instant - -public class UploadFileRequest private constructor( - public val acl: ObjectCannedAcl?, - public val body: ByteStream?, - public val bucket: String?, - public val bucketKeyEnabled: Boolean?, - public val cacheControl: String?, - public val checksumAlgorithm: ChecksumAlgorithm?, - public val checksumCrc32: String?, - public val checksumCrc32C: String?, - public val checksumCrc64Nvme: String?, - public val checksumSha1: String?, - public val checksumSha256: String?, - public val contentDisposition: String?, - public val contentEncoding: String?, - public val contentLanguage: String?, - public val contentType: String?, - public val expectedBucketOwner: String?, - public val expires: Instant?, - public val grantFullControl: String?, - public val grantRead: String?, - public val grantReadAcp: String?, - public val grantWriteAcp: String?, - public val ifMatch: String?, - public val ifNoneMatch: String?, - public val key: String?, - public val metadata: Map?, - public val objectLockLegalHoldStatus: ObjectLockLegalHoldStatus?, - public val objectLockMode: ObjectLockMode?, - public val objectLockRetainUntilDate: Instant?, - public val requestPayer: RequestPayer?, - public val sseCustomerAlgorithm: String?, - public val sseCustomerKey: String?, - public val sseCustomerKeyMd5: String?, - public val ssekmsEncryptionContext: String?, - public val ssekmsKeyId: String?, - public val serverSideEncryption: ServerSideEncryption?, - public val storageClass: StorageClass?, - public val tagging: String?, - public val websiteRedirectLocation: String?, -) { - public companion object { - public operator fun invoke(block: Builder.() -> Unit): UploadFileRequest = - Builder().apply(block).build() - } - - public class Builder { - public var acl: ObjectCannedAcl? = null - public var body: ByteStream? = null - public var bucket: String? = null - public var bucketKeyEnabled: Boolean? = null - public var cacheControl: String? = null - public var checksumAlgorithm: ChecksumAlgorithm? = null - public var checksumCrc32: String? = null - public var checksumCrc32C: String? = null - public var checksumCrc64Nvme: String? = null - public var checksumSha1: String? = null - public var checksumSha256: String? = null - public var contentDisposition: String? = null - public var contentEncoding: String? = null - public var contentLanguage: String? = null - public var contentType: String? = null - public var expectedBucketOwner: String? = null - public var expires: Instant? = null - public var grantFullControl: String? = null - public var grantRead: String? = null - public var grantReadAcp: String? = null - public var grantWriteAcp: String? = null - public var ifMatch: String? = null - public var ifNoneMatch: String? = null - public var key: String? = null - public var metadata: Map? = null - public var objectLockLegalHoldStatus: ObjectLockLegalHoldStatus? = null - public var objectLockMode: ObjectLockMode? = null - public var objectLockRetainUntilDate: Instant? = null - public var requestPayer: RequestPayer? = null - public var sseCustomerAlgorithm: String? = null - public var sseCustomerKey: String? = null - public var sseCustomerKeyMd5: String? = null - public var ssekmsEncryptionContext: String? = null - public var ssekmsKeyId: String? = null - public var serverSideEncryption: ServerSideEncryption? = null - public var storageClass: StorageClass? = null - public var tagging: String? = null - public var websiteRedirectLocation: String? = null - - public fun build(): UploadFileRequest = - UploadFileRequest( - acl, - body, - bucket, - bucketKeyEnabled, - cacheControl, - checksumAlgorithm, - checksumCrc32, - checksumCrc32C, - checksumCrc64Nvme, - checksumSha1, - checksumSha256, - contentDisposition, - contentEncoding, - contentLanguage, - contentType, - expectedBucketOwner, - expires, - grantFullControl, - grantRead, - grantReadAcp, - grantWriteAcp, - ifMatch, - ifNoneMatch, - key, - metadata, - objectLockLegalHoldStatus, - objectLockMode, - objectLockRetainUntilDate, - requestPayer, - sseCustomerAlgorithm, - sseCustomerKey, - sseCustomerKeyMd5, - ssekmsEncryptionContext, - ssekmsKeyId, - serverSideEncryption, - storageClass, - tagging, - websiteRedirectLocation, - ) - } -} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse.kt deleted file mode 100644 index 0d51e82e6c7..00000000000 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse.kt +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package aws.sdk.kotlin.hll.s3transfermanager.model - -import aws.sdk.kotlin.services.s3.model.ChecksumType -import aws.sdk.kotlin.services.s3.model.RequestCharged -import aws.sdk.kotlin.services.s3.model.ServerSideEncryption -import kotlin.String - -public class UploadFileResponse( - public val bucketKeyEnabled: Boolean?, - public val checksumCrc32: String?, - public val checksumCrc32C: String?, - public val checksumCrc64Nvme: String?, - public val checksumSha1: String?, - public val checksumSha256: String?, - public val checksumType: ChecksumType?, - public val eTag: String?, - public val expiration: String?, - public val requestCharged: RequestCharged?, - public val sseCustomerAlgorithm: String?, - public val sseCustomerKeyMd5: String?, - public val ssekmsEncryptionContext: String?, - public val ssekmsKeyId: String?, - public val serverSideEncryption: ServerSideEncryption?, - public val versionId: String?, -) { - public companion object { - public operator fun invoke(block: Builder.() -> Unit): UploadFileResponse = - Builder().apply(block).build() - } - - public class Builder { - public var bucketKeyEnabled: Boolean? = null - public var checksumCrc32: String? = null - public var checksumCrc32C: String? = null - public var checksumCrc64Nvme: String? = null - public var checksumSha1: String? = null - public var checksumSha256: String? = null - public var checksumType: ChecksumType? = null - public var eTag: String? = null - public var expiration: String? = null - public var requestCharged: RequestCharged? = null - public var sseCustomerAlgorithm: String? = null - public var sseCustomerKeyMd5: String? = null - public var ssekmsEncryptionContext: String? = null - public var ssekmsKeyId: String? = null - public var serverSideEncryption: ServerSideEncryption? = null - public var versionId: String? = null - - internal fun build(): UploadFileResponse = - UploadFileResponse( - bucketKeyEnabled, - checksumCrc32, - checksumCrc32C, - checksumCrc64Nvme, - checksumSha1, - checksumSha256, - checksumType, - eTag, - expiration, - requestCharged, - sseCustomerAlgorithm, - sseCustomerKeyMd5, - ssekmsEncryptionContext, - ssekmsKeyId, - serverSideEncryption, - versionId, - ) - } -} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/HelperFunctions.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/HelperFunctions.kt new file mode 100644 index 00000000000..8502d00672a --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/HelperFunctions.kt @@ -0,0 +1,116 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile + +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.io.SdkBuffer +import aws.smithy.kotlin.runtime.io.SdkByteReadChannel +import aws.smithy.kotlin.runtime.io.SdkSource +import aws.smithy.kotlin.runtime.io.readFully +import aws.smithy.kotlin.runtime.io.readRemaining +import aws.smithy.kotlin.runtime.telemetry.logging.Logger + +// S3 imposed limit for parts in a multipart upload +private const val MAX_NUMBER_PARTS = 10_000L + +/** + * Determines the actual part size to use for a multipart S3 upload. + * + * This function calculates the part size based on the total size + * of the file and the requested part size. If the requested part size is + * too small to allow the upload to fit within S3's 10,000-part limit, the + * part size will be automatically increased so that exactly 10,000 parts + * are uploaded. + */ +internal fun resolvePartSize(contentLength: Long, targetPartSize: Long, logger: Logger): Long { + val targetNumberOfParts = contentLength / targetPartSize + return if (targetNumberOfParts > MAX_NUMBER_PARTS) { + ceilDiv(contentLength, MAX_NUMBER_PARTS).also { + logger.warn { "Target part size is too small to meet the $MAX_NUMBER_PARTS S3 part limit. Increasing part size to $it" } + } + } else { + targetPartSize + } +} + +/** + * Determines what part source an S3 body will have: + * [ByteStream.Buffer] + * [ByteStream.ChannelStream] + * [ByteStream.SourceStream] + */ +internal fun resolvePartSource(body: ByteStream): Any = + when (body) { + is ByteStream.Buffer -> body.bytes() + is ByteStream.ChannelStream -> body.readFrom() + is ByteStream.SourceStream -> body.readFrom() + else -> + throw S3TransferManagerException( + "Unhandled body type: ${body::class.simpleName }", + ) + } + +/** + * Retrieves the bytes for the next part of a multipart upload from the given part source into a [SdkBuffer] + */ +internal suspend fun nextPartBytes( + partSource: Any, + partSize: Long, + lastPart: Boolean, + readBytes: Int, + readableBytes: Int, +): SdkBuffer { + val buffer = SdkBuffer() + + when (partSource) { + is ByteArray -> { + if (lastPart) { + buffer.write( + partSource.sliceArray(readBytes.. { + if (lastPart) { + partSource.readRemaining(buffer) + } else { + partSource.readFully(buffer, partSize) + } + } + is SdkSource -> { + if (lastPart) { + partSource.readRemaining(buffer) + } else { + partSource.readFully(buffer, partSize) + } + } + } + + return buffer +} + +/** + * Returns the ceiling of the division + * + * This means the result is rounded up to the nearest integer if the dividend is not + * evenly divisible by the divisor + */ +internal fun ceilDiv(dividend: Long, divisor: Long): Long { + val div = dividend / divisor + val remainder = dividend % divisor + return if (remainder != 0L) { + div + 1 + } else { + div + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/UploadFile.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/UploadFile.kt new file mode 100644 index 00000000000..74d304feb72 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/UploadFile.kt @@ -0,0 +1,78 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile + +import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager +import aws.sdk.kotlin.hll.s3transfermanager.TransferContext +import aws.sdk.kotlin.hll.s3transfermanager.TransferInterceptor +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileResponse +import aws.sdk.kotlin.hll.s3transfermanager.model.utils.toUploadFileResponse +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.hooks.completeTransfer +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.hooks.initiateTransfer +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.hooks.transferBytes +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadResponse +import aws.sdk.kotlin.services.s3.model.PutObjectResponse +import aws.smithy.kotlin.runtime.telemetry.TelemetryProviderContext +import aws.smithy.kotlin.runtime.telemetry.logging.logger +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.withContext + +internal suspend fun uploadFileImplementation( + uploadFileRequest: UploadFileRequest, + client: S3Client, + multipartUploadThresholdBytes: Long, + partSizeBytes: Long, + interceptors: List, + maxInMemoryParts: Int, + maxConcurrentPartUploads: Int, +): UploadFileResponse = withContext(currentCoroutineContext() + TelemetryProviderContext(client.config.telemetryProvider)) { + val contentLength = uploadFileRequest.body?.contentLength ?: throw S3TransferManagerException("Body content length must be known") + val multiPartUpload = contentLength > multipartUploadThresholdBytes + val logger = coroutineContext.logger() + val transferContext = TransferContext() + + val mpuUploadId = initiateTransfer( + multiPartUpload, + transferContext, + contentLength, + uploadFileRequest, + interceptors, + client, + ) + + val uploadedParts = transferBytes( + multiPartUpload, + contentLength, + partSizeBytes, + logger, + uploadFileRequest, + transferContext, + mpuUploadId, + interceptors, + client, + maxInMemoryParts, + maxConcurrentPartUploads, + ) + + completeTransfer( + multiPartUpload, + transferContext, + uploadFileRequest, + mpuUploadId, + uploadedParts, + interceptors, + client, + ) + + return@withContext when (transferContext.response) { + is PutObjectResponse -> (transferContext.response as PutObjectResponse).toUploadFileResponse() + is CompleteMultipartUploadResponse -> (transferContext.response as CompleteMultipartUploadResponse).toUploadFileResponse() + else -> throw S3TransferManagerException("Unexpected response type: ${transferContext.response}") + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/CompleteTransfer.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/CompleteTransfer.kt new file mode 100644 index 00000000000..5f8afb45f65 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/CompleteTransfer.kt @@ -0,0 +1,49 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.hooks + +import aws.sdk.kotlin.hll.s3transfermanager.TransferCompleted +import aws.sdk.kotlin.hll.s3transfermanager.TransferContext +import aws.sdk.kotlin.hll.s3transfermanager.TransferInterceptor +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.utils.toCompleteMultipartUploadRequest +import aws.sdk.kotlin.hll.s3transfermanager.operationHook +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.CompletedPart + +internal suspend fun completeTransfer( + multiPartUpload: Boolean, + context: TransferContext, + uploadFileRequest: UploadFileRequest, + mpuUploadId: String?, + uploadedParts: List, + interceptors: List, + client: S3Client, +) { + if (multiPartUpload) { + context.request = + uploadFileRequest.toCompleteMultipartUploadRequest( + mpuUploadId!!, + uploadedParts, + ) + } + + operationHook( + TransferCompleted, + context, + interceptors, + ) { + if (multiPartUpload) { + try { + context.response = client.completeMultipartUpload(context.request as CompleteMultipartUploadRequest) + } catch (e: Exception) { + throw S3TransferManagerException("Unable to complete multipart upload with ID: $mpuUploadId", e) + } + } + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/InitiateTransfer.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/InitiateTransfer.kt new file mode 100644 index 00000000000..b23114b7063 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/InitiateTransfer.kt @@ -0,0 +1,47 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.hooks + +import aws.sdk.kotlin.hll.s3transfermanager.TransferContext +import aws.sdk.kotlin.hll.s3transfermanager.TransferInitiated +import aws.sdk.kotlin.hll.s3transfermanager.TransferInterceptor +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.utils.toCreateMultipartUploadRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.utils.toPutObjectRequest +import aws.sdk.kotlin.hll.s3transfermanager.operationHook +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadResponse + +internal suspend fun initiateTransfer( + multiPartUpload: Boolean, + context: TransferContext, + contentLength: Long, + uploadFileRequest: UploadFileRequest, + interceptors: List, + client: S3Client, +): String? { + context.transferredBytes = 0L + context.transferableBytes = contentLength + context.request = if (multiPartUpload) { + uploadFileRequest.toCreateMultipartUploadRequest() + } else { + uploadFileRequest.toPutObjectRequest() + } + + var mpuUploadId: String? = null + operationHook( + TransferInitiated, + context, + interceptors, + ) { + if (multiPartUpload) { + context.response = client.createMultipartUpload(context.request as CreateMultipartUploadRequest) + mpuUploadId = (context.response as CreateMultipartUploadResponse).uploadId!! + } + } + return mpuUploadId +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/TransferBytes.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/TransferBytes.kt new file mode 100644 index 00000000000..e72eda752e9 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/TransferBytes.kt @@ -0,0 +1,226 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.hooks + +import aws.sdk.kotlin.hll.s3transfermanager.BytesTransferred +import aws.sdk.kotlin.hll.s3transfermanager.TransferContext +import aws.sdk.kotlin.hll.s3transfermanager.TransferInterceptor +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.utils.toUploadPartRequest +import aws.sdk.kotlin.hll.s3transfermanager.operationHook +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.ceilDiv +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.nextPartBytes +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.resolvePartSize +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.resolvePartSource +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.abortMultipartUpload +import aws.sdk.kotlin.services.s3.model.CompletedPart +import aws.sdk.kotlin.services.s3.model.PutObjectRequest +import aws.sdk.kotlin.services.s3.model.UploadPartRequest +import aws.sdk.kotlin.services.s3.model.UploadPartResponse +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.io.SdkBuffer +import aws.smithy.kotlin.runtime.io.SdkSource +import aws.smithy.kotlin.runtime.telemetry.logging.Logger +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.channels.ReceiveChannel +import kotlinx.coroutines.channels.produce +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +/** + * Represents a part in a multipart upload. + * + * @param number The part number. + * @param bytes The bytes of the part. + */ +internal data class Part( + val number: Int, + val bytes: SdkBuffer, +) + +internal suspend fun transferBytes( + multiPartUpload: Boolean, + contentLength: Long, + partSizeBytes: Long, + logger: Logger, + uploadFileRequest: UploadFileRequest, + context: TransferContext, + mpuUploadId: String?, + interceptors: List, + client: S3Client, + maxInMemoryParts: Int, + maxConcurrentPartUploads: Int, +): List = coroutineScope { + val uploadedParts = mutableListOf() + + if (multiPartUpload) { + try { + val partSize = resolvePartSize(contentLength, partSizeBytes, logger) + val numberOfParts = ceilDiv(contentLength, partSize).toInt() + val partSource = resolvePartSource(uploadFileRequest.body!!) + + val producer = produceParts( + context.transferableBytes!!, + partSource, + partSize, + numberOfParts, + maxInMemoryParts, + ) + + val mutex = Mutex() + repeat(maxConcurrentPartUploads) { + consumer( + producer, + uploadFileRequest, + mpuUploadId!!, + context, + interceptors, + client, + uploadedParts, + mutex, + ) + } + + if (uploadedParts.size != numberOfParts) { + throw S3TransferManagerException("The number of uploaded parts does not match the expected count. Expected $numberOfParts, actual: ${uploadedParts.size}") + } + } catch (uploadPartException: Exception) { + try { + client.abortMultipartUpload { + bucket = uploadFileRequest.bucket + expectedBucketOwner = uploadFileRequest.expectedBucketOwner + key = uploadFileRequest.key + requestPayer = uploadFileRequest.requestPayer + uploadId = mpuUploadId + } + throw S3TransferManagerException("Multipart upload failed (ID: $mpuUploadId). One or more parts could not be uploaded", uploadPartException) + } catch (abortException: Exception) { + throw S3TransferManagerException("Multipart upload failed (ID: $mpuUploadId). Unable to abort multipart upload.", abortException) + .also { it.addSuppressed(uploadPartException) } + } + } + } else { + operationHook( + BytesTransferred, + context, + interceptors, + ) { + context.currentBytes = uploadFileRequest.body // TODO: This will consume the bytes + context.response = client.putObject(context.request as PutObjectRequest) + context.transferredBytes = context.transferableBytes + } + } + + return@coroutineScope uploadedParts +} + +/** + * Produces multipart upload parts to be consumed by [consumer]. + * + * Uses a [kotlinx.coroutines.channels.Channel]. + * Produces until all readable bytes are read. + */ +internal fun CoroutineScope.produceParts( + readableBytes: Long, + partSource: Any, + partSize: Long, + numberOfParts: Int, + maxInMemoryParts: Int, +) = produce( + capacity = maxInMemoryParts, +) { + var readBytes = 0L + var currentPartNumber = 1 + + while (readBytes < readableBytes) { + send( + Part( + currentPartNumber, + nextPartBytes( + partSource, + partSize, + currentPartNumber == numberOfParts, + readBytes.toInt(), + readableBytes.toInt(), + ), + ).also { + if (currentPartNumber != numberOfParts && it.bytes.size != partSize) { + throw S3TransferManagerException("Part #$currentPartNumber size mismatch detected. Expected $partSize, actual: ${it.bytes.size}") + } + }, + ) + + currentPartNumber++ + readBytes += partSize + } +} + +/** + * Launches a coroutine that consumes and uploads multipart upload parts. + * + * It receives mutable shared state that may also be used by other coroutines and is + * intended for use in a [fan-out](https://kotlinlang.org/docs/channels.html#fan-out) pattern, + * where multiple consumers concurrently upload different parts of the same file. + */ +internal suspend fun consumer( + channel: ReceiveChannel, + uploadFileRequest: UploadFileRequest, + mpuUploadId: String, + context: TransferContext, + interceptors: List, + client: S3Client, + uploadedParts: MutableList, + mutex: Mutex, +) = coroutineScope { + launch { + for (part in channel) { + val partSize = part.bytes.size // Store the original size, as it will shrink when bytes are read + val localContext = context.copy() // Create a separate copy to avoid concurrent modifications + + localContext.request = uploadFileRequest.toUploadPartRequest( + part.bytes, + part.number, + mpuUploadId, + ) + localContext.currentBytes = object : ByteStream.SourceStream() { + override fun readFrom(): SdkSource = part.bytes.peek() // Peek so bytes aren’t consumed before sending + override val contentLength: Long = partSize + } + + operationHook( + BytesTransferred, + localContext, + interceptors, + ) { + localContext.response = client.uploadPart(localContext.request as UploadPartRequest) + localContext.transferredBytes = localContext.transferredBytes!! + partSize + } + + // Update shared state between coroutines + mutex.withLock { + context.request = localContext.request + context.response = localContext.response + context.transferableBytes = localContext.transferableBytes + context.currentBytes = localContext.currentBytes + context.transferredBytes = context.transferredBytes!! + partSize // Don't use transferredBytes from local context as it might be out of date + context.transferableFiles = localContext.transferableFiles + context.currentFile = localContext.currentFile + context.transferredFiles = localContext.transferredFiles + + uploadedParts.add( + CompletedPart { + partNumber = part.number + eTag = (localContext.response as UploadPartResponse).eTag + }, + ) + } + } + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Conversions.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Conversions.kt deleted file mode 100644 index 14b6b442466..00000000000 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Conversions.kt +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package aws.sdk.kotlin.hll.s3transfermanager.utils - -import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest -import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileResponse -import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadResponse -import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest -import aws.sdk.kotlin.services.s3.model.PutObjectRequest -import aws.sdk.kotlin.services.s3.model.PutObjectResponse - -internal fun PutObjectResponse.toUploadFileResponse(): UploadFileResponse = - UploadFileResponse { - bucketKeyEnabled = this@toUploadFileResponse.bucketKeyEnabled - checksumCrc32 = this@toUploadFileResponse.checksumCrc32 - checksumCrc32C = this@toUploadFileResponse.checksumCrc32C - checksumCrc64Nvme = this@toUploadFileResponse.checksumCrc64Nvme - checksumSha1 = this@toUploadFileResponse.checksumSha1 - checksumSha256 = this@toUploadFileResponse.checksumSha256 - checksumType = this@toUploadFileResponse.checksumType - eTag = this@toUploadFileResponse.eTag - expiration = this@toUploadFileResponse.expiration - requestCharged = this@toUploadFileResponse.requestCharged - sseCustomerAlgorithm = this@toUploadFileResponse.sseCustomerAlgorithm - sseCustomerKeyMd5 = this@toUploadFileResponse.sseCustomerKeyMd5 - ssekmsEncryptionContext = this@toUploadFileResponse.ssekmsEncryptionContext - ssekmsKeyId = this@toUploadFileResponse.ssekmsKeyId - serverSideEncryption = this@toUploadFileResponse.serverSideEncryption - versionId = this@toUploadFileResponse.versionId - } - -internal fun CompleteMultipartUploadResponse.toUploadFileResponse(): UploadFileResponse = - UploadFileResponse { - bucketKeyEnabled = this@toUploadFileResponse.bucketKeyEnabled - checksumCrc32 = this@toUploadFileResponse.checksumCrc32 - checksumCrc32C = this@toUploadFileResponse.checksumCrc32C - checksumCrc64Nvme = this@toUploadFileResponse.checksumCrc64Nvme - checksumSha1 = this@toUploadFileResponse.checksumSha1 - checksumSha256 = this@toUploadFileResponse.checksumSha256 - checksumType = this@toUploadFileResponse.checksumType - eTag = this@toUploadFileResponse.eTag - expiration = this@toUploadFileResponse.expiration - requestCharged = this@toUploadFileResponse.requestCharged - ssekmsKeyId = this@toUploadFileResponse.ssekmsKeyId - serverSideEncryption = this@toUploadFileResponse.serverSideEncryption - versionId = this@toUploadFileResponse.versionId - } - -internal fun UploadFileRequest.toPutObjectRequest(): PutObjectRequest = - PutObjectRequest { - acl = this@toPutObjectRequest.acl - body = this@toPutObjectRequest.body - bucket = this@toPutObjectRequest.bucket - bucketKeyEnabled = this@toPutObjectRequest.bucketKeyEnabled - cacheControl = this@toPutObjectRequest.cacheControl - checksumAlgorithm = this@toPutObjectRequest.checksumAlgorithm - checksumCrc32 = this@toPutObjectRequest.checksumCrc32 - checksumCrc32C = this@toPutObjectRequest.checksumCrc32C - checksumCrc64Nvme = this@toPutObjectRequest.checksumCrc64Nvme - checksumSha1 = this@toPutObjectRequest.checksumSha1 - checksumSha256 = this@toPutObjectRequest.checksumSha256 - contentDisposition = this@toPutObjectRequest.contentDisposition - contentEncoding = this@toPutObjectRequest.contentEncoding - contentLanguage = this@toPutObjectRequest.contentLanguage - contentLength = this@toPutObjectRequest.body?.contentLength - contentType = this@toPutObjectRequest.contentType - expectedBucketOwner = this@toPutObjectRequest.expectedBucketOwner - expires = this@toPutObjectRequest.expires - grantFullControl = this@toPutObjectRequest.grantFullControl - grantRead = this@toPutObjectRequest.grantRead - grantReadAcp = this@toPutObjectRequest.grantReadAcp - grantWriteAcp = this@toPutObjectRequest.grantWriteAcp - ifMatch = this@toPutObjectRequest.ifMatch - ifNoneMatch = this@toPutObjectRequest.ifNoneMatch - key = this@toPutObjectRequest.key - metadata = this@toPutObjectRequest.metadata - objectLockLegalHoldStatus = this@toPutObjectRequest.objectLockLegalHoldStatus - objectLockMode = this@toPutObjectRequest.objectLockMode - objectLockRetainUntilDate = this@toPutObjectRequest.objectLockRetainUntilDate - requestPayer = this@toPutObjectRequest.requestPayer - sseCustomerAlgorithm = this@toPutObjectRequest.sseCustomerAlgorithm - sseCustomerKey = this@toPutObjectRequest.sseCustomerKey - sseCustomerKeyMd5 = this@toPutObjectRequest.sseCustomerKeyMd5 - ssekmsEncryptionContext = this@toPutObjectRequest.ssekmsEncryptionContext - ssekmsKeyId = this@toPutObjectRequest.ssekmsKeyId - serverSideEncryption = this@toPutObjectRequest.serverSideEncryption - storageClass = this@toPutObjectRequest.storageClass - tagging = this@toPutObjectRequest.tagging - websiteRedirectLocation = this@toPutObjectRequest.websiteRedirectLocation - } - -internal fun UploadFileRequest.toCreateMultiPartUploadRequest(): CreateMultipartUploadRequest = - CreateMultipartUploadRequest { - acl = this@toCreateMultiPartUploadRequest.acl - bucket = this@toCreateMultiPartUploadRequest.bucket - bucketKeyEnabled = this@toCreateMultiPartUploadRequest.bucketKeyEnabled - cacheControl = this@toCreateMultiPartUploadRequest.cacheControl - checksumAlgorithm = this@toCreateMultiPartUploadRequest.checksumAlgorithm - contentDisposition = this@toCreateMultiPartUploadRequest.contentDisposition - contentEncoding = this@toCreateMultiPartUploadRequest.contentEncoding - contentLanguage = this@toCreateMultiPartUploadRequest.contentLanguage - contentType = this@toCreateMultiPartUploadRequest.contentType - expectedBucketOwner = this@toCreateMultiPartUploadRequest.expectedBucketOwner - expires = this@toCreateMultiPartUploadRequest.expires - grantFullControl = this@toCreateMultiPartUploadRequest.grantFullControl - grantRead = this@toCreateMultiPartUploadRequest.grantRead - grantReadAcp = this@toCreateMultiPartUploadRequest.grantReadAcp - grantWriteAcp = this@toCreateMultiPartUploadRequest.grantWriteAcp - key = this@toCreateMultiPartUploadRequest.key - metadata = this@toCreateMultiPartUploadRequest.metadata - objectLockLegalHoldStatus = this@toCreateMultiPartUploadRequest.objectLockLegalHoldStatus - objectLockMode = this@toCreateMultiPartUploadRequest.objectLockMode - objectLockRetainUntilDate = this@toCreateMultiPartUploadRequest.objectLockRetainUntilDate - requestPayer = this@toCreateMultiPartUploadRequest.requestPayer - sseCustomerAlgorithm = this@toCreateMultiPartUploadRequest.sseCustomerAlgorithm - sseCustomerKey = this@toCreateMultiPartUploadRequest.sseCustomerKey - sseCustomerKeyMd5 = this@toCreateMultiPartUploadRequest.sseCustomerKeyMd5 - ssekmsEncryptionContext = this@toCreateMultiPartUploadRequest.ssekmsEncryptionContext - ssekmsKeyId = this@toCreateMultiPartUploadRequest.ssekmsKeyId - serverSideEncryption = this@toCreateMultiPartUploadRequest.serverSideEncryption - storageClass = this@toCreateMultiPartUploadRequest.storageClass - tagging = this@toCreateMultiPartUploadRequest.tagging - websiteRedirectLocation = this@toCreateMultiPartUploadRequest.websiteRedirectLocation - } diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricInterceptor.kt similarity index 93% rename from hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricInterceptor.kt rename to hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricInterceptor.kt index 16331343cbb..1e697eaedb2 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricInterceptor.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricInterceptor.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package aws.sdk.kotlin.hll.s3transfermanager +package aws.sdk.kotlin.hll.s3transfermanager.utils import aws.sdk.kotlin.runtime.http.interceptors.businessmetrics.AwsBusinessMetric import aws.smithy.kotlin.runtime.businessmetrics.emitBusinessMetric diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Exceptions.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerException.kt similarity index 59% rename from hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Exceptions.kt rename to hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerException.kt index b4eded45c30..c7b09c6738e 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Exceptions.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerException.kt @@ -5,4 +5,10 @@ package aws.sdk.kotlin.hll.s3transfermanager.utils +/** + * Exception thrown when an error occurs during S3 transfer operations. + * + * @param message Description of the error. + * @param cause The underlying cause of the exception, if any. + */ internal class S3TransferManagerException(message: String, cause: Throwable? = null) : Exception(message, cause) diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/UploadFile.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/UploadFile.kt deleted file mode 100644 index bf4bf649115..00000000000 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/UploadFile.kt +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package aws.sdk.kotlin.hll.s3transfermanager.utils - -import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager -import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest -import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest -import aws.sdk.kotlin.services.s3.model.CompletedPart -import aws.sdk.kotlin.services.s3.model.UploadPartRequest -import aws.smithy.kotlin.runtime.content.ByteStream -import aws.smithy.kotlin.runtime.content.fromInputStream -import aws.smithy.kotlin.runtime.io.SdkBuffer -import aws.smithy.kotlin.runtime.io.SdkByteReadChannel -import aws.smithy.kotlin.runtime.io.SdkSource -import aws.smithy.kotlin.runtime.telemetry.logging.Logger - -// S3 imposed limit for parts in a multipart upload -private const val MAX_NUMBER_PARTS = 10_000L - -/** - * Determines the actual part size to use for a multipart S3 upload. - * - * This function calculates the part size based on the total size - * of the file and the requested part size. If the requested part size is - * too small to allow the upload to fit within S3's 10,000-part limit, the - * part size will be automatically increased so that exactly 10,000 parts - * are uploaded. - */ -internal fun resolvePartSize(contentLength: Long, tm: S3TransferManager, logger: Logger): Long { - val targetNumberOfParts = contentLength / tm.partSizeBytes - return if (targetNumberOfParts > MAX_NUMBER_PARTS) { - ceilDiv(contentLength, MAX_NUMBER_PARTS).also { - logger.debug { "Target part size is too small to meet the 10,000 S3 part limit. Increasing part size to $it" } - } - } else { - tm.partSizeBytes - } -} - -/** - * Retrieves the next part of a multipart upload from the given part source. - */ -internal suspend fun SdkBuffer.getNextPart(partSource: Any, partSize: Long, tm: S3TransferManager) { - when (partSource) { - is ByteArray -> { - this.write( - partSource.sliceArray( - tm.context.transferredBytes!!.toInt().. { - var readBytes = 0L - while (readBytes < partSize) { - readBytes += partSource.read(this, partSize - readBytes) - } - } - is SdkSource -> { - var readBytes = 0L - while (readBytes < partSize) { - readBytes += partSource.read(this, partSize - readBytes) - } - } - } -} - -/** - * Builds a low-level S3 upload part request from a high-level upload file request - * and data from the S3 Transfer Manager. - */ -internal fun buildUploadPartRequest( - uploadFileRequest: UploadFileRequest, - currentPart: SdkBuffer, - currentPartNumber: Long, - mpuUploadId: String, -): UploadPartRequest = - UploadPartRequest { - // From high-level request - bucket = uploadFileRequest.bucket - checksumAlgorithm = uploadFileRequest.checksumAlgorithm - expectedBucketOwner = uploadFileRequest.expectedBucketOwner - key = uploadFileRequest.key - requestPayer = uploadFileRequest.requestPayer - sseCustomerAlgorithm = uploadFileRequest.sseCustomerAlgorithm - sseCustomerKey = uploadFileRequest.sseCustomerKey - sseCustomerKeyMd5 = uploadFileRequest.sseCustomerKeyMd5 - - // From transfer manager - uploadId = mpuUploadId - body = ByteStream.fromInputStream(currentPart.inputStream(), currentPart.size) - partNumber = currentPartNumber.toInt() - } - -/** - * Builds a low-level S3 complete multipart upload request from a high-level upload file request - * and data from the S3 Transfer Manager. - */ -internal fun buildCompleteMultipartUploadRequest( - uploadFileRequest: UploadFileRequest, - mpuUploadId: String, - uploadedParts: List, -): CompleteMultipartUploadRequest = - CompleteMultipartUploadRequest { - // From high-level request - bucket = uploadFileRequest.bucket - checksumCrc32 = uploadFileRequest.checksumCrc32 - checksumCrc32C = uploadFileRequest.checksumCrc32C - checksumCrc64Nvme = uploadFileRequest.checksumCrc64Nvme - checksumSha1 = uploadFileRequest.checksumSha1 - checksumSha256 = uploadFileRequest.checksumSha256 - expectedBucketOwner = uploadFileRequest.expectedBucketOwner - ifMatch = uploadFileRequest.ifMatch - ifNoneMatch = uploadFileRequest.ifNoneMatch - key = uploadFileRequest.key - requestPayer = uploadFileRequest.requestPayer - sseCustomerAlgorithm = uploadFileRequest.sseCustomerAlgorithm - sseCustomerKey = uploadFileRequest.sseCustomerKey - sseCustomerKeyMd5 = uploadFileRequest.sseCustomerKeyMd5 - - // From transfer manager - uploadId = mpuUploadId - multipartUpload { - parts = uploadedParts - } - } - -/** - * Returns the ceiling of the division - * - * This means the result is rounded up to the nearest integer if the dividend is not - * evenly divisible by the divisor - */ -internal fun ceilDiv(dividend: Long, divisor: Long): Long { - val div = dividend / divisor - val remainder = dividend % divisor - return if (remainder != 0L) div + 1 else div -} diff --git a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/UploadFileTest.kt b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/UploadFileTest.kt new file mode 100644 index 00000000000..5dc1a4ce517 --- /dev/null +++ b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/UploadFileTest.kt @@ -0,0 +1,82 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile + +import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager +import aws.sdk.kotlin.services.s3.S3Client +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.content.asByteStream +import aws.smithy.kotlin.runtime.testing.RandomTempFile +import kotlinx.coroutines.runBlocking +import kotlin.invoke +import kotlin.test.Test + +// TODO: Setup e2e test environment - can't run these every build and in CI +class UploadFileTest { + @Test + fun singleObjectUpload(): Unit = runBlocking { + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager(s3Client) {}.uploadFile { + bucket = "aoperez" + key = "k" + body = ByteStream.fromString("Hello World") + } + } + } + + @Test + fun emptyBody(): Unit = runBlocking { + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager(s3Client) {}.uploadFile { + bucket = "aoperez" + key = "k" + body = ByteStream.fromString("") + } + } + } + + @Test + fun multipartUpload(): Unit = runBlocking { + val messageLength = 10L * 1024L * 1024L // 10 MB + val file = RandomTempFile(messageLength) + + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager(s3Client) { + multipartUploadThresholdBytes = 1 + partSizeBytes = 5L * 1024L * 1024L // 5 MB + }.uploadFile { + bucket = "aoperez" + key = "mpuK" + body = file.asByteStream() + } + } + } + + @Test + fun smallLastPart(): Unit = runBlocking { + val messageLength = 12L * 1024L * 1024L // 12 MB (last part will only be 2MB) + val file = RandomTempFile(messageLength) + + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager(s3Client) { + multipartUploadThresholdBytes = 1 + partSizeBytes = 5L * 1024L * 1024L // 5 MB + }.uploadFile { + bucket = "aoperez" + key = "mpuK" + body = file.asByteStream() + } + } + } +} diff --git a/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricsTest.kt b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricsInterceptorTest.kt similarity index 84% rename from hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricsTest.kt rename to hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricsInterceptorTest.kt index ed3c5720b3f..992d9ad46e0 100644 --- a/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricsTest.kt +++ b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricsInterceptorTest.kt @@ -3,8 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ -package aws.sdk.kotlin.hll.s3transfermanager +package aws.sdk.kotlin.hll.s3transfermanager.utils +import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.runtime.http.interceptors.businessmetrics.AwsBusinessMetric import aws.sdk.kotlin.services.s3.S3Client @@ -17,9 +18,10 @@ import aws.smithy.kotlin.runtime.http.request.HttpRequest import aws.smithy.kotlin.runtime.http.response.HttpResponse import aws.smithy.kotlin.runtime.httptest.TestEngine import kotlinx.coroutines.runBlocking +import kotlin.invoke import kotlin.test.Test -class S3TransferManagerBusinessMetricsTest { +class S3TransferManagerBusinessMetricsInterceptorTest { @Test fun s3Transfer(): Unit = runBlocking { val message = "Hello World" @@ -32,12 +34,10 @@ class S3TransferManagerBusinessMetricsTest { S3Client { region = "us-west-2" httpClient = TestEngine() - interceptors += testInterceptor + interceptors += listOf(testInterceptor) credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) }.use { s3Client -> - S3TransferManager { - client = s3Client - }.uploadFile { + S3TransferManager(s3Client) {}.uploadFile { bucket = "b" key = "k" body = ByteStream.fromString(message) diff --git a/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/TransferInterceptorTest.kt similarity index 51% rename from hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt rename to hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/TransferInterceptorTest.kt index 8199295bedd..12ebb5ba132 100644 --- a/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt +++ b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/TransferInterceptorTest.kt @@ -3,8 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -package aws.sdk.kotlin.hll.s3transfermanager +package aws.sdk.kotlin.hll.s3transfermanager.utils +import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager +import aws.sdk.kotlin.hll.s3transfermanager.TransferInterceptor +import aws.sdk.kotlin.hll.s3transfermanager.TransferInterceptorContext import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest @@ -14,8 +17,11 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.httptest.TestEngine import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.assertThrows import kotlin.collections.plusAssign +import kotlin.invoke import kotlin.test.Test +import kotlin.test.assertEquals class TransferInterceptorTest { @Test @@ -27,28 +33,24 @@ class TransferInterceptorTest { httpClient = TestEngine() credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) }.use { s3Client -> - S3TransferManager { - client = s3Client + S3TransferManager(s3Client) { interceptors += object : TransferInterceptor { // Test reads - override fun readBeforeTransferInitiated(context: TransferContext) { + override fun readBeforeTransferInitiated(context: TransferInterceptorContext) { assert(context.transferredBytes == 0L) assert(context.request is PutObjectRequest) } - - override fun readBeforeTransferCompleted(context: TransferContext) { + override fun readBeforeTransferCompleted(context: TransferInterceptorContext) { assert(context.transferredBytes == message.length.toLong()) assert(context.response is PutObjectResponse) } // Test modifications - override fun modifyBeforeTransferCompleted(context: TransferContext): TransferContext { + override fun modifyBeforeTransferCompleted(context: TransferInterceptorContext) { context.request = CompleteMultipartUploadRequest {} context.transferredBytes = message.length.toLong() * 10 - return context } - - override fun readAfterTransferCompleted(context: TransferContext) { + override fun readAfterTransferCompleted(context: TransferInterceptorContext) { assert(context.request is CompleteMultipartUploadRequest) assert(context.transferredBytes == message.length.toLong() * 10) } @@ -60,4 +62,42 @@ class TransferInterceptorTest { } } } + + @Test + fun interceptorsExceptionsAreSuppressed(): Unit = runBlocking { + val message = "Hello World" + + val exception = assertThrows { + S3Client { + region = "us-west-2" + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) + }.use { s3Client -> + S3TransferManager(s3Client) { + interceptors += listOf( + object : TransferInterceptor { + override fun readBeforeTransferInitiated(context: TransferInterceptorContext): Unit = + throw Exception("1") + }, + object : TransferInterceptor { + override fun readBeforeTransferInitiated(context: TransferInterceptorContext): Unit = + throw Exception("2") + }, + object : TransferInterceptor { + override fun readBeforeTransferInitiated(context: TransferInterceptorContext): Unit = + throw Exception("3") + }, + ) + }.uploadFile { + bucket = "b" + key = "k" + body = ByteStream.fromString(message) + } + } + } + + assertEquals(exception.message, "1") + assertEquals(exception.cause!!.suppressed[0].message, "2") + assertEquals(exception.cause!!.suppressed[1].message, "3") + } } diff --git a/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt deleted file mode 100644 index 893b9ab5bd7..00000000000 --- a/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package aws.sdk.kotlin.hll.s3transfermanager - -import aws.sdk.kotlin.services.s3.S3Client -import aws.smithy.kotlin.runtime.content.ByteStream -import aws.smithy.kotlin.runtime.content.fromInputStream -import aws.smithy.kotlin.runtime.testing.RandomTempFile -import kotlinx.coroutines.runBlocking -import kotlin.test.Ignore -import kotlin.test.Test - -// TODO: Setup e2e test environment - can't run these every build and in CI -class UploadFileTest { - @Ignore - @Test - fun singleObjectUpload(): Unit = runBlocking { - val message = "Hello World" - - S3Client { - region = "us-west-2" - }.use { s3Client -> - S3TransferManager { - client = s3Client - }.uploadFile { - bucket = "aoperez" - key = "k" - body = ByteStream.fromString(message) - } - } - } - - @Ignore - @Test - fun multiplePartUpload(): Unit = runBlocking { - val messageLength = 10L * 1024L * 1024L // 10 MB - val file = RandomTempFile(messageLength) - - S3Client { - region = "us-west-2" - }.use { s3Client -> - S3TransferManager { - client = s3Client - multipartUploadThresholdBytes = 1 - partSizeBytes = 5L * 1024L * 1024L // 5 MB - }.uploadFile { - bucket = "aoperez" - key = "mpuK" - body = ByteStream.fromInputStream(file.inputStream(), messageLength) - } - } - } -} diff --git a/settings.gradle.kts b/settings.gradle.kts index 665e55cfb82..1412cf2fbe2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -93,6 +93,7 @@ if ("dynamodb".isBootstrappedService) { if ("s3".isBootstrappedService) { include(":hll:s3-transfer-manager") + include(":hll:s3-transfer-manager-codegen") } else { logger.warn(":services:s3 is not bootstrapped, skipping :hll:s3-transfer-manager and subprojects") } From 010bd4beb1c197de7dba6b7fbc9820f5593cb847 Mon Sep 17 00:00:00 2001 From: 0marperez Date: Thu, 30 Oct 2025 00:05:31 -0400 Subject: [PATCH 11/11] add missing KDocs and self review --- .../s3transfermanager/S3TransferManager.kt | 106 ++++++++++++++---- .../operations/uploadfile/HelperFunctions.kt | 2 +- .../uploadfile/hooks/TransferBytes.kt | 7 +- 3 files changed, 90 insertions(+), 25 deletions(-) diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt index a2911a750ce..8a154b1f046 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt @@ -19,11 +19,52 @@ import aws.sdk.kotlin.services.s3.withConfig */ public class S3TransferManager private constructor(s3Client: S3Client, builder: Builder) { public val client: S3Client = s3Client.withConfig { interceptors += S3TransferManagerBusinessMetricInterceptor } + + /** + * Preferred part size for multipart uploads. + * If using this size would require more than 10,000 parts (the S3 limit), + * the smallest possible part size that results in 10,000 parts is used instead. + * + * Default to 8,000,000 bytes. + */ public val partSizeBytes: Long = builder.partSizeBytes + + /** + * Threshold size above which a file upload uses multipart upload + * instead of a single put object request. + * + * Defaults to 16,000,000 bytes. + */ public val multipartUploadThresholdBytes: Long = builder.multipartUploadThresholdBytes + + /** + * Strategy for multipart downloads, defined by [MultipartDownloadType]. + * Downloads can be performed either by specifying byte ranges or by requesting individual parts. + * + * Defaults to [Part]. + */ public val multipartDownloadType: MultipartDownloadType = builder.multipartDownloadType + + /** + * Mutable list of [TransferInterceptor]s, typically used to track transfers + * or inspect/modify low-level S3 requests. + */ public val interceptors: MutableList = builder.interceptors + + /** + * The maximum amount of parts to buffer in memory while waiting for uploads to complete. + * The actual number of parts buffered at any given time may be less than or equal but never greater. + * + * Defaults to 5. + */ public val maxInMemoryParts: Int = builder.maxInMemoryParts + + /** + * Maximum number of concurrent part uploads for a file. + * The actual number of uploads at any given time may be less than or equal but never greater. + * + * Defaults to 5. + */ public val maxConcurrentPartUploads: Int = builder.maxConcurrentPartUploads public companion object { @@ -32,12 +73,51 @@ public class S3TransferManager private constructor(s3Client: S3Client, builder: } public class Builder { - // TODO: K-docs for each one + /** + * Preferred part size for multipart uploads. + * If using this size would require more than 10,000 parts (the S3 limit), + * the smallest possible part size that results in 10,000 parts is used instead. + * + * Default to 8,000,000 bytes. + */ public var partSizeBytes: Long = 8_000_000 + + /** + * Threshold size above which a file upload uses multipart upload + * instead of a single put object request. + * + * Defaults to 16,000,000 bytes. + */ public var multipartUploadThresholdBytes: Long = 16_000_000L + + /** + * Strategy for multipart downloads, defined by [MultipartDownloadType]. + * Downloads can be performed either by specifying byte ranges or by requesting individual parts. + * + * Defaults to [Part]. + */ public var multipartDownloadType: MultipartDownloadType = Part + + /** + * Mutable list of [TransferInterceptor]s, typically used to track transfers + * or inspect/modify low-level S3 requests. + */ public var interceptors: MutableList = mutableListOf() + + /** + * The maximum amount of parts to buffer in memory while waiting for uploads to complete. + * The actual number of parts buffered at any given time may be less than or equal but never greater. + * + * Defaults to 5. + */ public var maxInMemoryParts: Int = 5 + + /** + * Maximum number of concurrent part uploads for a file. + * The actual number of uploads at any given time may be less than or equal but never greater. + * + * Defaults to 5. + */ public var maxConcurrentPartUploads: Int = 5 internal fun build(client: S3Client): S3TransferManager = @@ -45,16 +125,8 @@ public class S3TransferManager private constructor(s3Client: S3Client, builder: } /** - * Uploads a byte stream to Amazon S3, automatically using multipart uploads - * for large objects as needed. - * - * This function handles the complexity of splitting the data into parts, - * uploading each part, and completing the multipart upload. For object smaller than [multipartUploadThresholdBytes], - * a standard single-part upload is performed automatically. - * - * If the specified [partSizeBytes] for multipart uploads is too small to allow - * all parts to fit within S3's limit of 10,000 parts, the part size will be - * automatically increased so that exactly 10,000 parts are uploaded. + * Uploads a file to S3 via [aws.smithy.kotlin.runtime.content.ByteStream]. + * Uses multipart uploads with concurrent uploads if the object size is more than the configured [multipartUploadThresholdBytes]. */ public suspend fun uploadFile( uploadFileRequest: UploadFileRequest, @@ -70,16 +142,8 @@ public class S3TransferManager private constructor(s3Client: S3Client, builder: ) /** - * Uploads a byte stream to Amazon S3, automatically using multipart uploads - * for large objects as needed. - * - * This function handles the complexity of splitting the data into parts, - * uploading each part, and completing the multipart upload. For object smaller than [multipartUploadThresholdBytes], - * a standard single-part upload is performed automatically. - * - * If the specified [partSizeBytes] for multipart uploads is too small to allow - * all parts to fit within S3's limit of 10,000 parts, the part size will be - * automatically increased so that exactly 10,000 parts are uploaded. + * Uploads a file to S3 via [aws.smithy.kotlin.runtime.content.ByteStream]. + * Uses multipart uploads with concurrent uploads if the object size is more than the configured [multipartUploadThresholdBytes]. */ public suspend inline fun uploadFile( crossinline block: UploadFileRequest.Builder.() -> Unit, diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/HelperFunctions.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/HelperFunctions.kt index 8502d00672a..13bd1265678 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/HelperFunctions.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/HelperFunctions.kt @@ -43,7 +43,7 @@ internal fun resolvePartSize(contentLength: Long, targetPartSize: Long, logger: * [ByteStream.ChannelStream] * [ByteStream.SourceStream] */ -internal fun resolvePartSource(body: ByteStream): Any = +internal fun resolveSource(body: ByteStream): Any = when (body) { is ByteStream.Buffer -> body.bytes() is ByteStream.ChannelStream -> body.readFrom() diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/TransferBytes.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/TransferBytes.kt index e72eda752e9..ce7dd72bbbe 100644 --- a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/TransferBytes.kt +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/TransferBytes.kt @@ -14,7 +14,7 @@ import aws.sdk.kotlin.hll.s3transfermanager.operationHook import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.ceilDiv import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.nextPartBytes import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.resolvePartSize -import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.resolvePartSource +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.resolveSource import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.abortMultipartUpload @@ -64,7 +64,7 @@ internal suspend fun transferBytes( try { val partSize = resolvePartSize(contentLength, partSizeBytes, logger) val numberOfParts = ceilDiv(contentLength, partSize).toInt() - val partSource = resolvePartSource(uploadFileRequest.body!!) + val partSource = resolveSource(uploadFileRequest.body!!) val producer = produceParts( context.transferableBytes!!, @@ -107,12 +107,13 @@ internal suspend fun transferBytes( } } } else { + context.currentBytes = uploadFileRequest.body + operationHook( BytesTransferred, context, interceptors, ) { - context.currentBytes = uploadFileRequest.body // TODO: This will consume the bytes context.response = client.putObject(context.request as PutObjectRequest) context.transferredBytes = context.transferableBytes }