diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index fea5bdc046c..332992d53e3 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -171,6 +171,9 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexi public final class aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric : java/lang/Enum, aws/smithy/kotlin/runtime/businessmetrics/BusinessMetric { public static final field DDB_MAPPER Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; public static final field S3_EXPRESS_BUCKET Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; + public static final field S3_TRANSFER Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; + public static final field S3_TRANSFER_DOWNLOAD_DIRECTORY Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; + public static final field S3_TRANSFER_UPLOAD_DIRECTORY Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; public static fun getEntries ()Lkotlin/enums/EnumEntries; public fun getIdentifier ()Ljava/lang/String; public fun toString ()Ljava/lang/String; diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt index 239686ee3df..bcee46a0020 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt @@ -62,6 +62,9 @@ internal fun formatMetrics(metrics: MutableSet, logger: Logger): public enum class AwsBusinessMetric(public override val identifier: String) : BusinessMetric { S3_EXPRESS_BUCKET("J"), DDB_MAPPER("d"), + S3_TRANSFER("G"), + S3_TRANSFER_UPLOAD_DIRECTORY("9"), + S3_TRANSFER_DOWNLOAD_DIRECTORY("+"), ; @InternalApi diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index bc5a6c3c571..93fa5b883f2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -92,6 +92,8 @@ smithy-kotlin-telemetry-provider-micrometer = { module = "aws.smithy.kotlin:tele smithy-kotlin-telemetry-provider-otel = { module = "aws.smithy.kotlin:telemetry-provider-otel", version.ref = "smithy-kotlin-runtime-version" } smithy-kotlin-test-suite = { module = "aws.smithy.kotlin:test-suite", version.ref = "smithy-kotlin-runtime-version" } smithy-kotlin-testing = { module = "aws.smithy.kotlin:testing", version.ref = "smithy-kotlin-runtime-version" } +smithy-kotlin-test-jvm = { module = "aws.smithy.kotlin:http-test-jvm", version.ref = "smithy-kotlin-runtime-version" } +smithy-kotlin-testing-jvm = { module = "aws.smithy.kotlin:testing-jvm", version.ref = "smithy-kotlin-runtime-version" } smithy-kotlin-codegen = { module = "software.amazon.smithy.kotlin:smithy-kotlin-codegen", version.ref = "smithy-kotlin-codegen-version" } smithy-kotlin-codegen-testutils = { module = "software.amazon.smithy.kotlin:smithy-kotlin-codegen-testutils", version.ref = "smithy-kotlin-codegen-version" } diff --git a/hll/build.gradle.kts b/hll/build.gradle.kts index e6c5661b02a..ad8c73d6cb8 100644 --- a/hll/build.gradle.kts +++ b/hll/build.gradle.kts @@ -45,7 +45,7 @@ val hllPreviewVersion = if (sdkVersion.contains("-SNAPSHOT")) { // e.g. 1.3.29-b subprojects { group = "aws.sdk.kotlin" - version = hllPreviewVersion + version = if (name == "s3-transfer-manager") sdkVersion else hllPreviewVersion // TODO Use configurePublishing when migrating to Sonatype Publisher API / JReleaser configurePublishing("aws-sdk-kotlin") } @@ -112,6 +112,8 @@ val projectsToIgnore = listOf( "dynamodb-mapper-ops-codegen", "dynamodb-mapper-schema-codegen", "dynamodb-mapper-schema-generator-plugin-test", + + "s3-transfer-manager-codegen", // TODO: Disable publishing ? ).filter { it in subprojects.map { it.name }.toSet() } // Some projects may not be in the build depending on bootstrapping apiValidation { diff --git a/hll/dynamodb-mapper/dynamodb-mapper-ops-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/operations/rendering/OperationRenderer.kt b/hll/dynamodb-mapper/dynamodb-mapper-ops-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/operations/rendering/OperationRenderer.kt index 900aee48df4..7196565fac8 100644 --- a/hll/dynamodb-mapper/dynamodb-mapper-ops-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/operations/rendering/OperationRenderer.kt +++ b/hll/dynamodb-mapper/dynamodb-mapper-ops-codegen/src/main/kotlin/aws/sdk/kotlin/hll/dynamodbmapper/codegen/operations/rendering/OperationRenderer.kt @@ -97,7 +97,7 @@ internal class OperationRenderer( DataTypeGenerator(ctx, this, operation.request).generate() blankLine() - imports += ImportDirective(operation.request.lowLevel.type, operation.request.lowLevelName) + imports += ImportDirective(operation.request.lowLevel.type, operation.request.lowLevelName) // TODO: Bookmarking so I can implement this myself openBlock("private fun #T.convert(", operation.request.type) requestMembers(MemberCodegenBehavior.Hoist) { write("#L: #T, ", name, type) } diff --git a/hll/hll-codegen/build.gradle.kts b/hll/hll-codegen/build.gradle.kts index 19c65cade4b..d01abfa1d68 100644 --- a/hll/hll-codegen/build.gradle.kts +++ b/hll/hll-codegen/build.gradle.kts @@ -1,3 +1,12 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.gradle.kotlin.dsl.withType +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + /* * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 @@ -48,3 +57,14 @@ publishing { } } } + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} + +tasks.withType { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_1_8) + } +} diff --git a/hll/hll-codegen/src/main/kotlin/aws/sdk/kotlin/hll/codegen/model/Member.kt b/hll/hll-codegen/src/main/kotlin/aws/sdk/kotlin/hll/codegen/model/Member.kt index 54d62727b03..bbde36912be 100644 --- a/hll/hll-codegen/src/main/kotlin/aws/sdk/kotlin/hll/codegen/model/Member.kt +++ b/hll/hll-codegen/src/main/kotlin/aws/sdk/kotlin/hll/codegen/model/Member.kt @@ -23,6 +23,7 @@ public data class Member( val type: Type, val mutable: Boolean = false, val attributes: Attributes = emptyAttributes(), + val kDocs: String? = null, ) { @InternalSdkApi public companion object { @@ -34,6 +35,7 @@ public data class Member( name = prop.simpleName.getShortName(), type = Type.from(prop.type), mutable = prop.isMutable, + kDocs = prop.docString, ) return ModelParsingPlugin.transform(member, ModelParsingPlugin::postProcessMember) diff --git a/hll/s3-transfer-manager-codegen/build.gradle.kts b/hll/s3-transfer-manager-codegen/build.gradle.kts new file mode 100644 index 00000000000..cdbbc37a6e9 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/build.gradle.kts @@ -0,0 +1,51 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.gradle.kotlin.dsl.withType +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +description = "S3 Transfer Manager Code Generation" +extra["displayName"] = "AWS :: SDK :: Kotlin :: HLL :: S3 Transfer Manager Codegen" +extra["moduleName"] = "aws.sdk.kotlin.hll.s3transfermanager.codegen" + +plugins { + id(libs.plugins.kotlin.jvm.get().pluginId) +} + +dependencies { + implementation(libs.ksp.api) + implementation(project(":hll:hll-codegen")) + implementation(project(":services:s3")) +} + +kotlin { + explicitApi() + sourceSets.all { + listOf( + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", + "kotlin.RequiresOptIn", + ).forEach(languageSettings::optIn) + } +} + +tasks.withType { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_1_8) + freeCompilerArgs.add("-Xjdk-release=1.8") + freeCompilerArgs.add("-opt-in=kotlin.RequiresOptIn") + } +} + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessor.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessor.kt new file mode 100644 index 00000000000..36ea5e6c6ae --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessor.kt @@ -0,0 +1,59 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen + +import aws.sdk.kotlin.hll.codegen.core.CodeGeneratorFactory +import aws.sdk.kotlin.hll.codegen.ksp.processors.HllKspProcessor +import aws.sdk.kotlin.hll.codegen.rendering.RenderContext +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.conversionMappings +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.ioMappings +import aws.sdk.kotlin.hll.s3transfermanager.codegen.renderers.ConversionRenderer +import aws.sdk.kotlin.hll.s3transfermanager.codegen.renderers.IORenderer +import com.google.devtools.ksp.processing.Resolver +import com.google.devtools.ksp.processing.SymbolProcessorEnvironment +import com.google.devtools.ksp.symbol.KSAnnotated + +internal class S3TransferManagerSymbolProcessor(environment: SymbolProcessorEnvironment) : HllKspProcessor(environment) { + val rendererName = "s3-transfer-manager-code-generator" + val codeGenerator = environment.codeGenerator + val logger = environment.logger + + override fun processImpl(resolver: Resolver): List { + val ioMappingsContext = + RenderContext( + logger, + CodeGeneratorFactory(codeGenerator, logger), + "aws.sdk.kotlin.hll.s3transfermanager.model", + rendererName, + ) + + ioMappings.forEach { mapping -> + IORenderer( + ioMappingsContext, + mapping.className, + mapping, + resolver, + ).render() + } + + val conversionMappingsContext = + RenderContext( + logger, + CodeGeneratorFactory(codeGenerator, logger), + "aws.sdk.kotlin.hll.s3transfermanager.model.utils", + rendererName, + ) + + ConversionRenderer( + conversionMappingsContext, + "Converters", // TODO: Will this override the file after each conversion ? + conversionMappings, + resolver, + ).render() + + return listOf() + } +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessorProvider.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessorProvider.kt new file mode 100644 index 00000000000..3deb764223a --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/S3TransferManagerSymbolProcessorProvider.kt @@ -0,0 +1,15 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen + +import com.google.devtools.ksp.processing.SymbolProcessor +import com.google.devtools.ksp.processing.SymbolProcessorEnvironment +import com.google.devtools.ksp.processing.SymbolProcessorProvider + +public class S3TransferManagerSymbolProcessorProvider : SymbolProcessorProvider { + override fun create(environment: SymbolProcessorEnvironment): SymbolProcessor = + S3TransferManagerSymbolProcessor(environment) +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/MappingTypes.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/MappingTypes.kt new file mode 100644 index 00000000000..e248f58362f --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/MappingTypes.kt @@ -0,0 +1,42 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings + +import aws.sdk.kotlin.hll.codegen.model.TypeRef + +/** + * Converts one type to another + */ +internal data class ConversionMapping( + val source: TypeRef, + val destination: TypeRef, + val members: Set, + val additionalImports: List = emptyList(), + val additionalParameters: List = emptyList(), + val additionalLogic: String = "", +) + +/** + * High level S3 TM request/response from low level S3 operation members + */ +internal data class IOMapping( + val type: MappingType, + val className: String, + val sourceOperation: String, + val members: Set, +) + +internal enum class MappingType { + /** + * Maps high level operation request members to low level request members + */ + REQUEST, + + /** + * Maps high level operation response members to low level response members + */ + RESPONSE, +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/Mappings.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/Mappings.kt new file mode 100644 index 00000000000..384779082f1 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/Mappings.kt @@ -0,0 +1,12 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings + +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.uploadfile.uploadFileConversions +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.uploadfile.uploadFileIOMappings + +internal val ioMappings = uploadFileIOMappings +internal val conversionMappings = uploadFileConversions diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadfile/Converters.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadfile/Converters.kt new file mode 100644 index 00000000000..4db601d3f3b --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadfile/Converters.kt @@ -0,0 +1,247 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.uploadfile + +import aws.sdk.kotlin.hll.codegen.model.TypeRef +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.ConversionMapping + +internal val uploadFileConversions = listOf( + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "PutObjectResponse", + ), + destination = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadFileResponse", + ), + setOf( + "bucketKeyEnabled", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "checksumType", + "eTag", + "expiration", + "requestCharged", + "sseCustomerAlgorithm", + "sseCustomerKeyMd5", + "ssekmsEncryptionContext", + "ssekmsKeyId", + "serverSideEncryption", + "versionId", + ), + ), + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "CompleteMultipartUploadResponse", + ), + destination = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadFileResponse", + ), + setOf( + "bucketKeyEnabled", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "checksumType", + "eTag", + "expiration", + "requestCharged", + "ssekmsKeyId", + "serverSideEncryption", + "versionId", + ), + ), + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadFileRequest", + ), + destination = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "PutObjectRequest", + ), + setOf( + "acl", + "body", + "bucket", + "bucketKeyEnabled", + "cacheControl", + "checksumAlgorithm", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "contentDisposition", + "contentEncoding", + "contentLanguage", + "contentType", + "expectedBucketOwner", + "expires", + "grantFullControl", + "grantRead", + "grantReadAcp", + "grantWriteAcp", + "ifMatch", + "ifNoneMatch", + "key", + "metadata", + "objectLockLegalHoldStatus", + "objectLockMode", + "objectLockRetainUntilDate", + "requestPayer", + "sseCustomerAlgorithm", + "sseCustomerKey", + "sseCustomerKeyMd5", + "ssekmsEncryptionContext", + "ssekmsKeyId", + "serverSideEncryption", + "storageClass", + "tagging", + "websiteRedirectLocation", + ), + additionalLogic = "contentLength = this@toPutObjectRequest.body?.contentLength", + ), + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadFileRequest", + ), + destination = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "CreateMultipartUploadRequest", + ), + setOf( + "acl", + "bucket", + "bucketKeyEnabled", + "cacheControl", + "checksumAlgorithm", + "contentDisposition", + "contentEncoding", + "contentLanguage", + "contentType", + "expectedBucketOwner", + "expires", + "grantFullControl", + "grantRead", + "grantReadAcp", + "grantWriteAcp", + "key", + "metadata", + "objectLockLegalHoldStatus", + "objectLockMode", + "objectLockRetainUntilDate", + "requestPayer", + "sseCustomerAlgorithm", + "sseCustomerKey", + "sseCustomerKeyMd5", + "ssekmsEncryptionContext", + "ssekmsKeyId", + "serverSideEncryption", + "storageClass", + "tagging", + "websiteRedirectLocation", + ), + ), + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadFileRequest", + ), + destination = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "UploadPartRequest", + ), + setOf( + "bucket", + "checksumAlgorithm", + "expectedBucketOwner", + "key", + "requestPayer", + "sseCustomerAlgorithm", + "sseCustomerKey", + "sseCustomerKeyMd5", + ), + listOf( + TypeRef( + "aws.smithy.kotlin.runtime.io", + "SdkBuffer", + ), + TypeRef( + "aws.smithy.kotlin.runtime.io", + "SdkSource", + ), + TypeRef( + "aws.smithy.kotlin.runtime.content", + "ByteStream", + ), + ), + listOf( + "currentPart: SdkBuffer", + "currentPartNumber: Int", + "mpuUploadId: String", + ), + """ + uploadId = mpuUploadId + body = object : ByteStream.SourceStream() { + override fun readFrom(): SdkSource = currentPart + override val contentLength: Long = currentPart.size + } + partNumber = currentPartNumber + """.trimIndent(), + ), + ConversionMapping( + source = TypeRef( + "aws.sdk.kotlin.hll.s3transfermanager.model", + "UploadFileRequest", + ), + destination = TypeRef( + "aws.sdk.kotlin.services.s3.model", + "CompleteMultipartUploadRequest", + ), + setOf( + "bucket", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "expectedBucketOwner", + "ifMatch", + "ifNoneMatch", + "key", + "requestPayer", + "sseCustomerAlgorithm", + "sseCustomerKey", + "sseCustomerKeyMd5", + ), + listOf( + TypeRef( + "aws.sdk.kotlin.services.s3.model", + "CompletedPart", + ), + ), + listOf( + "mpuUploadId: String", + "uploadedParts: List", + ), + """ + uploadId = mpuUploadId + multipartUpload { + parts = uploadedParts + } + """.trimIndent(), + ), +) diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadfile/IO.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadfile/IO.kt new file mode 100644 index 00000000000..c8efa2b93e9 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/mappings/uploadfile/IO.kt @@ -0,0 +1,80 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.uploadfile + +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.IOMapping +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.MappingType + +internal val uploadFileIOMappings = listOf( + IOMapping( + MappingType.REQUEST, + "UploadFileRequest", + "putObject", + setOf( + "acl", + "body", + "bucket", + "bucketKeyEnabled", + "cacheControl", + "checksumAlgorithm", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "contentDisposition", + "contentEncoding", + "contentLanguage", + "contentType", + "expectedBucketOwner", + "expires", + "grantFullControl", + "grantRead", + "grantReadAcp", + "grantWriteAcp", + "ifMatch", + "ifNoneMatch", + "key", + "metadata", + "objectLockLegalHoldStatus", + "objectLockMode", + "objectLockRetainUntilDate", + "requestPayer", + "sseCustomerAlgorithm", + "sseCustomerKey", + "sseCustomerKeyMd5", + "ssekmsEncryptionContext", + "ssekmsKeyId", + "serverSideEncryption", + "storageClass", + "tagging", + "websiteRedirectLocation", + ), + ), + IOMapping( + MappingType.RESPONSE, + "UploadFileResponse", + "putObject", + setOf( + "bucketKeyEnabled", + "checksumCrc32", + "checksumCrc32C", + "checksumCrc64Nvme", + "checksumSha1", + "checksumSha256", + "checksumType", + "eTag", + "expiration", + "requestCharged", + "sseCustomerAlgorithm", + "sseCustomerKeyMd5", + "ssekmsEncryptionContext", + "ssekmsKeyId", + "serverSideEncryption", + "versionId", + ), + ), +) diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/ConversionRenderer.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/ConversionRenderer.kt new file mode 100644 index 00000000000..dbea9b462b3 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/ConversionRenderer.kt @@ -0,0 +1,43 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.renderers + +import aws.sdk.kotlin.hll.codegen.core.ImportDirective +import aws.sdk.kotlin.hll.codegen.rendering.RenderContext +import aws.sdk.kotlin.hll.codegen.rendering.RendererBase +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.ConversionMapping +import com.google.devtools.ksp.processing.Resolver + +internal class ConversionRenderer( + ctx: RenderContext, + fileName: String, + val conversions: List, + val resolver: Resolver, +) : RendererBase(ctx, fileName) { + override fun generate() { + conversions.forEach { conversion -> + val functionName = "to${conversion.destination.shortName}" + + imports += ImportDirective(conversion.source) + imports += ImportDirective(conversion.destination) + + conversion.additionalImports.forEach { + imports += ImportDirective(it) + } + + withBlock( + "internal fun ${conversion.source.shortName}.$functionName(${conversion.additionalParameters.joinToString(", ")}): ${conversion.destination.shortName} = ${conversion.destination.shortName} {", + "}", + ) { + conversion.members.forEach { member -> + write("$member = this@$functionName.$member") + } + write(conversion.additionalLogic) + } + blankLine() + } + } +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/IORenderer.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/IORenderer.kt new file mode 100644 index 00000000000..08e019f2455 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/renderers/IORenderer.kt @@ -0,0 +1,77 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.renderers + +import aws.sdk.kotlin.hll.codegen.core.ImportDirective +import aws.sdk.kotlin.hll.codegen.model.TypeRef +import aws.sdk.kotlin.hll.codegen.rendering.RenderContext +import aws.sdk.kotlin.hll.codegen.rendering.RendererBase +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.IOMapping +import aws.sdk.kotlin.hll.s3transfermanager.codegen.utils.operationMembers +import aws.sdk.kotlin.hll.s3transfermanager.codegen.utils.renderMember +import com.google.devtools.ksp.processing.Resolver + +/** + * Renders request and response types + */ +internal class IORenderer( + ctx: RenderContext, + val className: String, + val mapping: IOMapping, + val resolver: Resolver, +) : RendererBase(ctx, className) { + override fun generate() { + val members = resolver + .operationMembers( + mapping.sourceOperation, + mapping.type, + mapping.members, + ) + + withBlock( + "public class $className private constructor(builder: Builder) {", + "}", + ) { + members.forEach { member -> + val memberType = member.type as TypeRef + + imports += ImportDirective(memberType) // Type: SomeType + memberType.genericArgs.forEach { genericArg -> + imports += ImportDirective(genericArg as TypeRef) // Type: Map + } + + member.kDocs?.let { write(it) } // FIXME: KSP isn't detecting KDocs + write( + "public val ${member.name}: ${member.type.renderMember()}? = builder.${member.name}", + ) + } + blankLine() + + withBlock( + "public companion object {", + "}", + ) { + write("public operator fun invoke(block: Builder.() -> Unit): $className = Builder().apply(block).build()") + } + blankLine() + + withBlock( + "public class Builder {", + "}", + ) { + members.forEach { member -> + write( + "public var ${member.name}: ${(member.type as TypeRef).renderMember()}? = null", + ) + } + blankLine() + + write("@PublishedApi") + write("internal fun build(): $className = $className(this)") + } + } + } +} diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/S3TransferManagerCodegenException.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/S3TransferManagerCodegenException.kt new file mode 100644 index 00000000000..a4d46df39b9 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/S3TransferManagerCodegenException.kt @@ -0,0 +1,14 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.utils + +/** + * Exception thrown when an error occurs during S3 transfer codegen. + * + * @param message Description of the error. + * @param cause The underlying cause of the exception, if any. + */ +internal class S3TransferManagerCodegenException(message: String, cause: Throwable? = null) : Exception(message, cause) diff --git a/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/Utils.kt b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/Utils.kt new file mode 100644 index 00000000000..aa574a93a37 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/kotlin/aws/sdk/kotlin/hll/s3transfermanager/codegen/utils/Utils.kt @@ -0,0 +1,55 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.codegen.utils + +import aws.sdk.kotlin.hll.codegen.model.Member +import aws.sdk.kotlin.hll.codegen.model.Operation +import aws.sdk.kotlin.hll.codegen.model.Type +import aws.sdk.kotlin.hll.codegen.model.TypeRef +import aws.sdk.kotlin.hll.s3transfermanager.codegen.mappings.MappingType +import aws.sdk.kotlin.services.s3.S3Client +import com.google.devtools.ksp.getClassDeclarationByName +import com.google.devtools.ksp.getDeclaredFunctions +import com.google.devtools.ksp.processing.Resolver + +internal fun Resolver.operationMembers( + operationName: String, + type: MappingType, + relevantMembers: Set, +): List = + Operation.from( + this + .getClassDeclarationByName()!! + .getDeclaredFunctions() + .find { it.simpleName.getShortName().equals(operationName, ignoreCase = true) } + ?: throw S3TransferManagerCodegenException("Operation $operationName not found"), + ) + .let { + if (type == MappingType.REQUEST) { + it.request + } else { + it.response + } + } + .members + .filter { member -> + relevantMembers.any { it.equals(member.name, ignoreCase = true) } + } + +internal fun Type.renderMember(): String { + val code = StringBuilder() + code.append(this.shortName) // Map + + (this as TypeRef).genericArgs.let { args -> + if (args.isNotEmpty()) { + code.append( + args.joinToString(", ", "<", ">") { it.shortName }, // Map + ) + } + } + + return code.toString() +} diff --git a/hll/s3-transfer-manager-codegen/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider b/hll/s3-transfer-manager-codegen/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider new file mode 100644 index 00000000000..6526cc79fb7 --- /dev/null +++ b/hll/s3-transfer-manager-codegen/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider @@ -0,0 +1 @@ +aws.sdk.kotlin.hll.s3transfermanager.codegen.S3TransferManagerSymbolProcessorProvider diff --git a/hll/s3-transfer-manager/api/s3-transfer-manager.api b/hll/s3-transfer-manager/api/s3-transfer-manager.api new file mode 100644 index 00000000000..cb8f989238b --- /dev/null +++ b/hll/s3-transfer-manager/api/s3-transfer-manager.api @@ -0,0 +1,292 @@ +public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager { + public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Companion; + public synthetic fun (Laws/sdk/kotlin/services/s3/S3Client;Laws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Builder;Lkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getClient ()Laws/sdk/kotlin/services/s3/S3Client; + public final fun getInterceptors ()Ljava/util/List; + public final fun getMaxConcurrentPartUploads ()I + public final fun getMaxInMemoryParts ()I + public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType; + public final fun getMultipartUploadThresholdBytes ()J + public final fun getPartSizeBytes ()J + public final fun uploadFile (Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public final fun uploadFile (Lkotlin/jvm/functions/Function1;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Builder { + public fun ()V + public final fun getInterceptors ()Ljava/util/List; + public final fun getMaxConcurrentPartUploads ()I + public final fun getMaxInMemoryParts ()I + public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType; + public final fun getMultipartUploadThresholdBytes ()J + public final fun getPartSizeBytes ()J + public final fun setInterceptors (Ljava/util/List;)V + public final fun setMaxConcurrentPartUploads (I)V + public final fun setMaxInMemoryParts (I)V + public final fun setMultipartDownloadType (Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType;)V + public final fun setMultipartUploadThresholdBytes (J)V + public final fun setPartSizeBytes (J)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Companion { + public final fun invoke (Laws/sdk/kotlin/services/s3/S3Client;Lkotlin/jvm/functions/Function1;)Laws/sdk/kotlin/hll/s3transfermanager/S3TransferManager; +} + +public abstract interface class aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor { + public fun modifyAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun modifyAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun modifyAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun modifyAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun modifyBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun modifyBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun modifyBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun modifyBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public fun readBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor$DefaultImpls { + public static fun modifyAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun modifyAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun modifyAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun modifyAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun modifyBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun modifyBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun modifyBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun modifyBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V + public static fun readBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext;)V +} + +public abstract interface class aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorContext { + public abstract fun getCurrentBytes ()Laws/smithy/kotlin/runtime/content/ByteStream; + public abstract fun getCurrentFile ()Ljava/lang/String; + public abstract fun getRequest ()Ljava/lang/Object; + public abstract fun getResponse ()Ljava/lang/Object; + public abstract fun getTransferableBytes ()Ljava/lang/Long; + public abstract fun getTransferableFiles ()Ljava/lang/Long; + public abstract fun getTransferredBytes ()Ljava/lang/Long; + public abstract fun getTransferredFiles ()Ljava/lang/Long; + public abstract fun setCurrentBytes (Laws/smithy/kotlin/runtime/content/ByteStream;)V + public abstract fun setCurrentFile (Ljava/lang/String;)V + public abstract fun setRequest (Ljava/lang/Object;)V + public abstract fun setResponse (Ljava/lang/Object;)V + public abstract fun setTransferableBytes (Ljava/lang/Long;)V + public abstract fun setTransferableFiles (Ljava/lang/Long;)V + public abstract fun setTransferredBytes (Ljava/lang/Long;)V + public abstract fun setTransferredFiles (Ljava/lang/Long;)V +} + +public abstract interface class aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType { +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/Part : aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType { + public static final field INSTANCE Laws/sdk/kotlin/hll/s3transfermanager/model/Part; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/Range : aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType { + public static final field INSTANCE Laws/sdk/kotlin/hll/s3transfermanager/model/Range; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest { + public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$Companion; + public synthetic fun (Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$Builder;Lkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getAcl ()Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl; + public final fun getBody ()Laws/smithy/kotlin/runtime/content/ByteStream; + public final fun getBucket ()Ljava/lang/String; + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getCacheControl ()Ljava/lang/String; + public final fun getChecksumAlgorithm ()Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getContentDisposition ()Ljava/lang/String; + public final fun getContentEncoding ()Ljava/lang/String; + public final fun getContentLanguage ()Ljava/lang/String; + public final fun getContentType ()Ljava/lang/String; + public final fun getExpectedBucketOwner ()Ljava/lang/String; + public final fun getExpires ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getGrantFullControl ()Ljava/lang/String; + public final fun getGrantRead ()Ljava/lang/String; + public final fun getGrantReadAcp ()Ljava/lang/String; + public final fun getGrantWriteAcp ()Ljava/lang/String; + public final fun getIfMatch ()Ljava/lang/String; + public final fun getIfNoneMatch ()Ljava/lang/String; + public final fun getKey ()Ljava/lang/String; + public final fun getMetadata ()Ljava/util/Map; + public final fun getObjectLockLegalHoldStatus ()Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus; + public final fun getObjectLockMode ()Laws/sdk/kotlin/services/s3/model/ObjectLockMode; + public final fun getObjectLockRetainUntilDate ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getRequestPayer ()Laws/sdk/kotlin/services/s3/model/RequestPayer; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKey ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getStorageClass ()Laws/sdk/kotlin/services/s3/model/StorageClass; + public final fun getTagging ()Ljava/lang/String; + public final fun getWebsiteRedirectLocation ()Ljava/lang/String; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$Builder { + public fun ()V + public final fun build ()Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest; + public final fun getAcl ()Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl; + public final fun getBody ()Laws/smithy/kotlin/runtime/content/ByteStream; + public final fun getBucket ()Ljava/lang/String; + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getCacheControl ()Ljava/lang/String; + public final fun getChecksumAlgorithm ()Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getContentDisposition ()Ljava/lang/String; + public final fun getContentEncoding ()Ljava/lang/String; + public final fun getContentLanguage ()Ljava/lang/String; + public final fun getContentType ()Ljava/lang/String; + public final fun getExpectedBucketOwner ()Ljava/lang/String; + public final fun getExpires ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getGrantFullControl ()Ljava/lang/String; + public final fun getGrantRead ()Ljava/lang/String; + public final fun getGrantReadAcp ()Ljava/lang/String; + public final fun getGrantWriteAcp ()Ljava/lang/String; + public final fun getIfMatch ()Ljava/lang/String; + public final fun getIfNoneMatch ()Ljava/lang/String; + public final fun getKey ()Ljava/lang/String; + public final fun getMetadata ()Ljava/util/Map; + public final fun getObjectLockLegalHoldStatus ()Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus; + public final fun getObjectLockMode ()Laws/sdk/kotlin/services/s3/model/ObjectLockMode; + public final fun getObjectLockRetainUntilDate ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getRequestPayer ()Laws/sdk/kotlin/services/s3/model/RequestPayer; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKey ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getStorageClass ()Laws/sdk/kotlin/services/s3/model/StorageClass; + public final fun getTagging ()Ljava/lang/String; + public final fun getWebsiteRedirectLocation ()Ljava/lang/String; + public final fun setAcl (Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl;)V + public final fun setBody (Laws/smithy/kotlin/runtime/content/ByteStream;)V + public final fun setBucket (Ljava/lang/String;)V + public final fun setBucketKeyEnabled (Ljava/lang/Boolean;)V + public final fun setCacheControl (Ljava/lang/String;)V + public final fun setChecksumAlgorithm (Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm;)V + public final fun setChecksumCrc32 (Ljava/lang/String;)V + public final fun setChecksumCrc32C (Ljava/lang/String;)V + public final fun setChecksumCrc64Nvme (Ljava/lang/String;)V + public final fun setChecksumSha1 (Ljava/lang/String;)V + public final fun setChecksumSha256 (Ljava/lang/String;)V + public final fun setContentDisposition (Ljava/lang/String;)V + public final fun setContentEncoding (Ljava/lang/String;)V + public final fun setContentLanguage (Ljava/lang/String;)V + public final fun setContentType (Ljava/lang/String;)V + public final fun setExpectedBucketOwner (Ljava/lang/String;)V + public final fun setExpires (Laws/smithy/kotlin/runtime/time/Instant;)V + public final fun setGrantFullControl (Ljava/lang/String;)V + public final fun setGrantRead (Ljava/lang/String;)V + public final fun setGrantReadAcp (Ljava/lang/String;)V + public final fun setGrantWriteAcp (Ljava/lang/String;)V + public final fun setIfMatch (Ljava/lang/String;)V + public final fun setIfNoneMatch (Ljava/lang/String;)V + public final fun setKey (Ljava/lang/String;)V + public final fun setMetadata (Ljava/util/Map;)V + public final fun setObjectLockLegalHoldStatus (Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus;)V + public final fun setObjectLockMode (Laws/sdk/kotlin/services/s3/model/ObjectLockMode;)V + public final fun setObjectLockRetainUntilDate (Laws/smithy/kotlin/runtime/time/Instant;)V + public final fun setRequestPayer (Laws/sdk/kotlin/services/s3/model/RequestPayer;)V + public final fun setServerSideEncryption (Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;)V + public final fun setSseCustomerAlgorithm (Ljava/lang/String;)V + public final fun setSseCustomerKey (Ljava/lang/String;)V + public final fun setSseCustomerKeyMd5 (Ljava/lang/String;)V + public final fun setSsekmsEncryptionContext (Ljava/lang/String;)V + public final fun setSsekmsKeyId (Ljava/lang/String;)V + public final fun setStorageClass (Laws/sdk/kotlin/services/s3/model/StorageClass;)V + public final fun setTagging (Ljava/lang/String;)V + public final fun setWebsiteRedirectLocation (Ljava/lang/String;)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$Companion { + public final fun invoke (Lkotlin/jvm/functions/Function1;)Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse { + public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse$Companion; + public synthetic fun (Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse$Builder;Lkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getChecksumType ()Laws/sdk/kotlin/services/s3/model/ChecksumType; + public final fun getETag ()Ljava/lang/String; + public final fun getExpiration ()Ljava/lang/String; + public final fun getRequestCharged ()Laws/sdk/kotlin/services/s3/model/RequestCharged; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getVersionId ()Ljava/lang/String; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse$Builder { + public fun ()V + public final fun build ()Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse; + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getChecksumType ()Laws/sdk/kotlin/services/s3/model/ChecksumType; + public final fun getETag ()Ljava/lang/String; + public final fun getExpiration ()Ljava/lang/String; + public final fun getRequestCharged ()Laws/sdk/kotlin/services/s3/model/RequestCharged; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getVersionId ()Ljava/lang/String; + public final fun setBucketKeyEnabled (Ljava/lang/Boolean;)V + public final fun setChecksumCrc32 (Ljava/lang/String;)V + public final fun setChecksumCrc32C (Ljava/lang/String;)V + public final fun setChecksumCrc64Nvme (Ljava/lang/String;)V + public final fun setChecksumSha1 (Ljava/lang/String;)V + public final fun setChecksumSha256 (Ljava/lang/String;)V + public final fun setChecksumType (Laws/sdk/kotlin/services/s3/model/ChecksumType;)V + public final fun setETag (Ljava/lang/String;)V + public final fun setExpiration (Ljava/lang/String;)V + public final fun setRequestCharged (Laws/sdk/kotlin/services/s3/model/RequestCharged;)V + public final fun setServerSideEncryption (Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;)V + public final fun setSseCustomerAlgorithm (Ljava/lang/String;)V + public final fun setSseCustomerKeyMd5 (Ljava/lang/String;)V + public final fun setSsekmsEncryptionContext (Ljava/lang/String;)V + public final fun setSsekmsKeyId (Ljava/lang/String;)V + public final fun setVersionId (Ljava/lang/String;)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse$Companion { + public final fun invoke (Lkotlin/jvm/functions/Function1;)Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse; +} + diff --git a/hll/s3-transfer-manager/build.gradle.kts b/hll/s3-transfer-manager/build.gradle.kts new file mode 100644 index 00000000000..903865efed7 --- /dev/null +++ b/hll/s3-transfer-manager/build.gradle.kts @@ -0,0 +1,105 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import aws.sdk.kotlin.gradle.kmp.NATIVE_ENABLED +import com.google.devtools.ksp.gradle.KspTaskJvm +import com.google.devtools.ksp.gradle.KspTaskMetadata +import org.gradle.kotlin.dsl.dependencies +import org.gradle.kotlin.dsl.project +import org.gradle.kotlin.dsl.sourceSets +import org.gradle.kotlin.dsl.withType +import org.jetbrains.kotlin.gradle.tasks.KotlinCompilationTask +import java.nio.file.Files +import java.nio.file.StandardCopyOption + +description = "S3 Transfer Manager for the AWS SDK for Kotlin" +extra["displayName"] = "AWS :: SDK :: Kotlin :: HLL :: S3 Transfer Manager" +extra["moduleName"] = "aws.sdk.kotlin.hll.s3transfermanager" + +plugins { + alias(libs.plugins.ksp) +} + +kotlin { + sourceSets { + commonMain { + dependencies { + implementation(project(":aws-runtime:aws-http")) + implementation(project(":services:s3")) + } + } + commonTest { + dependencies { + implementation(libs.smithy.kotlin.test.jvm) + implementation(libs.smithy.kotlin.testing.jvm) + } + } + } +} + +ksp { + dependencies { + ksp(project(":hll:s3-transfer-manager-codegen")) + } +} + +// This is copied from :hll:dynamodb-mapper:dynamodb-mapper. TODO: Commonize +if (project.NATIVE_ENABLED) { + // Configure KSP for multiplatform: https://kotlinlang.org/docs/ksp-multiplatform.html + // https://github.com/google/ksp/issues/963#issuecomment-1894144639 + // https://github.com/google/ksp/issues/965 + kotlin.sourceSets.commonMain { + tasks.withType { + // Wire up the generated source to the commonMain source set + kotlin.srcDir(destinationDirectory) + } + } +} else { + // FIXME This is a dirty hack for JVM-only builds which KSP doesn't consider to be "multiplatform". Explanation of + // hack follows in narrative, minimally-opinionated comments. + + // Then we need to move the generated source from jvm to common + val moveGenSrc by tasks.registering { + // Can't move src until the src is generated + dependsOn(tasks.named("kspKotlinJvm")) + + // Detecting these paths programmatically is complex; just hardcode them + val srcDir = file("build/generated/ksp/jvm/jvmMain") + val destDir = file("build/generated/ksp/common/commonMain") + + inputs.dir(srcDir) + outputs.dirs(srcDir, destDir) + + doLast { + if (destDir.exists()) { + // Clean out the existing destination, otherwise move fails + require(destDir.deleteRecursively()) { "Failed to delete $destDir before moving from $srcDir" } + } else { + // Create the destination directories, otherwise move fails + require(destDir.mkdirs()) { "Failed to create path $destDir" } + } + + Files.move(srcDir.toPath(), destDir.toPath(), StandardCopyOption.REPLACE_EXISTING) + } + } + + listOf("jvmSourcesJar", "metadataSourcesJar", "jvmProcessResources").forEach { + tasks.named(it) { + dependsOn(moveGenSrc) + } + } + + tasks.withType> { + if (this !is KspTaskJvm) { + // Ensure that any **non-KSP** compile tasks depend on the generated src move + dependsOn(moveGenSrc) + } + } + + // Finally, wire up the generated source to the commonMain source set + kotlin.sourceSets.commonMain { + kotlin.srcDir("build/generated/ksp/common/commonMain/kotlin") + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt new file mode 100644 index 00000000000..8a154b1f046 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt @@ -0,0 +1,152 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager + +import aws.sdk.kotlin.hll.s3transfermanager.model.MultipartDownloadType +import aws.sdk.kotlin.hll.s3transfermanager.model.Part +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileResponse +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.uploadFileImplementation +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerBusinessMetricInterceptor +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.withConfig + +/** + * High level utility for managing transfers to Amazon S3. + */ +public class S3TransferManager private constructor(s3Client: S3Client, builder: Builder) { + public val client: S3Client = s3Client.withConfig { interceptors += S3TransferManagerBusinessMetricInterceptor } + + /** + * Preferred part size for multipart uploads. + * If using this size would require more than 10,000 parts (the S3 limit), + * the smallest possible part size that results in 10,000 parts is used instead. + * + * Default to 8,000,000 bytes. + */ + public val partSizeBytes: Long = builder.partSizeBytes + + /** + * Threshold size above which a file upload uses multipart upload + * instead of a single put object request. + * + * Defaults to 16,000,000 bytes. + */ + public val multipartUploadThresholdBytes: Long = builder.multipartUploadThresholdBytes + + /** + * Strategy for multipart downloads, defined by [MultipartDownloadType]. + * Downloads can be performed either by specifying byte ranges or by requesting individual parts. + * + * Defaults to [Part]. + */ + public val multipartDownloadType: MultipartDownloadType = builder.multipartDownloadType + + /** + * Mutable list of [TransferInterceptor]s, typically used to track transfers + * or inspect/modify low-level S3 requests. + */ + public val interceptors: MutableList = builder.interceptors + + /** + * The maximum amount of parts to buffer in memory while waiting for uploads to complete. + * The actual number of parts buffered at any given time may be less than or equal but never greater. + * + * Defaults to 5. + */ + public val maxInMemoryParts: Int = builder.maxInMemoryParts + + /** + * Maximum number of concurrent part uploads for a file. + * The actual number of uploads at any given time may be less than or equal but never greater. + * + * Defaults to 5. + */ + public val maxConcurrentPartUploads: Int = builder.maxConcurrentPartUploads + + public companion object { + public operator fun invoke(client: S3Client, block: Builder.() -> Unit): S3TransferManager = + Builder().apply(block).build(client) + } + + public class Builder { + /** + * Preferred part size for multipart uploads. + * If using this size would require more than 10,000 parts (the S3 limit), + * the smallest possible part size that results in 10,000 parts is used instead. + * + * Default to 8,000,000 bytes. + */ + public var partSizeBytes: Long = 8_000_000 + + /** + * Threshold size above which a file upload uses multipart upload + * instead of a single put object request. + * + * Defaults to 16,000,000 bytes. + */ + public var multipartUploadThresholdBytes: Long = 16_000_000L + + /** + * Strategy for multipart downloads, defined by [MultipartDownloadType]. + * Downloads can be performed either by specifying byte ranges or by requesting individual parts. + * + * Defaults to [Part]. + */ + public var multipartDownloadType: MultipartDownloadType = Part + + /** + * Mutable list of [TransferInterceptor]s, typically used to track transfers + * or inspect/modify low-level S3 requests. + */ + public var interceptors: MutableList = mutableListOf() + + /** + * The maximum amount of parts to buffer in memory while waiting for uploads to complete. + * The actual number of parts buffered at any given time may be less than or equal but never greater. + * + * Defaults to 5. + */ + public var maxInMemoryParts: Int = 5 + + /** + * Maximum number of concurrent part uploads for a file. + * The actual number of uploads at any given time may be less than or equal but never greater. + * + * Defaults to 5. + */ + public var maxConcurrentPartUploads: Int = 5 + + internal fun build(client: S3Client): S3TransferManager = + S3TransferManager(client, this) + } + + /** + * Uploads a file to S3 via [aws.smithy.kotlin.runtime.content.ByteStream]. + * Uses multipart uploads with concurrent uploads if the object size is more than the configured [multipartUploadThresholdBytes]. + */ + public suspend fun uploadFile( + uploadFileRequest: UploadFileRequest, + ): UploadFileResponse = + uploadFileImplementation( + uploadFileRequest, + client, + multipartUploadThresholdBytes, + partSizeBytes, + interceptors, + maxInMemoryParts, + maxConcurrentPartUploads, + ) + + /** + * Uploads a file to S3 via [aws.smithy.kotlin.runtime.content.ByteStream]. + * Uses multipart uploads with concurrent uploads if the object size is more than the configured [multipartUploadThresholdBytes]. + */ + public suspend inline fun uploadFile( + crossinline block: UploadFileRequest.Builder.() -> Unit, + ): UploadFileResponse = + uploadFile(UploadFileRequest.Builder().apply(block).build()) +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt new file mode 100644 index 00000000000..5a5b6a86c7a --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt @@ -0,0 +1,169 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager + +import aws.smithy.kotlin.runtime.content.ByteStream + +/** + * A transfer interceptor allows peeking into the progress + * and context of an [S3TransferManager] transfer at a certain point of time using hooks. + * Also allows modifying a transfer in progress using [TransferInterceptorContext] parameters such as [TransferInterceptorContext.response]. + * + * Terminology: + * Hook - A specific execution point of an S3 transfer manager transfer. Exposed via methods in the [TransferInterceptor]. + * Transfer context - See: [TransferInterceptorContext] + * Transfer initiated - The point in time a transfer is initiated. For example, in multipart uploads this is when a [aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest] is sent to S3. + * Bytes transferred - Any time bytes are transferred to S3 for either an upload or download + * File transferred - Any time files are transferred to S3 for either an upload or download + * Transfer completed - The point in time a transfer is completed. For example in multipart uploads this is when a [aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest] is sent to S3. + */ +public interface TransferInterceptor { + // Transfer initialization hooks + public fun readBeforeTransferInitiated(context: TransferInterceptorContext) {} + public fun modifyBeforeTransferInitiated(context: TransferInterceptorContext) {} + public fun readAfterTransferInitiated(context: TransferInterceptorContext) {} + public fun modifyAfterTransferInitiated(context: TransferInterceptorContext) {} + + // Byte transferring hooks + public fun readBeforeBytesTransferred(context: TransferInterceptorContext) {} + public fun modifyBeforeBytesTransferred(context: TransferInterceptorContext) {} + public fun readAfterBytesTransferred(context: TransferInterceptorContext) {} + public fun modifyAfterBytesTransferred(context: TransferInterceptorContext) {} + + // File transfer hooks + public fun readBeforeFileTransferred(context: TransferInterceptorContext) {} + public fun modifyBeforeFileTransferred(context: TransferInterceptorContext) {} + public fun readAfterFileTransferred(context: TransferInterceptorContext) {} + public fun modifyAfterFileTransferred(context: TransferInterceptorContext) {} + + // Transfer completion hooks + public fun readBeforeTransferCompleted(context: TransferInterceptorContext) {} + public fun modifyBeforeTransferCompleted(context: TransferInterceptorContext) {} + public fun readAfterTransferCompleted(context: TransferInterceptorContext) {} + public fun modifyAfterTransferCompleted(context: TransferInterceptorContext) {} +} + +/** + * Executes a sequence of operations around a hook. + * + * The execution flow is as follows: + * 1. Runs all interceptors scheduled to execute **before** the hook. + * 2. Executes the main hook logic. + * 3. Runs all interceptors scheduled to execute **after** the hook. + */ +internal suspend fun operationHook( + hook: TransferHook, + context: TransferContext, + interceptors: List, + block: suspend () -> Unit, +) { + when (hook) { + is TransferInitiated -> { + interceptors.forEachCatching { readBeforeTransferInitiated(context) } + interceptors.forEachCatching { modifyBeforeTransferInitiated(context) } + block.invoke() + interceptors.forEachCatching { readAfterTransferInitiated(context) } + interceptors.forEachCatching { modifyAfterTransferInitiated(context) } + } + is BytesTransferred -> { + interceptors.forEachCatching { readBeforeBytesTransferred(context) } + interceptors.forEachCatching { modifyBeforeBytesTransferred(context) } + block.invoke() + interceptors.forEachCatching { readAfterBytesTransferred(context) } + interceptors.forEachCatching { modifyAfterBytesTransferred(context) } + } + is FileTransferred -> { + interceptors.forEachCatching { readBeforeFileTransferred(context) } + interceptors.forEachCatching { modifyBeforeFileTransferred(context) } + block.invoke() + interceptors.forEachCatching { readAfterFileTransferred(context) } + interceptors.forEachCatching { modifyAfterFileTransferred(context) } + } + is TransferCompleted -> { + interceptors.forEachCatching { readBeforeTransferCompleted(context) } + interceptors.forEachCatching { modifyBeforeTransferCompleted(context) } + block.invoke() + interceptors.forEachCatching { readAfterTransferCompleted(context) } + interceptors.forEachCatching { modifyAfterTransferCompleted(context) } + } + else -> { + error("TransferHook not implemented: ${hook::class.simpleName}") + } + } +} + +/** + * Executes an action for each [TransferInterceptor]. + * Collects all exceptions, if any, and finally throws the first one with the others suppressed. + */ +private fun List.forEachCatching( + action: TransferInterceptor.() -> Unit, +) { + var exception: Exception? = null + + this.forEach { + try { + it.action() + } catch (e: Exception) { + if (exception == null) { + exception = e + } else { + exception.addSuppressed(e) + } + } + } + + exception?.let { throw it } +} + +/** + * Describes a type of hook that is used during an [S3TransferManager] transfer + */ +internal interface TransferHook +internal object TransferInitiated : TransferHook +internal object BytesTransferred : TransferHook +internal object FileTransferred : TransferHook +internal object TransferCompleted : TransferHook + +/** + * The context around an [S3TransferManager] transfer. + * Used to track transfer progress or to modify in progress transfers, such as low level requests/responses from S3. + */ +public interface TransferInterceptorContext { + // Req/Resp + public var request: Any? + public var response: Any? + + // Byte transfers + public var transferableBytes: Long? + public var currentBytes: ByteStream? + public var transferredBytes: Long? + + // File transfers + public var transferableFiles: Long? + public var currentFile: String? + public var transferredFiles: Long? +} + +/** + * Concrete implementation of [TransferInterceptorContext]. + * Used internally by the [S3TransferManager]. + */ +internal data class TransferContext( + // Req/Resp + override var request: Any? = null, + override var response: Any? = null, + + // Byte transfers + override var transferableBytes: Long? = null, + override var currentBytes: ByteStream? = null, + override var transferredBytes: Long? = null, + + // File transfers + override var transferableFiles: Long? = null, + override var currentFile: String? = null, + override var transferredFiles: Long? = null, +) : TransferInterceptorContext diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt new file mode 100644 index 00000000000..a58911474fb --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt @@ -0,0 +1,23 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.model + +/** + * Defines the strategy used for multipart downloads in [aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager]. + * + * A multipart download can either be performed by specifying byte ranges or by requesting individual parts. + */ +public sealed interface MultipartDownloadType + +/** + * Download specific byte ranges from an object. + */ +public object Range : MultipartDownloadType + +/** + * Download individual parts of an object as defined by the multipart upload structure. + */ +public object Part : MultipartDownloadType diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/HelperFunctions.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/HelperFunctions.kt new file mode 100644 index 00000000000..13bd1265678 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/HelperFunctions.kt @@ -0,0 +1,116 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile + +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.io.SdkBuffer +import aws.smithy.kotlin.runtime.io.SdkByteReadChannel +import aws.smithy.kotlin.runtime.io.SdkSource +import aws.smithy.kotlin.runtime.io.readFully +import aws.smithy.kotlin.runtime.io.readRemaining +import aws.smithy.kotlin.runtime.telemetry.logging.Logger + +// S3 imposed limit for parts in a multipart upload +private const val MAX_NUMBER_PARTS = 10_000L + +/** + * Determines the actual part size to use for a multipart S3 upload. + * + * This function calculates the part size based on the total size + * of the file and the requested part size. If the requested part size is + * too small to allow the upload to fit within S3's 10,000-part limit, the + * part size will be automatically increased so that exactly 10,000 parts + * are uploaded. + */ +internal fun resolvePartSize(contentLength: Long, targetPartSize: Long, logger: Logger): Long { + val targetNumberOfParts = contentLength / targetPartSize + return if (targetNumberOfParts > MAX_NUMBER_PARTS) { + ceilDiv(contentLength, MAX_NUMBER_PARTS).also { + logger.warn { "Target part size is too small to meet the $MAX_NUMBER_PARTS S3 part limit. Increasing part size to $it" } + } + } else { + targetPartSize + } +} + +/** + * Determines what part source an S3 body will have: + * [ByteStream.Buffer] + * [ByteStream.ChannelStream] + * [ByteStream.SourceStream] + */ +internal fun resolveSource(body: ByteStream): Any = + when (body) { + is ByteStream.Buffer -> body.bytes() + is ByteStream.ChannelStream -> body.readFrom() + is ByteStream.SourceStream -> body.readFrom() + else -> + throw S3TransferManagerException( + "Unhandled body type: ${body::class.simpleName }", + ) + } + +/** + * Retrieves the bytes for the next part of a multipart upload from the given part source into a [SdkBuffer] + */ +internal suspend fun nextPartBytes( + partSource: Any, + partSize: Long, + lastPart: Boolean, + readBytes: Int, + readableBytes: Int, +): SdkBuffer { + val buffer = SdkBuffer() + + when (partSource) { + is ByteArray -> { + if (lastPart) { + buffer.write( + partSource.sliceArray(readBytes.. { + if (lastPart) { + partSource.readRemaining(buffer) + } else { + partSource.readFully(buffer, partSize) + } + } + is SdkSource -> { + if (lastPart) { + partSource.readRemaining(buffer) + } else { + partSource.readFully(buffer, partSize) + } + } + } + + return buffer +} + +/** + * Returns the ceiling of the division + * + * This means the result is rounded up to the nearest integer if the dividend is not + * evenly divisible by the divisor + */ +internal fun ceilDiv(dividend: Long, divisor: Long): Long { + val div = dividend / divisor + val remainder = dividend % divisor + return if (remainder != 0L) { + div + 1 + } else { + div + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/UploadFile.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/UploadFile.kt new file mode 100644 index 00000000000..74d304feb72 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/UploadFile.kt @@ -0,0 +1,78 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile + +import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager +import aws.sdk.kotlin.hll.s3transfermanager.TransferContext +import aws.sdk.kotlin.hll.s3transfermanager.TransferInterceptor +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileResponse +import aws.sdk.kotlin.hll.s3transfermanager.model.utils.toUploadFileResponse +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.hooks.completeTransfer +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.hooks.initiateTransfer +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.hooks.transferBytes +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadResponse +import aws.sdk.kotlin.services.s3.model.PutObjectResponse +import aws.smithy.kotlin.runtime.telemetry.TelemetryProviderContext +import aws.smithy.kotlin.runtime.telemetry.logging.logger +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.withContext + +internal suspend fun uploadFileImplementation( + uploadFileRequest: UploadFileRequest, + client: S3Client, + multipartUploadThresholdBytes: Long, + partSizeBytes: Long, + interceptors: List, + maxInMemoryParts: Int, + maxConcurrentPartUploads: Int, +): UploadFileResponse = withContext(currentCoroutineContext() + TelemetryProviderContext(client.config.telemetryProvider)) { + val contentLength = uploadFileRequest.body?.contentLength ?: throw S3TransferManagerException("Body content length must be known") + val multiPartUpload = contentLength > multipartUploadThresholdBytes + val logger = coroutineContext.logger() + val transferContext = TransferContext() + + val mpuUploadId = initiateTransfer( + multiPartUpload, + transferContext, + contentLength, + uploadFileRequest, + interceptors, + client, + ) + + val uploadedParts = transferBytes( + multiPartUpload, + contentLength, + partSizeBytes, + logger, + uploadFileRequest, + transferContext, + mpuUploadId, + interceptors, + client, + maxInMemoryParts, + maxConcurrentPartUploads, + ) + + completeTransfer( + multiPartUpload, + transferContext, + uploadFileRequest, + mpuUploadId, + uploadedParts, + interceptors, + client, + ) + + return@withContext when (transferContext.response) { + is PutObjectResponse -> (transferContext.response as PutObjectResponse).toUploadFileResponse() + is CompleteMultipartUploadResponse -> (transferContext.response as CompleteMultipartUploadResponse).toUploadFileResponse() + else -> throw S3TransferManagerException("Unexpected response type: ${transferContext.response}") + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/CompleteTransfer.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/CompleteTransfer.kt new file mode 100644 index 00000000000..5f8afb45f65 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/CompleteTransfer.kt @@ -0,0 +1,49 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.hooks + +import aws.sdk.kotlin.hll.s3transfermanager.TransferCompleted +import aws.sdk.kotlin.hll.s3transfermanager.TransferContext +import aws.sdk.kotlin.hll.s3transfermanager.TransferInterceptor +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.utils.toCompleteMultipartUploadRequest +import aws.sdk.kotlin.hll.s3transfermanager.operationHook +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.CompletedPart + +internal suspend fun completeTransfer( + multiPartUpload: Boolean, + context: TransferContext, + uploadFileRequest: UploadFileRequest, + mpuUploadId: String?, + uploadedParts: List, + interceptors: List, + client: S3Client, +) { + if (multiPartUpload) { + context.request = + uploadFileRequest.toCompleteMultipartUploadRequest( + mpuUploadId!!, + uploadedParts, + ) + } + + operationHook( + TransferCompleted, + context, + interceptors, + ) { + if (multiPartUpload) { + try { + context.response = client.completeMultipartUpload(context.request as CompleteMultipartUploadRequest) + } catch (e: Exception) { + throw S3TransferManagerException("Unable to complete multipart upload with ID: $mpuUploadId", e) + } + } + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/InitiateTransfer.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/InitiateTransfer.kt new file mode 100644 index 00000000000..b23114b7063 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/InitiateTransfer.kt @@ -0,0 +1,47 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.hooks + +import aws.sdk.kotlin.hll.s3transfermanager.TransferContext +import aws.sdk.kotlin.hll.s3transfermanager.TransferInitiated +import aws.sdk.kotlin.hll.s3transfermanager.TransferInterceptor +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.utils.toCreateMultipartUploadRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.utils.toPutObjectRequest +import aws.sdk.kotlin.hll.s3transfermanager.operationHook +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadResponse + +internal suspend fun initiateTransfer( + multiPartUpload: Boolean, + context: TransferContext, + contentLength: Long, + uploadFileRequest: UploadFileRequest, + interceptors: List, + client: S3Client, +): String? { + context.transferredBytes = 0L + context.transferableBytes = contentLength + context.request = if (multiPartUpload) { + uploadFileRequest.toCreateMultipartUploadRequest() + } else { + uploadFileRequest.toPutObjectRequest() + } + + var mpuUploadId: String? = null + operationHook( + TransferInitiated, + context, + interceptors, + ) { + if (multiPartUpload) { + context.response = client.createMultipartUpload(context.request as CreateMultipartUploadRequest) + mpuUploadId = (context.response as CreateMultipartUploadResponse).uploadId!! + } + } + return mpuUploadId +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/TransferBytes.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/TransferBytes.kt new file mode 100644 index 00000000000..ce7dd72bbbe --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/hooks/TransferBytes.kt @@ -0,0 +1,227 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.hooks + +import aws.sdk.kotlin.hll.s3transfermanager.BytesTransferred +import aws.sdk.kotlin.hll.s3transfermanager.TransferContext +import aws.sdk.kotlin.hll.s3transfermanager.TransferInterceptor +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.utils.toUploadPartRequest +import aws.sdk.kotlin.hll.s3transfermanager.operationHook +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.ceilDiv +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.nextPartBytes +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.resolvePartSize +import aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile.resolveSource +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.abortMultipartUpload +import aws.sdk.kotlin.services.s3.model.CompletedPart +import aws.sdk.kotlin.services.s3.model.PutObjectRequest +import aws.sdk.kotlin.services.s3.model.UploadPartRequest +import aws.sdk.kotlin.services.s3.model.UploadPartResponse +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.io.SdkBuffer +import aws.smithy.kotlin.runtime.io.SdkSource +import aws.smithy.kotlin.runtime.telemetry.logging.Logger +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.channels.ReceiveChannel +import kotlinx.coroutines.channels.produce +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +/** + * Represents a part in a multipart upload. + * + * @param number The part number. + * @param bytes The bytes of the part. + */ +internal data class Part( + val number: Int, + val bytes: SdkBuffer, +) + +internal suspend fun transferBytes( + multiPartUpload: Boolean, + contentLength: Long, + partSizeBytes: Long, + logger: Logger, + uploadFileRequest: UploadFileRequest, + context: TransferContext, + mpuUploadId: String?, + interceptors: List, + client: S3Client, + maxInMemoryParts: Int, + maxConcurrentPartUploads: Int, +): List = coroutineScope { + val uploadedParts = mutableListOf() + + if (multiPartUpload) { + try { + val partSize = resolvePartSize(contentLength, partSizeBytes, logger) + val numberOfParts = ceilDiv(contentLength, partSize).toInt() + val partSource = resolveSource(uploadFileRequest.body!!) + + val producer = produceParts( + context.transferableBytes!!, + partSource, + partSize, + numberOfParts, + maxInMemoryParts, + ) + + val mutex = Mutex() + repeat(maxConcurrentPartUploads) { + consumer( + producer, + uploadFileRequest, + mpuUploadId!!, + context, + interceptors, + client, + uploadedParts, + mutex, + ) + } + + if (uploadedParts.size != numberOfParts) { + throw S3TransferManagerException("The number of uploaded parts does not match the expected count. Expected $numberOfParts, actual: ${uploadedParts.size}") + } + } catch (uploadPartException: Exception) { + try { + client.abortMultipartUpload { + bucket = uploadFileRequest.bucket + expectedBucketOwner = uploadFileRequest.expectedBucketOwner + key = uploadFileRequest.key + requestPayer = uploadFileRequest.requestPayer + uploadId = mpuUploadId + } + throw S3TransferManagerException("Multipart upload failed (ID: $mpuUploadId). One or more parts could not be uploaded", uploadPartException) + } catch (abortException: Exception) { + throw S3TransferManagerException("Multipart upload failed (ID: $mpuUploadId). Unable to abort multipart upload.", abortException) + .also { it.addSuppressed(uploadPartException) } + } + } + } else { + context.currentBytes = uploadFileRequest.body + + operationHook( + BytesTransferred, + context, + interceptors, + ) { + context.response = client.putObject(context.request as PutObjectRequest) + context.transferredBytes = context.transferableBytes + } + } + + return@coroutineScope uploadedParts +} + +/** + * Produces multipart upload parts to be consumed by [consumer]. + * + * Uses a [kotlinx.coroutines.channels.Channel]. + * Produces until all readable bytes are read. + */ +internal fun CoroutineScope.produceParts( + readableBytes: Long, + partSource: Any, + partSize: Long, + numberOfParts: Int, + maxInMemoryParts: Int, +) = produce( + capacity = maxInMemoryParts, +) { + var readBytes = 0L + var currentPartNumber = 1 + + while (readBytes < readableBytes) { + send( + Part( + currentPartNumber, + nextPartBytes( + partSource, + partSize, + currentPartNumber == numberOfParts, + readBytes.toInt(), + readableBytes.toInt(), + ), + ).also { + if (currentPartNumber != numberOfParts && it.bytes.size != partSize) { + throw S3TransferManagerException("Part #$currentPartNumber size mismatch detected. Expected $partSize, actual: ${it.bytes.size}") + } + }, + ) + + currentPartNumber++ + readBytes += partSize + } +} + +/** + * Launches a coroutine that consumes and uploads multipart upload parts. + * + * It receives mutable shared state that may also be used by other coroutines and is + * intended for use in a [fan-out](https://kotlinlang.org/docs/channels.html#fan-out) pattern, + * where multiple consumers concurrently upload different parts of the same file. + */ +internal suspend fun consumer( + channel: ReceiveChannel, + uploadFileRequest: UploadFileRequest, + mpuUploadId: String, + context: TransferContext, + interceptors: List, + client: S3Client, + uploadedParts: MutableList, + mutex: Mutex, +) = coroutineScope { + launch { + for (part in channel) { + val partSize = part.bytes.size // Store the original size, as it will shrink when bytes are read + val localContext = context.copy() // Create a separate copy to avoid concurrent modifications + + localContext.request = uploadFileRequest.toUploadPartRequest( + part.bytes, + part.number, + mpuUploadId, + ) + localContext.currentBytes = object : ByteStream.SourceStream() { + override fun readFrom(): SdkSource = part.bytes.peek() // Peek so bytes aren’t consumed before sending + override val contentLength: Long = partSize + } + + operationHook( + BytesTransferred, + localContext, + interceptors, + ) { + localContext.response = client.uploadPart(localContext.request as UploadPartRequest) + localContext.transferredBytes = localContext.transferredBytes!! + partSize + } + + // Update shared state between coroutines + mutex.withLock { + context.request = localContext.request + context.response = localContext.response + context.transferableBytes = localContext.transferableBytes + context.currentBytes = localContext.currentBytes + context.transferredBytes = context.transferredBytes!! + partSize // Don't use transferredBytes from local context as it might be out of date + context.transferableFiles = localContext.transferableFiles + context.currentFile = localContext.currentFile + context.transferredFiles = localContext.transferredFiles + + uploadedParts.add( + CompletedPart { + partNumber = part.number + eTag = (localContext.response as UploadPartResponse).eTag + }, + ) + } + } + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricInterceptor.kt new file mode 100644 index 00000000000..1e697eaedb2 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricInterceptor.kt @@ -0,0 +1,21 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.utils + +import aws.sdk.kotlin.runtime.http.interceptors.businessmetrics.AwsBusinessMetric +import aws.smithy.kotlin.runtime.businessmetrics.emitBusinessMetric +import aws.smithy.kotlin.runtime.client.RequestInterceptorContext +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor + +/** + * An interceptor that emits the S3 Transfer Manager business metric + */ +internal object S3TransferManagerBusinessMetricInterceptor : HttpInterceptor { + override suspend fun modifyBeforeSerialization(context: RequestInterceptorContext): Any { + context.executionContext.emitBusinessMetric(AwsBusinessMetric.S3_TRANSFER) + return context.request + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerException.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerException.kt new file mode 100644 index 00000000000..c7b09c6738e --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerException.kt @@ -0,0 +1,14 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.utils + +/** + * Exception thrown when an error occurs during S3 transfer operations. + * + * @param message Description of the error. + * @param cause The underlying cause of the exception, if any. + */ +internal class S3TransferManagerException(message: String, cause: Throwable? = null) : Exception(message, cause) diff --git a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/UploadFileTest.kt b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/UploadFileTest.kt new file mode 100644 index 00000000000..5dc1a4ce517 --- /dev/null +++ b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/operations/uploadfile/UploadFileTest.kt @@ -0,0 +1,82 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.operations.uploadfile + +import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager +import aws.sdk.kotlin.services.s3.S3Client +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.content.asByteStream +import aws.smithy.kotlin.runtime.testing.RandomTempFile +import kotlinx.coroutines.runBlocking +import kotlin.invoke +import kotlin.test.Test + +// TODO: Setup e2e test environment - can't run these every build and in CI +class UploadFileTest { + @Test + fun singleObjectUpload(): Unit = runBlocking { + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager(s3Client) {}.uploadFile { + bucket = "aoperez" + key = "k" + body = ByteStream.fromString("Hello World") + } + } + } + + @Test + fun emptyBody(): Unit = runBlocking { + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager(s3Client) {}.uploadFile { + bucket = "aoperez" + key = "k" + body = ByteStream.fromString("") + } + } + } + + @Test + fun multipartUpload(): Unit = runBlocking { + val messageLength = 10L * 1024L * 1024L // 10 MB + val file = RandomTempFile(messageLength) + + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager(s3Client) { + multipartUploadThresholdBytes = 1 + partSizeBytes = 5L * 1024L * 1024L // 5 MB + }.uploadFile { + bucket = "aoperez" + key = "mpuK" + body = file.asByteStream() + } + } + } + + @Test + fun smallLastPart(): Unit = runBlocking { + val messageLength = 12L * 1024L * 1024L // 12 MB (last part will only be 2MB) + val file = RandomTempFile(messageLength) + + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager(s3Client) { + multipartUploadThresholdBytes = 1 + partSizeBytes = 5L * 1024L * 1024L // 5 MB + }.uploadFile { + bucket = "aoperez" + key = "mpuK" + body = file.asByteStream() + } + } + } +} diff --git a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricsInterceptorTest.kt b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricsInterceptorTest.kt new file mode 100644 index 00000000000..992d9ad46e0 --- /dev/null +++ b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/S3TransferManagerBusinessMetricsInterceptorTest.kt @@ -0,0 +1,47 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.utils + +import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.runtime.http.interceptors.businessmetrics.AwsBusinessMetric +import aws.sdk.kotlin.services.s3.S3Client +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.businessmetrics.containsBusinessMetric +import aws.smithy.kotlin.runtime.client.ProtocolResponseInterceptorContext +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.httptest.TestEngine +import kotlinx.coroutines.runBlocking +import kotlin.invoke +import kotlin.test.Test + +class S3TransferManagerBusinessMetricsInterceptorTest { + @Test + fun s3Transfer(): Unit = runBlocking { + val message = "Hello World" + val testInterceptor = object : HttpInterceptor { + override fun readAfterTransmit(context: ProtocolResponseInterceptorContext) { + assert(context.executionContext.containsBusinessMetric(AwsBusinessMetric.S3_TRANSFER)) + } + } + + S3Client { + region = "us-west-2" + httpClient = TestEngine() + interceptors += listOf(testInterceptor) + credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) + }.use { s3Client -> + S3TransferManager(s3Client) {}.uploadFile { + bucket = "b" + key = "k" + body = ByteStream.fromString(message) + } + } + } +} diff --git a/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/TransferInterceptorTest.kt b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/TransferInterceptorTest.kt new file mode 100644 index 00000000000..12ebb5ba132 --- /dev/null +++ b/hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3transfermanager/utils/TransferInterceptorTest.kt @@ -0,0 +1,103 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.utils + +import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager +import aws.sdk.kotlin.hll.s3transfermanager.TransferInterceptor +import aws.sdk.kotlin.hll.s3transfermanager.TransferInterceptorContext +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.PutObjectRequest +import aws.sdk.kotlin.services.s3.model.PutObjectResponse +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.httptest.TestEngine +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.assertThrows +import kotlin.collections.plusAssign +import kotlin.invoke +import kotlin.test.Test +import kotlin.test.assertEquals + +class TransferInterceptorTest { + @Test + fun interceptorsCanReadAndModify(): Unit = runBlocking { + val message = "Hello World" + + S3Client { + region = "us-west-2" + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) + }.use { s3Client -> + S3TransferManager(s3Client) { + interceptors += object : TransferInterceptor { + // Test reads + override fun readBeforeTransferInitiated(context: TransferInterceptorContext) { + assert(context.transferredBytes == 0L) + assert(context.request is PutObjectRequest) + } + override fun readBeforeTransferCompleted(context: TransferInterceptorContext) { + assert(context.transferredBytes == message.length.toLong()) + assert(context.response is PutObjectResponse) + } + + // Test modifications + override fun modifyBeforeTransferCompleted(context: TransferInterceptorContext) { + context.request = CompleteMultipartUploadRequest {} + context.transferredBytes = message.length.toLong() * 10 + } + override fun readAfterTransferCompleted(context: TransferInterceptorContext) { + assert(context.request is CompleteMultipartUploadRequest) + assert(context.transferredBytes == message.length.toLong() * 10) + } + } + }.uploadFile { + bucket = "b" + key = "k" + body = ByteStream.fromString(message) + } + } + } + + @Test + fun interceptorsExceptionsAreSuppressed(): Unit = runBlocking { + val message = "Hello World" + + val exception = assertThrows { + S3Client { + region = "us-west-2" + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) + }.use { s3Client -> + S3TransferManager(s3Client) { + interceptors += listOf( + object : TransferInterceptor { + override fun readBeforeTransferInitiated(context: TransferInterceptorContext): Unit = + throw Exception("1") + }, + object : TransferInterceptor { + override fun readBeforeTransferInitiated(context: TransferInterceptorContext): Unit = + throw Exception("2") + }, + object : TransferInterceptor { + override fun readBeforeTransferInitiated(context: TransferInterceptorContext): Unit = + throw Exception("3") + }, + ) + }.uploadFile { + bucket = "b" + key = "k" + body = ByteStream.fromString(message) + } + } + } + + assertEquals(exception.message, "1") + assertEquals(exception.cause!!.suppressed[0].message, "2") + assertEquals(exception.cause!!.suppressed[1].message, "3") + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 826adedd84d..1412cf2fbe2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -91,6 +91,13 @@ if ("dynamodb".isBootstrappedService) { logger.warn(":services:dynamodb is not bootstrapped, skipping :hll:dynamodb-mapper and subprojects") } +if ("s3".isBootstrappedService) { + include(":hll:s3-transfer-manager") + include(":hll:s3-transfer-manager-codegen") +} else { + logger.warn(":services:s3 is not bootstrapped, skipping :hll:s3-transfer-manager and subprojects") +} + // Service benchmarks project val benchmarkServices = listOf( // keep this list in sync with tests/benchmarks/service-benchmarks/build.gradle.kts