From 9e4bd6f3588b69c49a45d9ab4f43c867325b208f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 19 May 2025 16:52:22 -0700 Subject: [PATCH 1/6] sync --- .../encrypted/paginator.py | 199 ++++++++++++++++++ .../test/integ/encrypted/test_paginator.py | 190 +++++++++++++++++ .../test/unit/encrypted/test_paginator.py | 78 +++++++ .../src/encrypted_paginator/__init__.py | 3 + .../encrypted_paginator_example.py | 159 ++++++++++++++ .../test/encrypted_paginator/__init__.py | 3 + .../test_encrypted_paginator_example.py | 15 ++ 7 files changed, 647 insertions(+) create mode 100644 DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/paginator.py create mode 100644 DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py create mode 100644 DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_paginator.py create mode 100644 Examples/runtimes/python/DynamoDBEncryption/src/encrypted_paginator/__init__.py create mode 100644 Examples/runtimes/python/DynamoDBEncryption/src/encrypted_paginator/encrypted_paginator_example.py create mode 100644 Examples/runtimes/python/DynamoDBEncryption/test/encrypted_paginator/__init__.py create mode 100644 Examples/runtimes/python/DynamoDBEncryption/test/encrypted_paginator/test_encrypted_paginator_example.py diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/paginator.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/paginator.py new file mode 100644 index 000000000..78d837aff --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/paginator.py @@ -0,0 +1,199 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""High-level helper class to provide an encrypting wrapper for boto3 DynamoDB paginators.""" +from collections.abc import Callable, Generator +from copy import deepcopy +from typing import Any + +from botocore.paginate import ( + Paginator, +) + +from aws_dbesdk_dynamodb.encrypted.boto3_interface import EncryptedBotoInterface +from aws_dbesdk_dynamodb.internal.client_to_resource import ClientShapeToResourceShapeConverter +from aws_dbesdk_dynamodb.internal.resource_to_client import ResourceShapeToClientShapeConverter +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb.models import ( + DynamoDbTablesEncryptionConfig, +) +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb_transforms.client import ( + DynamoDbEncryptionTransforms, +) +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb_transforms.models import ( + QueryInputTransformInput, + QueryOutputTransformInput, + ScanInputTransformInput, + ScanOutputTransformInput, +) + + +class EncryptedPaginator(EncryptedBotoInterface): + """Wrapping class for the boto3 Paginator that decrypts returned items before returning them.""" + + def __init__( + self, + *, + paginator: Paginator, + encryption_config: DynamoDbTablesEncryptionConfig, + expect_standard_dictionaries: bool | None = False, + ): + """ + Create an EncryptedPaginator. + + Args: + paginator (Paginator): A boto3 Paginator object for DynamoDB operations. + This can be either a "query" or "scan" Paginator. + encryption_config (DynamoDbTablesEncryptionConfig): Encryption configuration object. + expect_standard_dictionaries (Optional[bool]): Does the underlying boto3 client expect items + to be standard Python dictionaries? This should only be set to True if you are using a + client obtained from a service resource or table resource (ex: ``table.meta.client``). + If this is True, EncryptedClient will expect item-like shapes to be + standard Python dictionaries (default: False). + + """ + self._paginator = paginator + self._encryption_config = encryption_config + self._transformer = DynamoDbEncryptionTransforms(config=encryption_config) + self._expect_standard_dictionaries = expect_standard_dictionaries + self._resource_to_client_shape_converter = ResourceShapeToClientShapeConverter() + self._client_to_resource_shape_converter = ClientShapeToResourceShapeConverter(delete_table_name=False) + + def paginate(self, **kwargs) -> Generator[dict, None, None]: + """ + Yield a generator that paginates through responses from DynamoDB, decrypting items. + + Note: + Calling ``botocore.paginate.Paginator``'s ``paginate`` method for Query or Scan + returns a ``PageIterator`` object, but this implementation returns a Python generator. + However, you can use this generator to iterate exactly as described in the + boto3 documentation: + + https://botocore.amazonaws.com/v1/documentation/api/latest/topics/paginators.html + + Any other operations on this class will defer to the underlying boto3 Paginator's implementation. + + Args: + **kwargs: Keyword arguments passed directly to the underlying DynamoDB paginator. + + For a Scan operation, structure these arguments according to: + + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/paginator/Scan.html + + For a Query operation, structure these arguments according to: + + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/paginator/Query.html + + Returns: + Generator[dict, None, None]: A generator yielding pages as dictionaries. + The items in the pages will be decrypted locally after being read from DynamoDB. + + """ + if self._paginator._model.name == "Query": + yield from self._paginate_query(**kwargs) + elif self._paginator._model.name == "Scan": + yield from self._paginate_scan(**kwargs) + else: + yield from self._paginator.paginate(**kwargs) + + def _paginate_query(self, **paginate_query_kwargs): + return self._paginate_request( + paginate_kwargs=paginate_query_kwargs, + input_item_to_ddb_transform_method=self._resource_to_client_shape_converter.query_request, + input_item_to_dict_transform_method=self._client_to_resource_shape_converter.query_request, + input_transform_method=self._transformer.query_input_transform, + input_transform_shape=QueryInputTransformInput, + output_item_to_ddb_transform_method=self._resource_to_client_shape_converter.query_response, + output_item_to_dict_transform_method=self._client_to_resource_shape_converter.query_response, + output_transform_method=self._transformer.query_output_transform, + output_transform_shape=QueryOutputTransformInput, + ) + + def _paginate_scan(self, **paginate_scan_kwargs): + return self._paginate_request( + paginate_kwargs=paginate_scan_kwargs, + input_item_to_ddb_transform_method=self._resource_to_client_shape_converter.scan_request, + input_item_to_dict_transform_method=self._client_to_resource_shape_converter.scan_request, + input_transform_method=self._transformer.scan_input_transform, + input_transform_shape=ScanInputTransformInput, + output_item_to_ddb_transform_method=self._resource_to_client_shape_converter.scan_response, + output_item_to_dict_transform_method=self._client_to_resource_shape_converter.scan_response, + output_transform_method=self._transformer.scan_output_transform, + output_transform_shape=ScanOutputTransformInput, + ) + + def _paginate_request( + self, + *, + paginate_kwargs: dict[str, Any], + input_item_to_ddb_transform_method: Callable, + input_item_to_dict_transform_method: Callable, + input_transform_method: Callable, + input_transform_shape: Any, + output_item_to_ddb_transform_method: Callable, + output_item_to_dict_transform_method: Callable, + output_transform_method: Callable, + output_transform_shape: Any, + ): + client_kwargs = deepcopy(paginate_kwargs) + try: + # Remove PaginationConfig from the request if it exists. + # The input_transform_method does not expect it. + # It is added back to the request sent to the SDK. + pagination_config = client_kwargs["PaginationConfig"] + del client_kwargs["PaginationConfig"] + except KeyError: + pagination_config = None + + # If _expect_standard_dictionaries is true, input items are expected to be standard dictionaries, + # and need to be converted to DDB-JSON before encryption. + if self._expect_standard_dictionaries: + if "TableName" in client_kwargs: + self._resource_to_client_shape_converter.table_name = client_kwargs["TableName"] + client_kwargs = input_item_to_ddb_transform_method(client_kwargs) + + # Apply DBESDK transformations to the input + transformed_request = input_transform_method(input_transform_shape(sdk_input=client_kwargs)).transformed_input + + # If _expect_standard_dictionaries is true, the boto3 client expects items to be standard dictionaries, + # and need to be converted from DDB-JSON to a standard dictionary before being passed to the boto3 client. + if self._expect_standard_dictionaries: + transformed_request = input_item_to_dict_transform_method(transformed_request) + + if pagination_config is not None: + transformed_request["PaginationConfig"] = pagination_config + + sdk_page_response = self._paginator.paginate(**transformed_request) + + for page in sdk_page_response: + # If _expect_standard_dictionaries is true, the boto3 client returns items as standard dictionaries, + # and needs to convert the standard dictionary to DDB-JSON before passing the response to the DBESDK. + if self._expect_standard_dictionaries: + page = output_item_to_ddb_transform_method(page) + + # Apply DBESDK transformation to the boto3 output + dbesdk_response = output_transform_method( + output_transform_shape( + original_input=client_kwargs, + sdk_output=page, + ) + ).transformed_output + + # Copy any missing fields from the SDK output to the response (e.g. ConsumedCapacity) + dbesdk_response = self._copy_sdk_response_to_dbesdk_response(page, dbesdk_response) + + # If _expect_standard_dictionaries is true, the boto3 client expects items to be standard dictionaries, + # and need to be converted from DDB-JSON to a standard dictionary before returning the response. + if self._expect_standard_dictionaries: + dbesdk_response = output_item_to_dict_transform_method(dbesdk_response) + + yield dbesdk_response + + @property + def _boto_client_attr_name(self) -> str: + """ + Name of the attribute containing the underlying boto3 client. + + Returns: + str: '_paginator' + + """ + return "_paginator" diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py new file mode 100644 index 000000000..03d72f2d1 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py @@ -0,0 +1,190 @@ +import boto3 +import pytest + +from aws_dbesdk_dynamodb.encrypted.client import EncryptedClient + +from ...constants import ( + INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, + INTEG_TEST_DEFAULT_TABLE_CONFIGS, +) +from ...items import ( + complex_item_ddb, + complex_item_dict, + complex_key_ddb, + complex_key_dict, + simple_item_ddb, + simple_item_dict, + simple_key_ddb, + simple_key_dict, +) +from ...requests import ( + basic_put_item_request_ddb, + basic_put_item_request_dict, + basic_query_paginator_request, + basic_scan_paginator_request, +) +from . import sort_dynamodb_json_lists + + +# Creates a matrix of tests for each value in param, +# with a user-friendly string for test output: +# expect_standard_dictionaries = True -> "standard_dicts" +# expect_standard_dictionaries = False -> "ddb_json" +@pytest.fixture(params=[True, False], ids=["standard_dicts", "ddb_json"]) +def expect_standard_dictionaries(request): + return request.param + + +def encrypted_client(expect_standard_dictionaries): + return EncryptedClient( + client=plaintext_client(expect_standard_dictionaries), + encryption_config=INTEG_TEST_DEFAULT_TABLE_CONFIGS, + expect_standard_dictionaries=expect_standard_dictionaries, + ) + + +def plaintext_client(expect_standard_dictionaries): + if expect_standard_dictionaries: + client = boto3.resource("dynamodb").Table(INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME).meta.client + else: + client = boto3.client("dynamodb") + return client + + +# Creates a matrix of tests for each value in param, +# with a user-friendly string for test output: +# encrypted = True -> "encrypted" +# encrypted = False -> "plaintext" +@pytest.fixture(params=[True, False], ids=["encrypted", "plaintext"]) +def encrypted(request): + return request.param + + +@pytest.fixture +def client(encrypted, expect_standard_dictionaries): + if encrypted: + return encrypted_client(expect_standard_dictionaries) + else: + return plaintext_client(expect_standard_dictionaries) + + +@pytest.fixture +def query_paginator(client): + return client.get_paginator("query") + + +@pytest.fixture +def scan_paginator(client): + return client.get_paginator("scan") + + +# Creates a matrix of tests for each value in param, +# with a user-friendly string for test output: +# use_complex_item = True -> "complex_item" +# use_complex_item = False -> "simple_item" +@pytest.fixture(params=[True, False], ids=["complex_item", "simple_item"]) +def use_complex_item(request): + return request.param + + +@pytest.fixture +def test_key(expect_standard_dictionaries, use_complex_item): + """Get a single test item in the appropriate format for the client.""" + if expect_standard_dictionaries: + if use_complex_item: + return complex_key_dict + return simple_key_dict + if use_complex_item: + return complex_key_ddb + return simple_key_ddb + + +@pytest.fixture +def multiple_test_keys(expect_standard_dictionaries): + """Get two test keys in the appropriate format for the client.""" + if expect_standard_dictionaries: + return [simple_key_dict, complex_key_dict] + return [simple_key_ddb, complex_key_ddb] + + +@pytest.fixture +def test_item(expect_standard_dictionaries, use_complex_item): + """Get a single test item in the appropriate format for the client.""" + if expect_standard_dictionaries: + if use_complex_item: + return complex_item_dict + return simple_item_dict + if use_complex_item: + return complex_item_ddb + return simple_item_ddb + + +@pytest.fixture +def paginate_query_request(expect_standard_dictionaries, test_key): + if expect_standard_dictionaries: + return {**basic_query_paginator_request(test_key), "TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME} + return basic_query_paginator_request(test_key) + + +@pytest.fixture +def put_item_request(expect_standard_dictionaries, test_item): + if expect_standard_dictionaries: + # Client requests with `expect_standard_dictionaries=True` use dict-formatted requests + # with an added "TableName" key. + return {**basic_put_item_request_dict(test_item), "TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME} + return basic_put_item_request_ddb(test_item) + + +def test_GIVEN_query_paginator_WHEN_paginate_THEN_returns_expected_items( + client, query_paginator, paginate_query_request, put_item_request, test_item +): + # Given: item in table + client.put_item(**put_item_request) + # Given: Query paginator + # When: Paginate + response = query_paginator.paginate(**paginate_query_request) + # Then: Returns encrypted items + items = [] + for page in response: + if "Items" in page: + for item in page["Items"]: + items.append(item) + assert len(items) == 1 + # DynamoDB JSON uses lists to represent sets, so strict equality can fail. + # Sort lists to ensure consistent ordering when comparing expected and actual items. + expected_item = sort_dynamodb_json_lists(test_item) + actual_item = sort_dynamodb_json_lists(items[0]) + # Then: Items are equal + assert expected_item == actual_item + + +@pytest.fixture +def paginate_scan_request(expect_standard_dictionaries, test_item): + if expect_standard_dictionaries: + request = {**basic_scan_paginator_request(test_item), "TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME} + else: + request = basic_scan_paginator_request(test_item) + return request + + +def test_GIVEN_scan_paginator_WHEN_paginate_THEN_returns_expected_items( + client, scan_paginator, paginate_scan_request, put_item_request, test_item +): + # Given: item in table + client.put_item(**put_item_request) + # Given: Scan paginator + # When: Paginate + response = scan_paginator.paginate(**paginate_scan_request) + # Then: Returns encrypted items + items = [] + for page in response: + if "Items" in page: + for item in page["Items"]: + items.append(item) + assert len(items) == 1 + # DynamoDB JSON uses lists to represent sets, so strict equality can fail. + # Sort lists to ensure consistent ordering when comparing expected and actual items. + expected_item = sort_dynamodb_json_lists(test_item) + actual_item = sort_dynamodb_json_lists(items[0]) + # Then: Items are equal + assert expected_item == actual_item diff --git a/DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_paginator.py b/DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_paginator.py new file mode 100644 index 000000000..d4fe7d896 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_paginator.py @@ -0,0 +1,78 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import pytest +from botocore.client import BaseClient +from botocore.paginate import Paginator +from mock import MagicMock + +from aws_dbesdk_dynamodb.encrypted.client import ( + EncryptedPaginator, +) +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb.models import ( + DynamoDbTablesEncryptionConfig, +) +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb_transforms.models import ( + QueryInputTransformInput, + QueryInputTransformOutput, +) + +pytestmark = [pytest.mark.unit, pytest.mark.local] + +mock_boto3_dynamodb_client = MagicMock(__class__=BaseClient) +mock_tables_encryption_config = MagicMock(__class__=DynamoDbTablesEncryptionConfig) + + +def test_GIVEN_paginator_not_query_nor_scan_WHEN_paginate_THEN_defers_to_underlying_paginator(): + # Given: A paginator that is not a Query or Scan paginator + underlying_paginator = MagicMock(__class__=Paginator) + underlying_paginator._model.name = "NotQueryNorScan" + non_query_scan_paginator = EncryptedPaginator( + paginator=underlying_paginator, + encryption_config=mock_tables_encryption_config, + ) + # When: Call paginate + for _ in non_query_scan_paginator.paginate(): + pass # Drain the generator + # Then: Call goes to underlying paginator + underlying_paginator.paginate.assert_called_once() + + +def test_GIVEN_kwargs_has_PaginationConfig_WHEN_paginate_THEN_PaginationConfig_is_added_back_to_request(): + mock_underlying_paginator = MagicMock(__class__=Paginator) + mock_underlying_paginator._model.name = "Query" + paginator = EncryptedPaginator( + paginator=mock_underlying_paginator, + encryption_config=mock_tables_encryption_config, + ) + mock_input_transform_method = MagicMock() + mock_input_transform_method.return_value = QueryInputTransformOutput(transformed_input={"TableName": "test-table"}) + paginator._transformer.query_input_transform = mock_input_transform_method + # Given: A kwargs that has a PaginationConfig + kwargs_without_pagination_config = { + "TableName": "test-table", + } + kwargs_with_pagination_config = {**kwargs_without_pagination_config, "PaginationConfig": {"MaxItems": 10}} + # When: Call paginate + for _ in paginator.paginate(**kwargs_with_pagination_config): + pass # Drain the generator + # Then: PaginationConfig is added back to the request sent to the SDK + mock_underlying_paginator.paginate.assert_called_once_with(**kwargs_with_pagination_config) + # And: input_transform_method is called with kwargs without PaginationConfig + mock_input_transform_method.assert_called_once_with( + QueryInputTransformInput(sdk_input=kwargs_without_pagination_config) + ) + + +def test_GIVEN_invalid_class_attribute_WHEN_getattr_THEN_raise_error(): + # Create a mock with a specific spec that excludes our unknown attribute + mock_boto3_dynamodb_client = MagicMock(spec=["put_item", "get_item", "query", "scan"]) + encrypted_paginator = EncryptedPaginator( + paginator=mock_boto3_dynamodb_client, + encryption_config=mock_tables_encryption_config, + ) + + # Then: AttributeError is raised + with pytest.raises(AttributeError): + # Given: Invalid class attribute: not_a_valid_attribute_on_EncryptedClient_nor_boto3_client + # When: getattr is called + encrypted_paginator.not_a_valid_attribute_on_EncryptedPaginator_nor_boto3_paginator() diff --git a/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_paginator/__init__.py b/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_paginator/__init__.py new file mode 100644 index 000000000..fa977e22f --- /dev/null +++ b/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_paginator/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub to allow relative imports of examples from tests.""" diff --git a/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_paginator/encrypted_paginator_example.py b/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_paginator/encrypted_paginator_example.py new file mode 100644 index 000000000..4bee4179c --- /dev/null +++ b/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_paginator/encrypted_paginator_example.py @@ -0,0 +1,159 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +Example for using the EncryptedPaginator provided by EncryptedClient. + +https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/paginator/Query.html + +Running this example requires access to the DDB Table whose name +is provided in the function arguments. +This table must be configured with the following primary key configuration: +- Partition key is named "partition_key" with type (S) +- Sort key is named "sort_key" with type (N) +""" + +import boto3 +from aws_cryptographic_material_providers.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_material_providers.mpl.config import MaterialProvidersConfig +from aws_cryptographic_material_providers.mpl.models import ( + CreateAwsKmsMrkMultiKeyringInput, + DBEAlgorithmSuiteId, +) +from aws_cryptographic_material_providers.mpl.references import IKeyring +from aws_dbesdk_dynamodb.encrypted.client import EncryptedClient +from aws_dbesdk_dynamodb.structures.dynamodb import ( + DynamoDbTableEncryptionConfig, + DynamoDbTablesEncryptionConfig, +) +from aws_dbesdk_dynamodb.structures.structured_encryption import ( + CryptoAction, +) + + +def encrypted_paginator_example( + kms_key_id: str, + dynamodb_table_name: str, +): + """Use an EncryptedPaginator to paginate through items in a table.""" + # 1. Create a Keyring. This Keyring will be responsible for protecting the data keys that protect your data. + # For this example, we will create a AWS KMS Keyring with the AWS KMS Key we want to use. + # We will use the `CreateMrkMultiKeyring` method to create this keyring, + # as it will correctly handle both single region and Multi-Region KMS Keys. + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders(config=MaterialProvidersConfig()) + kms_mrk_multi_keyring_input: CreateAwsKmsMrkMultiKeyringInput = CreateAwsKmsMrkMultiKeyringInput( + generator=kms_key_id, + ) + kms_mrk_multi_keyring: IKeyring = mat_prov.create_aws_kms_mrk_multi_keyring(input=kms_mrk_multi_keyring_input) + + # 2. Configure which attributes are encrypted and/or signed when writing new items. + # For each attribute that may exist on the items we plan to write to our DynamoDbTable, + # we must explicitly configure how they should be treated during item encryption: + # - ENCRYPT_AND_SIGN: The attribute is encrypted and included in the signature + # - SIGN_ONLY: The attribute not encrypted, but is still included in the signature + # - DO_NOTHING: The attribute is not encrypted and not included in the signature + attribute_actions_on_encrypt = { + "partition_key": CryptoAction.SIGN_ONLY, + "sort_key": CryptoAction.SIGN_ONLY, + "attribute1": CryptoAction.ENCRYPT_AND_SIGN, + "attribute2": CryptoAction.SIGN_ONLY, + ":attribute3": CryptoAction.DO_NOTHING, + } + + # 3. Configure which attributes we expect to be included in the signature + # when reading items. There are two options for configuring this: + # + # - (Recommended) Configure `allowedUnsignedAttributesPrefix`: + # When defining your DynamoDb schema and deciding on attribute names, + # choose a distinguishing prefix (such as ":") for all attributes that + # you do not want to include in the signature. + # This has two main benefits: + # - It is easier to reason about the security and authenticity of data within your item + # when all unauthenticated data is easily distinguishable by their attribute name. + # - If you need to add new unauthenticated attributes in the future, + # you can easily make the corresponding update to your `attributeActionsOnEncrypt` + # and immediately start writing to that new attribute, without + # any other configuration update needed. + # Once you configure this field, it is not safe to update it. + # + # - Configure `allowedUnsignedAttributes`: You may also explicitly list + # a set of attributes that should be considered unauthenticated when encountered + # on read. Be careful if you use this configuration. Do not remove an attribute + # name from this configuration, even if you are no longer writing with that attribute, + # as old items may still include this attribute, and our configuration needs to know + # to continue to exclude this attribute from the signature scope. + # If you add new attribute names to this field, you must first deploy the update to this + # field to all readers in your host fleet before deploying the update to start writing + # with that new attribute. + # + # For this example, we have designed our DynamoDb table such that any attribute name with + # the ":" prefix should be considered unauthenticated. + unsignAttrPrefix: str = ":" + + # 4. Create the DynamoDb Encryption configuration for the tables we will be writing to. + # For each table, we create a DynamoDbTableEncryptionConfig and add it to a dictionary. + # This dictionary is then added to a DynamoDbTablesEncryptionConfig, which is used to create the + # EncryptedResource. + table_configs = {} + table_config = DynamoDbTableEncryptionConfig( + logical_table_name=dynamodb_table_name, + partition_key_name="partition_key", + sort_key_name="sort_key", + attribute_actions_on_encrypt=attribute_actions_on_encrypt, + keyring=kms_mrk_multi_keyring, + allowed_unsigned_attribute_prefix=unsignAttrPrefix, + # Specifying an algorithm suite is not required, + # but is done here to demonstrate how to do so. + # We suggest using the + # `ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384_SYMSIG_HMAC_SHA384` suite, + # which includes AES-GCM with key derivation, signing, and key commitment. + # This is also the default algorithm suite if one is not specified in this config. + # For more information on supported algorithm suites, see: + # https://docs.aws.amazon.com/database-encryption-sdk/latest/devguide/supported-algorithms.html + algorithm_suite_id=DBEAlgorithmSuiteId.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384_SYMSIG_HMAC_SHA384, + ) + table_configs[dynamodb_table_name] = table_config + tables_config = DynamoDbTablesEncryptionConfig(table_encryption_configs=table_configs) + + # 5. Create the EncryptedClient + encrypted_client = EncryptedClient( + client=boto3.client("dynamodb"), + encryption_config=tables_config, + ) + + # 6. Put an item into the table. The EncryptedPaginator will paginate through the items in the table + # to find this item. + item = { + "partition_key": {"S": "PythonEncryptedPaginatorExample"}, + "sort_key": {"N": "0"}, + "attribute1": {"S": "encrypt and sign me!"}, + "attribute2": {"S": "sign me!"}, + ":attribute3": {"S": "ignore me!"}, + } + + encrypted_client.put_item( + TableName=dynamodb_table_name, + Item=item, + ) + + # 7. Create the EncryptedPaginator. + # We will use the encrypted `query` paginator, but an encrypted `scan` paginator is also available. + encrypted_paginator = encrypted_client.get_paginator("query") + + # 8. Use the EncryptedPaginator to paginate through the items in the table. + # The `paginate` method returns a generator that yields pages as dictionaries. + # The EncryptedPaginator will transparently decrypt the items in each page as they are returned. + # Once the generator is exhausted, the loop will exit. + items = [] + for page in encrypted_paginator.paginate( + TableName=dynamodb_table_name, + KeyConditionExpression="partition_key = :partition_key", + ExpressionAttributeValues={":partition_key": {"S": "PythonEncryptedPaginatorExample"}}, + ): + for item in page["Items"]: + items.append(item) + + # 9. Assert the items are as expected. + assert len(items) == 1 + assert items[0]["attribute1"]["S"] == "encrypt and sign me!" + assert items[0]["attribute2"]["S"] == "sign me!" + assert items[0][":attribute3"]["S"] == "ignore me!" diff --git a/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_paginator/__init__.py b/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_paginator/__init__.py new file mode 100644 index 000000000..fa977e22f --- /dev/null +++ b/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_paginator/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub to allow relative imports of examples from tests.""" diff --git a/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_paginator/test_encrypted_paginator_example.py b/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_paginator/test_encrypted_paginator_example.py new file mode 100644 index 000000000..15d874a60 --- /dev/null +++ b/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_paginator/test_encrypted_paginator_example.py @@ -0,0 +1,15 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test suite for the EncryptedPaginator example.""" +import pytest + +from ...src.encrypted_paginator.encrypted_paginator_example import encrypted_paginator_example + +pytestmark = [pytest.mark.examples] + + +def test_encrypted_paginator_example(): + """Test function for encrypt and decrypt using the EncryptedClient example.""" + test_kms_key_id = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" + test_dynamodb_table_name = "DynamoDbEncryptionInterceptorTestTable" + encrypted_paginator_example(test_kms_key_id, test_dynamodb_table_name) From cdc70cbf76095580ecac4dadd9e658e1745873f4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 19 May 2025 17:02:37 -0700 Subject: [PATCH 2/6] sync --- .../python/src/aws_dbesdk_dynamodb/encrypted/paginator.py | 5 +++-- .../runtimes/python/test/unit/encrypted/test_paginator.py | 5 ++++- .../src/encrypted_paginator/encrypted_paginator_example.py | 2 +- .../encrypted_paginator/test_encrypted_paginator_example.py | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/paginator.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/paginator.py index 78d837aff..6068f3c80 100644 --- a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/paginator.py +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/paginator.py @@ -27,7 +27,7 @@ class EncryptedPaginator(EncryptedBotoInterface): - """Wrapping class for the boto3 Paginator that decrypts returned items before returning them.""" + """Wrapping class for boto3 Paginators that decrypts returned items before returning them.""" def __init__( self, @@ -84,7 +84,8 @@ def paginate(self, **kwargs) -> Generator[dict, None, None]: Returns: Generator[dict, None, None]: A generator yielding pages as dictionaries. - The items in the pages will be decrypted locally after being read from DynamoDB. + For "scan" or "query" operations, the items in the pages will be decrypted locally after being read from + DynamoDB. """ if self._paginator._model.name == "Query": diff --git a/DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_paginator.py b/DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_paginator.py index d4fe7d896..566008bb0 100644 --- a/DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_paginator.py +++ b/DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_paginator.py @@ -24,6 +24,7 @@ def test_GIVEN_paginator_not_query_nor_scan_WHEN_paginate_THEN_defers_to_underlying_paginator(): # Given: A paginator that is not a Query or Scan paginator + # Mock an underlying paginator to spy on its call pattern underlying_paginator = MagicMock(__class__=Paginator) underlying_paginator._model.name = "NotQueryNorScan" non_query_scan_paginator = EncryptedPaginator( @@ -38,12 +39,14 @@ def test_GIVEN_paginator_not_query_nor_scan_WHEN_paginate_THEN_defers_to_underly def test_GIVEN_kwargs_has_PaginationConfig_WHEN_paginate_THEN_PaginationConfig_is_added_back_to_request(): + # Mock an underlying paginator to spy on its call pattern mock_underlying_paginator = MagicMock(__class__=Paginator) mock_underlying_paginator._model.name = "Query" paginator = EncryptedPaginator( paginator=mock_underlying_paginator, encryption_config=mock_tables_encryption_config, ) + # Mock the input transform method to spy on its arguments mock_input_transform_method = MagicMock() mock_input_transform_method.return_value = QueryInputTransformOutput(transformed_input={"TableName": "test-table"}) paginator._transformer.query_input_transform = mock_input_transform_method @@ -73,6 +76,6 @@ def test_GIVEN_invalid_class_attribute_WHEN_getattr_THEN_raise_error(): # Then: AttributeError is raised with pytest.raises(AttributeError): - # Given: Invalid class attribute: not_a_valid_attribute_on_EncryptedClient_nor_boto3_client + # Given: Invalid class attribute: not_a_valid_attribute_on_EncryptedPaginator_nor_boto3_paginator # When: getattr is called encrypted_paginator.not_a_valid_attribute_on_EncryptedPaginator_nor_boto3_paginator() diff --git a/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_paginator/encrypted_paginator_example.py b/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_paginator/encrypted_paginator_example.py index 4bee4179c..9baf46f42 100644 --- a/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_paginator/encrypted_paginator_example.py +++ b/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_paginator/encrypted_paginator_example.py @@ -152,7 +152,7 @@ def encrypted_paginator_example( for item in page["Items"]: items.append(item) - # 9. Assert the items are as expected. + # 9. Assert the items are returned as expected. assert len(items) == 1 assert items[0]["attribute1"]["S"] == "encrypt and sign me!" assert items[0]["attribute2"]["S"] == "sign me!" diff --git a/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_paginator/test_encrypted_paginator_example.py b/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_paginator/test_encrypted_paginator_example.py index 15d874a60..a3be81388 100644 --- a/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_paginator/test_encrypted_paginator_example.py +++ b/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_paginator/test_encrypted_paginator_example.py @@ -9,7 +9,7 @@ def test_encrypted_paginator_example(): - """Test function for encrypt and decrypt using the EncryptedClient example.""" + """Test function for encrypt and decrypt using the EncryptedPaginator example.""" test_kms_key_id = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" test_dynamodb_table_name = "DynamoDbEncryptionInterceptorTestTable" encrypted_paginator_example(test_kms_key_id, test_dynamodb_table_name) From c5ff19ecf367191c8cfdd5629ecfb79fba9089b4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 May 2025 09:25:11 -0700 Subject: [PATCH 3/6] sync --- .../python/src/aws_dbesdk_dynamodb/encrypted/paginator.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/paginator.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/paginator.py index 6068f3c80..9c22db791 100644 --- a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/paginator.py +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/paginator.py @@ -188,6 +188,9 @@ def _paginate_request( yield dbesdk_response + # Clean up the expression builder for the next operation + self._resource_to_client_shape_converter.expression_builder.reset() + @property def _boto_client_attr_name(self) -> str: """ From 3552baecf56c82b7131974eb618d04c548eaa80d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 May 2025 14:06:41 -0700 Subject: [PATCH 4/6] sync --- .../test/integ/encrypted/test_paginator.py | 77 +++++++++++++++---- 1 file changed, 62 insertions(+), 15 deletions(-) diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py index 03d72f2d1..1bc14e220 100644 --- a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py @@ -1,3 +1,6 @@ +import uuid +from copy import deepcopy + import boto3 import pytest @@ -18,6 +21,7 @@ simple_key_dict, ) from ...requests import ( + basic_delete_item_request_ddb, basic_put_item_request_ddb, basic_put_item_request_dict, basic_query_paginator_request, @@ -87,36 +91,68 @@ def use_complex_item(request): return request.param +# Append a suffix to the partition key to avoid collisions between test runs. +@pytest.fixture(scope="module") +def test_run_suffix(): + return str(uuid.uuid4()) + + @pytest.fixture -def test_key(expect_standard_dictionaries, use_complex_item): +def test_key(expect_standard_dictionaries, use_complex_item, test_run_suffix): """Get a single test item in the appropriate format for the client.""" if expect_standard_dictionaries: if use_complex_item: - return complex_key_dict - return simple_key_dict - if use_complex_item: - return complex_key_ddb - return simple_key_ddb + key = deepcopy(complex_key_dict) + else: + key = deepcopy(simple_key_dict) + else: + if use_complex_item: + key = deepcopy(complex_key_ddb) + else: + key = deepcopy(simple_key_ddb) + # Add a suffix to the partition key to avoid collisions between test runs. + if isinstance(key["partition_key"], dict): + key["partition_key"]["S"] += test_run_suffix + else: + key["partition_key"] += test_run_suffix + return key @pytest.fixture -def multiple_test_keys(expect_standard_dictionaries): +def multiple_test_keys(expect_standard_dictionaries, test_run_suffix): """Get two test keys in the appropriate format for the client.""" if expect_standard_dictionaries: - return [simple_key_dict, complex_key_dict] - return [simple_key_ddb, complex_key_ddb] + keys = [deepcopy(simple_key_dict), deepcopy(complex_key_dict)] + else: + keys = [deepcopy(simple_key_ddb), deepcopy(complex_key_ddb)] + # Add a suffix to the partition key to avoid collisions between test runs. + for key in keys: + if isinstance(key["partition_key"], dict): + key["partition_key"]["S"] += test_run_suffix + else: + key["partition_key"] += test_run_suffix + return keys @pytest.fixture -def test_item(expect_standard_dictionaries, use_complex_item): +def test_item(expect_standard_dictionaries, use_complex_item, test_run_suffix): """Get a single test item in the appropriate format for the client.""" if expect_standard_dictionaries: if use_complex_item: - return complex_item_dict - return simple_item_dict - if use_complex_item: - return complex_item_ddb - return simple_item_ddb + item = deepcopy(complex_item_dict) + else: + item = deepcopy(simple_item_dict) + else: + if use_complex_item: + item = deepcopy(complex_item_ddb) + else: + item = deepcopy(simple_item_ddb) + # Add a suffix to the partition key to avoid collisions between test runs. + if isinstance(item["partition_key"], dict): + item["partition_key"]["S"] += test_run_suffix + else: + item["partition_key"] += test_run_suffix + return item @pytest.fixture @@ -188,3 +224,14 @@ def test_GIVEN_scan_paginator_WHEN_paginate_THEN_returns_expected_items( actual_item = sort_dynamodb_json_lists(items[0]) # Then: Items are equal assert expected_item == actual_item + + +# Delete the items in the table after the module runs +@pytest.fixture(scope="module", autouse=True) +def cleanup_after_module(test_run_suffix): + yield + table = boto3.client("dynamodb") + items = [deepcopy(simple_item_ddb), deepcopy(complex_item_ddb)] + for item in items: + item["partition_key"]["S"] += test_run_suffix + table.delete_item(**basic_delete_item_request_ddb(item)) From 953f66979b4c3c91545a28703e81db034dc84ed0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 29 May 2025 13:21:44 -0700 Subject: [PATCH 5/6] sync --- .../runtimes/python/test/integ/encrypted/test_paginator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py index 1bc14e220..8b2e104e2 100644 --- a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py @@ -94,7 +94,7 @@ def use_complex_item(request): # Append a suffix to the partition key to avoid collisions between test runs. @pytest.fixture(scope="module") def test_run_suffix(): - return str(uuid.uuid4()) + return "-" + str(uuid.uuid4()) @pytest.fixture From 4d0e88d58dfc2138f1d159c0ec9bb33f7914ae98 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 29 May 2025 14:59:27 -0700 Subject: [PATCH 6/6] sync --- .../runtimes/python/test/integ/encrypted/test_paginator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py index 8b2e104e2..723fb6fac 100644 --- a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_paginator.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 import uuid from copy import deepcopy