Merge branch 'master' into master

Author: wangzlei

.github/workflows/IntegrationTesting.yaml

@@ -4,6 +4,10 @@ on:
     branches:
       - master
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   build_SDK:
     name: Build X-Ray Python SDK
@@ -78,10 +82,9 @@ jobs:
         run: cp deploy.zip ./terraform
 
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
+        uses: aws-actions/configure-aws-credentials@v4
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          role-to-assume: ${{ secrets.AWS_INTEG_TEST_ROLE_ARN }}
           aws-region: us-west-2
 
       - name: Setup Terraform
@@ -126,10 +129,9 @@ jobs:
           java-version: 14
 
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
+        uses: aws-actions/configure-aws-credentials@v4
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          role-to-assume: ${{ secrets.AWS_INTEG_TEST_ROLE_ARN }}
           aws-region: us-west-2
 
       - name: Checkout test framework

.github/workflows/UnitTesting.yaml

@@ -16,13 +16,14 @@ jobs:
       py39: 3.9
       py310: '3.10'
       py311: '3.11'
+      py312: '3.12'
       DB_DATABASE: test_db
       DB_USER: root
       DB_PASSWORD: root
     strategy:
       fail-fast: false
       matrix:
-        python-version: [py37, py38, py39, py310, py311]
+        python-version: [py37, py38, py39, py310, py311, py312]
         testenv: [core, ext]
     steps:
       - name: Checkout repo
@@ -33,14 +34,17 @@ jobs:
         run: |
           sudo /etc/init.d/mysql start
           mysql -e 'CREATE DATABASE ${{ env.DB_DATABASE }};' -u${{ env.DB_USER }} -p${{ env.DB_PASSWORD }}
-
+          mysql -e 'CREATE DATABASE test_dburl;' -u${{ env.DB_USER }} -p${{ env.DB_PASSWORD }}
+          mysql -e "CREATE USER test_dburl_user@localhost IDENTIFIED BY 'test]password';" -u${{ env.DB_USER }} -p${{ env.DB_PASSWORD }}
+          mysql -e "GRANT ALL PRIVILEGES ON test_dburl.* TO test_dburl_user@localhost;" -u${{ env.DB_USER }} -p${{ env.DB_PASSWORD }}
+          mysql -e "FLUSH PRIVILEGES;" -u${{ env.DB_USER }} -p${{ env.DB_PASSWORD }}
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
           python-version: ${{ env[matrix.python-version] }}
 
       - name: Install tox
-        run: pip install "tox<=3.27.1" -U tox-factor
+        run: pip install "tox<=3.27.1" -U tox-factor setuptools
 
       - name: Cache tox environment
         # Preserves .tox directory between runs for faster installs

.github/workflows/continuous-monitoring.yml

@@ -4,6 +4,10 @@ on:
   schedule:
     - cron:  '*/10 * * * *'
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   smoke-tests:
     name: Run smoke tests
@@ -13,10 +17,9 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
+        uses: aws-actions/configure-aws-credentials@v4
         with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          role-to-assume: ${{ secrets.AWS_INTEG_TEST_ROLE_ARN }}
           aws-region: us-east-1
 
       - uses: actions/setup-python@v4

CHANGELOG.rst

@@ -5,6 +5,32 @@ CHANGELOG
 Unreleased
 ==========
 
+2.14.0
+==========
+* bugfix: Fix warning message condition for subsegment ending `https://github.com/aws/aws-xray-sdk-python/pull/434`
+
+2.13.1
+==========
+* improvement: Bump idna from 3.6 to 3.7 in /sample-apps/flask `https://github.com/aws/aws-xray-sdk-python/pull/425`
+* bugfix: Fix end_time param type docstring from int to float `https://github.com/aws/aws-xray-sdk-python/pull/426`
+* improvement:  Bump werkzeug from 3.0.1 to 3.0.3 in /sample-apps/flask `https://github.com/aws/aws-xray-sdk-python/pull/428`
+* improvement: [LambdaContext] Create dummy segment when trace header is incomplete `https://github.com/aws/aws-xray-sdk-python/pull/429`
+* bugfix: [LambdaContext] Fix logging to only happen inside lambda function `https://github.com/aws/aws-xray-sdk-python/pull/431`
+
+2.13.0
+==========
+* bugfix: Fix passing multiple values in testenv.passenv in tox.ini `https://github.com/aws/aws-xray-sdk-python/pull/399`
+* improvement: Pin flask < 3.x for flask sqlalchemy tests `https://github.com/aws/aws-xray-sdk-python/pull/412`
+* improvement: Bump werkzeug from 2.2.3 to 3.0.1 in /sample-apps/flask `https://github.com/aws/aws-xray-sdk-python/pull/413`
+* improvement: Fix typo in docs `https://github.com/aws/aws-xray-sdk-python/pull/419`
+* bugfix: Fix sqlalchemy_core patch errors for unencoded special characters in db url `https://github.com/aws/aws-xray-sdk-python/pull/418`
+* bugfix: Fix EB platform version for integration test `https://github.com/aws/aws-xray-sdk-python/pull/420`
+
+2.12.1
+==========
+* bugfix: set_trace_entity() in lambda adds segment to thread `PR409 https://github.com/aws/aws-xray-sdk-python/pull/409`
+* bugfix: Cleanup after drop of support for Python `PR387 https://github.com/aws/aws-xray-sdk-python/pull/387`
+
 2.12.0
 ==========
 * improvement: Default Context Missing Strategy set to Log Error `PR372 https://github.com/aws/aws-xray-sdk-python/pull/372`

aws_xray_sdk/core/context.py

@@ -22,7 +22,7 @@ class Context:
     replace the current stored entities and to clean up the storage.
 
     For any data access or data mutation, if there is no active segment present
-    if will use user-defined behavior to handle such case. By default it throws
+    it will use user-defined behavior to handle such case. By default it throws
     an runtime error.
 
     This data structure is thread-safe.
@@ -44,7 +44,7 @@ def end_segment(self, end_time=None):
         """
         End the current active segment.
 
-        :param int end_time: epoch in seconds. If not specified the current
+        :param float end_time: epoch in seconds. If not specified the current
             system time will be used.
         """
         entity = self.get_trace_entity()
@@ -75,14 +75,16 @@ def end_subsegment(self, end_time=None):
         End the current active segment. Return False if there is no
         subsegment to end.
 
-        :param int end_time: epoch in seconds. If not specified the current
+        :param float end_time: epoch in seconds. If not specified the current
             system time will be used.
         """
-        subsegment = self.get_trace_entity()
-        if self._is_subsegment(subsegment):
-            subsegment.close(end_time)
+        entity = self.get_trace_entity()
+        if self._is_subsegment(entity):
+            entity.close(end_time)
             self._local.entities.pop()
             return True
+        elif isinstance(entity, DummySegment):
+            return False
         else:
             log.warning("No subsegment to end.")
             return False

aws_xray_sdk/core/lambda_launcher.py

@@ -3,6 +3,7 @@
 import threading
 
 from aws_xray_sdk import global_sdk_config
+from .models.dummy_entities import DummySegment
 from .models.facade_segment import FacadeSegment
 from .models.trace_header import TraceHeader
 from .context import Context
@@ -44,7 +45,7 @@ class LambdaContext(Context):
     """
     Lambda service will generate a segment for each function invocation which
     cannot be mutated. The context doesn't keep any manually created segment
-    but instead every time ``get_trace_entity()`` gets called it refresh the facade
+    but instead every time ``get_trace_entity()`` gets called it refresh the
     segment based on environment variables set by Lambda worker.
     """
     def __init__(self):
@@ -65,19 +66,31 @@ def end_segment(self, end_time=None):
 
     def put_subsegment(self, subsegment):
         """
-        Refresh the facade segment every time this function is invoked to prevent
+        Refresh the segment every time this function is invoked to prevent
         a new subsegment from being attached to a leaked segment/subsegment.
         """
         current_entity = self.get_trace_entity()
 
-        if not self._is_subsegment(current_entity) and current_entity.initializing:
-            if global_sdk_config.sdk_enabled():
+        if not self._is_subsegment(current_entity) and (getattr(current_entity, 'initializing', None) or isinstance(current_entity, DummySegment)):
+            if global_sdk_config.sdk_enabled() and not os.getenv(LAMBDA_TRACE_HEADER_KEY):
                 log.warning("Subsegment %s discarded due to Lambda worker still initializing" % subsegment.name)
             return
 
         current_entity.add_subsegment(subsegment)
         self._local.entities.append(subsegment)
 
+    def set_trace_entity(self, trace_entity):
+        """
+        For Lambda context, we additionally store the segment in the thread local.
+        """
+        if self._is_subsegment(trace_entity):
+            segment = trace_entity.parent_segment
+        else:
+            segment = trace_entity
+
+        setattr(self._local, 'segment', segment)
+        setattr(self._local, 'entities', [trace_entity])
+
     def get_trace_entity(self):
         self._refresh_context()
         if getattr(self._local, 'entities', None):
@@ -87,9 +100,9 @@ def get_trace_entity(self):
 
     def _refresh_context(self):
         """
-        Get current facade segment. To prevent resource leaking in Lambda worker,
+        Get current segment. To prevent resource leaking in Lambda worker,
         every time there is segment present, we compare its trace id to current
-        environment variables. If it is different we create a new facade segment
+        environment variables. If it is different we create a new segment
         and clean up subsegments stored.
         """
         header_str = os.getenv(LAMBDA_TRACE_HEADER_KEY)
@@ -124,8 +137,8 @@ def handle_context_missing(self):
 
     def _initialize_context(self, trace_header):
         """
-        Create a facade segment based on environment variables
-        set by AWS Lambda and initialize storage for subsegments.
+        Create a segment based on environment variables set by
+        AWS Lambda and initialize storage for subsegments.
         """
         sampled = None
         if not global_sdk_config.sdk_enabled():
@@ -136,12 +149,17 @@ def _initialize_context(self, trace_header):
         elif trace_header.sampled == 1:
             sampled = True
 
-        segment = FacadeSegment(
-            name='facade',
-            traceid=trace_header.root,
-            entityid=trace_header.parent,
-            sampled=sampled,
-        )
+        segment = None
+        if not trace_header.root or not trace_header.parent or trace_header.sampled is None:
+            segment = DummySegment()
+            log.debug("Creating NoOp/Dummy parent segment")
+        else:
+            segment = FacadeSegment(
+                name='facade',
+                traceid=trace_header.root,
+                entityid=trace_header.parent,
+                sampled=sampled,
+            )
         segment.save_origin_trace_header(trace_header)
         setattr(self._local, 'segment', segment)
         setattr(self._local, 'entities', [])

aws_xray_sdk/core/models/entity.py

@@ -64,7 +64,7 @@ def close(self, end_time=None):
         Close the trace entity by setting `end_time`
         and flip the in progress flag to False.
 
-        :param int end_time: Epoch in seconds. If not specified
+        :param float end_time: Epoch in seconds. If not specified
             current time will be used.
         """
         self._check_ended()

aws_xray_sdk/core/models/subsegment.py

@@ -133,7 +133,7 @@ def close(self, end_time=None):
         and flip the in progress flag to False. Also decrement
         parent segment's ref counter by 1.
 
-        :param int end_time: Epoch in seconds. If not specified
+        :param float end_time: Epoch in seconds. If not specified
             current time will be used.
         """
         super().close(end_time)

aws_xray_sdk/core/patcher.py

@@ -49,6 +49,18 @@
 
 
 def patch_all(double_patch=False):
+    """
+    The X-Ray Python SDK supports patching aioboto3, aiobotocore, boto3, botocore, pynamodb, requests, 
+    sqlite3, mysql, httplib, pymongo, pymysql, psycopg2, pg8000, sqlalchemy_core, httpx, and mysql-connector.
+
+    To patch all supported libraries::
+
+        from aws_xray_sdk.core import patch_all
+
+        patch_all()
+
+    :param bool double_patch: enable or disable patching of indirect dependencies.
+    """
     if double_patch:
         patch(SUPPORTED_MODULES, raise_errors=False)
     else:
@@ -68,6 +80,16 @@ def _is_valid_import(module):
 
 
 def patch(modules_to_patch, raise_errors=True, ignore_module_patterns=None):
+    """
+    To patch specific modules::
+
+        from aws_xray_sdk.core import patch
+
+        i_want_to_patch = ('botocore') # a tuple that contains the libs you want to patch
+        patch(i_want_to_patch)
+
+    :param tuple modules_to_patch: a tuple containing the list of libraries to be patched
+    """
     enabled = global_sdk_config.sdk_enabled()
     if not enabled:
         log.debug("Skipped patching modules %s because the SDK is currently disabled." % ', '.join(modules_to_patch))

aws_xray_sdk/core/recorder.py

@@ -88,6 +88,7 @@ def configure(self, sampling=None, plugins=None,
 
         Configure needs to run before patching thrid party libraries
         to avoid creating dangling subsegment.
+
         :param bool sampling: If sampling is enabled, every time the recorder
             creates a segment it decides whether to send this segment to
             the X-Ray daemon. This setting is not used if the recorder

aws_xray_sdk/core/sampling/local/sampling_rule.py

@@ -5,7 +5,7 @@
 
 class SamplingRule:
     """
-    One SamolingRule represents one rule defined from local rule json file
+    One SamplingRule represents one rule defined from local rule json file
     or from a dictionary. It can be either a custom rule or default rule.
     """
     FIXED_TARGET = 'fixed_target'

aws_xray_sdk/ext/sqlalchemy_core/patch.py

@@ -1,6 +1,6 @@
 import logging
 import sys
-from urllib.parse import urlparse, uses_netloc
+from urllib.parse import urlparse, uses_netloc, quote_plus
 
 import wrapt
 from sqlalchemy.sql.expression import ClauseElement
@@ -14,18 +14,21 @@
 def _sql_meta(engine_instance, args):
     try:
         metadata = {}
-        url = urlparse(str(engine_instance.engine.url))
+        # Workaround for https://github.com/sqlalchemy/sqlalchemy/issues/10662
+        # sqlalchemy.engine.url.URL's __repr__ does not url encode username nor password.
+        # This will continue to work once sqlalchemy fixes the bug.
+        sa_url = engine_instance.engine.url
+        username = sa_url.username
+        sa_url = sa_url._replace(username=None, password=None)
+        url = urlparse(str(sa_url))
+        name = url.netloc
+        if username:
+            # Restore url encoded username
+            quoted_username = quote_plus(username)
+            url = url._replace(netloc='{}@{}'.format(quoted_username, url.netloc))
         # Add Scheme to uses_netloc or // will be missing from url.
         uses_netloc.append(url.scheme)
-        if url.password is None:
-            metadata['url'] = url.geturl()
-            name = url.netloc
-        else:
-            # Strip password from URL
-            host_info = url.netloc.rpartition('@')[-1]
-            parts = url._replace(netloc='{}@{}'.format(url.username, host_info))
-            metadata['url'] = parts.geturl()
-            name = host_info
+        metadata['url'] = url.geturl()
         metadata['user'] = url.username
         metadata['database_type'] = engine_instance.engine.name
         try:

aws_xray_sdk/version.py

@@ -1 +1 @@
-VERSION = '2.12.0'
+VERSION = '2.14.0'