Add support for env-defined endpoint URLs (#3488)

## Motivation and Context
<!--- Why is this change required? What problem does it solve? -->
<!--- If it fixes an open issue, please link to the issue here -->
Part of the endpoint config work I'm doing

## Description
<!--- Describe your changes in detail -->
This change does two things:
- add support for setting an endpoint URL from the env or profile file
- add support for ignoring endpoint URLs sourced from the env and
profile file

## Testing
<!--- Please describe in detail how you tested your changes -->
<!--- Include details of your testing environment, and the tests you ran
to -->
<!--- see how your change affects other areas of the code, etc. -->
I wrote many unit tests.

## Checklist
<!--- If a checkbox below is not applicable, then please DELETE it
rather than leaving it unchecked -->
- [x] I have updated `CHANGELOG.next.toml` if I made changes to the AWS
SDK, generated SDK code, or SDK runtime crates

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

---------

Co-authored-by: John DiSanti <[email protected]>

Author: Zelda Hessler

CHANGELOG.next.toml

@@ -22,3 +22,19 @@ message = "`DefaultS3ExpressIdentityProvider` now uses `BehaviorVersion` threade
 references = ["smithy-rs#3478"]
 meta = { "breaking" = false, "bug" = true, "tada" = false }
 author = "ysaito1001"
+
+[[aws-sdk-rust]]
+message = """
+Users may now set an endpoint URL from the env or profile file:
+
+- env: `AWS_ENDPOINT_URL="http://localhost"`
+- profile: `endpoint_url = http://localhost`
+
+Users may also ignore endpoint URLs sourced from the env and profile files:
+
+- env: `AWS_IGNORE_CONFIGURED_ENDPOINT_URLS="true"`
+- profile: `ignore_configured_endpoint_urls = true`
+"""
+references = ["smithy-rs#3488"]
+meta = { "breaking" = false, "tada" = true, "bug" = false }
+authors = ["Velfi"]

aws/rust-runtime/aws-config/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "aws-config"
-version = "1.1.8"
+version = "1.1.9"
 authors = [
     "AWS Rust SDK Team <[email protected]>",
     "Russell Cohen <[email protected]>",
@@ -26,19 +26,20 @@ allow-compilation = []
 
 [dependencies]
 aws-credential-types = { path = "../../sdk/build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] }
+aws-runtime = { path = "../../sdk/build/aws-sdk/sdk/aws-runtime" }
 aws-sdk-sts = { path = "../../sdk/build/aws-sdk/sdk/sts", default-features = false }
 aws-smithy-async = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-async" }
 aws-smithy-http = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-http" }
 aws-smithy-json = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-json" }
 aws-smithy-runtime = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime", features = ["client"] }
 aws-smithy-runtime-api = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime-api", features = ["client"] }
 aws-smithy-types = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-types" }
-aws-runtime = { path = "../../sdk/build/aws-sdk/sdk/aws-runtime" }
 aws-types = { path = "../../sdk/build/aws-sdk/sdk/aws-types" }
 hyper = { version = "0.14.26", default-features = false }
 time = { version = "0.3.4", features = ["parsing"] }
 tokio = { version = "1.13.1", features = ["sync"] }
 tracing = { version = "0.1" }
+url = "2.3.1"
 
 # implementation detail of IMDS credentials provider
 fastrand = "2.0.0"
@@ -59,7 +60,7 @@ aws-sdk-ssooidc = { path = "../../sdk/build/aws-sdk/sdk/ssooidc", default-featur
 aws-smithy-runtime = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime", features = ["client", "connector-hyper-0-14-x", "test-util"] }
 aws-smithy-runtime-api = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime-api", features = ["test-util"] }
 futures-util = { version = "0.3.29", default-features = false }
-tracing-test = "0.2.1"
+tracing-test = "0.2.4"
 tracing-subscriber = { version = "0.3.16", features = ["fmt", "json"] }
 
 tokio = { version = "1.23.1", features = ["full", "test-util"] }

aws/rust-runtime/aws-config/src/default_provider.rs

@@ -51,3 +51,9 @@ pub mod use_dual_stack;
 /// Default access token provider chain
 #[cfg(feature = "sso")]
 pub mod token;
+
+/// Default "ignore configured endpoint URLs" provider chain
+pub mod ignore_configured_endpoint_urls;
+
+/// Default endpoint URL provider chain
+pub mod endpoint_url;

aws/rust-runtime/aws-config/src/default_provider/endpoint_url.rs

@@ -0,0 +1,79 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+use crate::environment::parse_url;
+use crate::provider_config::ProviderConfig;
+use crate::standard_property::StandardProperty;
+use aws_smithy_types::error::display::DisplayErrorContext;
+
+mod env {
+    pub(super) const ENDPOINT_URL: &str = "AWS_ENDPOINT_URL";
+}
+
+mod profile_key {
+    pub(super) const ENDPOINT_URL: &str = "endpoint_url";
+}
+
+/// Load the value for an endpoint URL
+///
+/// This checks the following sources:
+/// 1. The environment variable `AWS_ENDPOINT_URL=http://localhost`
+/// 2. The profile key `endpoint_url=http://localhost`
+///
+/// If invalid values are found, the provider will return None and an error will be logged.
+pub async fn endpoint_url_provider(provider_config: &ProviderConfig) -> Option<String> {
+    StandardProperty::new()
+        .env(env::ENDPOINT_URL)
+        .profile(profile_key::ENDPOINT_URL)
+        .validate(provider_config, parse_url)
+        .await
+        .map_err(
+            |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for endpoint URL setting"),
+        )
+        .unwrap_or(None)
+}
+
+#[cfg(test)]
+mod test {
+    use super::endpoint_url_provider;
+    use super::env;
+    use crate::profile::profile_file::{ProfileFileKind, ProfileFiles};
+    use crate::provider_config::ProviderConfig;
+    use aws_types::os_shim_internal::{Env, Fs};
+    use tracing_test::traced_test;
+
+    #[tokio::test]
+    #[traced_test]
+    async fn log_error_on_invalid_value() {
+        let conf =
+            ProviderConfig::empty().with_env(Env::from_slice(&[(env::ENDPOINT_URL, "not-a-url")]));
+        assert_eq!(None, endpoint_url_provider(&conf).await);
+        assert!(logs_contain("invalid value for endpoint URL setting"));
+        assert!(logs_contain(env::ENDPOINT_URL));
+    }
+
+    #[tokio::test]
+    #[traced_test]
+    async fn environment_priority() {
+        let conf = ProviderConfig::empty()
+            .with_env(Env::from_slice(&[(env::ENDPOINT_URL, "http://localhost")]))
+            .with_profile_config(
+                Some(
+                    ProfileFiles::builder()
+                        .with_file(ProfileFileKind::Config, "conf")
+                        .build(),
+                ),
+                None,
+            )
+            .with_fs(Fs::from_slice(&[(
+                "conf",
+                "[default]\nendpoint_url = http://production",
+            )]));
+        assert_eq!(
+            Some("http://localhost".to_owned()),
+            endpoint_url_provider(&conf).await,
+        );
+    }
+}

aws/rust-runtime/aws-config/src/default_provider/ignore_configured_endpoint_urls.rs

@@ -0,0 +1,88 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+use crate::environment::parse_bool;
+use crate::provider_config::ProviderConfig;
+use crate::standard_property::StandardProperty;
+use aws_smithy_types::error::display::DisplayErrorContext;
+
+mod env {
+    pub(super) const IGNORE_CONFIGURED_ENDPOINT_URLS: &str = "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS";
+}
+
+mod profile_key {
+    pub(super) const IGNORE_CONFIGURED_ENDPOINT_URLS: &str = "ignore_configured_endpoint_urls";
+}
+
+/// Load the value for "ignore configured endpoint URLs"
+///
+/// This checks the following sources:
+/// 1. The environment variable `AWS_IGNORE_CONFIGURED_ENDPOINT_URLS_ENDPOINT=true/false`
+/// 2. The profile key `ignore_configured_endpoint_urls=true/false`
+///
+/// If invalid values are found, the provider will return None and an error will be logged.
+pub async fn ignore_configured_endpoint_urls_provider(
+    provider_config: &ProviderConfig,
+) -> Option<bool> {
+    StandardProperty::new()
+        .env(env::IGNORE_CONFIGURED_ENDPOINT_URLS)
+        .profile(profile_key::IGNORE_CONFIGURED_ENDPOINT_URLS)
+        .validate(provider_config, parse_bool)
+        .await
+        .map_err(
+            |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for 'ignore configured endpoint URLs' setting"),
+        )
+        .unwrap_or(None)
+}
+
+#[cfg(test)]
+mod test {
+    use super::env;
+    use super::ignore_configured_endpoint_urls_provider;
+    use crate::profile::profile_file::{ProfileFileKind, ProfileFiles};
+    use crate::provider_config::ProviderConfig;
+    use aws_types::os_shim_internal::{Env, Fs};
+    use tracing_test::traced_test;
+
+    #[tokio::test]
+    #[traced_test]
+    async fn log_error_on_invalid_value() {
+        let conf = ProviderConfig::empty().with_env(Env::from_slice(&[(
+            env::IGNORE_CONFIGURED_ENDPOINT_URLS,
+            "not-a-boolean",
+        )]));
+        assert_eq!(None, ignore_configured_endpoint_urls_provider(&conf).await,);
+        assert!(logs_contain(
+            "invalid value for 'ignore configured endpoint URLs' setting"
+        ));
+        assert!(logs_contain(env::IGNORE_CONFIGURED_ENDPOINT_URLS));
+    }
+
+    #[tokio::test]
+    #[traced_test]
+    async fn environment_priority() {
+        let conf = ProviderConfig::empty()
+            .with_env(Env::from_slice(&[(
+                env::IGNORE_CONFIGURED_ENDPOINT_URLS,
+                "TRUE",
+            )]))
+            .with_profile_config(
+                Some(
+                    ProfileFiles::builder()
+                        .with_file(ProfileFileKind::Config, "conf")
+                        .build(),
+                ),
+                None,
+            )
+            .with_fs(Fs::from_slice(&[(
+                "conf",
+                "[default]\nignore_configured_endpoint_urls = false",
+            )]));
+        assert_eq!(
+            Some(true),
+            ignore_configured_endpoint_urls_provider(&conf).await,
+        );
+    }
+}

aws/rust-runtime/aws-config/src/environment/mod.rs

@@ -6,7 +6,7 @@
 //! Providers that load configuration from environment variables
 
 use std::error::Error;
-use std::fmt::{Display, Formatter};
+use std::fmt;
 
 /// Load credentials from the environment
 pub mod credentials;
@@ -21,9 +21,9 @@ pub(crate) struct InvalidBooleanValue {
     value: String,
 }
 
-impl Display for InvalidBooleanValue {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{} was not a valid boolean", self.value)
+impl fmt::Display for InvalidBooleanValue {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{} is not a valid boolean", self.value)
     }
 }
 
@@ -40,3 +40,26 @@ pub(crate) fn parse_bool(value: &str) -> Result<bool, InvalidBooleanValue> {
         })
     }
 }
+
+#[derive(Debug)]
+pub(crate) struct InvalidUrlValue {
+    value: String,
+}
+
+impl fmt::Display for InvalidUrlValue {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{} is not a valid URL", self.value)
+    }
+}
+
+impl Error for InvalidUrlValue {}
+
+pub(crate) fn parse_url(value: &str) -> Result<String, InvalidUrlValue> {
+    match url::Url::parse(value) {
+        // We discard the parse result because it includes a trailing slash
+        Ok(_) => Ok(value.to_string()),
+        Err(_) => Err(InvalidUrlValue {
+            value: value.to_string(),
+        }),
+    }
+}