Add support for ther EXCLUDE_REGEX environment variable to avoid some keys

This allows for optionally avoiding the copy of some large or otherwise
unnecessary objects for the lambda, without needing to configure wacky
event bridge rules or something similar

Author: rtyler

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lambda-s3-restructure"
-version = "0.3.0"
+version = "0.4.0"
 edition = "2021"
 homepage = "https://github.com/buoyant-data/lambda-s3-restructure"
 authors = [
@@ -9,16 +9,17 @@ authors = [
 
 [dependencies]
 anyhow = "1.0.75"
-aws-config = "0.56.0"
-aws-sdk-s3 = "0.29.0"
-aws_lambda_events = { version = "0.12.0", default-features = false, features = ["sns", "sqs", "s3"] }
+aws-config = { version = "1.5", features = ["behavior-version-latest"]}
+aws-sdk-s3 = "1.49"
+aws_lambda_events = { version = "0.15.1", default-features = false, features = ["sns", "sqs", "s3"] }
 chrono = "0.4.31"
 liquid = "0.26"
 serde = { version = "=1", features = ["rc"] }
-lambda_runtime = "0.8.1"
+lambda_runtime = "0.13.0"
+regex = "1.10"
 routefinder = "0.5.3"
 serde_json = "1.0.105"
-tokio = { version = "1", features = ["macros"] }
+tokio = { version = "1.40", features = ["macros"] }
 tracing = { version = "0.1", features = ["log"] }
 tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "env-filter", "tracing-log"] }
 urlencoding = "2.1.3"

README.adoc

@@ -66,6 +66,10 @@ The following environment variables must be set for the function to run properly
 | _required byt empty by default_
 | A link:https://crates.io/crates/liquid[liquid] template which produces a compatible path for outputting the file into the S3 bucket.
 
+| `EXCLUDE_REGEX`
+| 
+| optional regular expression for keys to exclude from consideration
+
 |===
 
 [WARNING]

src/main.rs

@@ -2,6 +2,7 @@ use aws_lambda_events::event::s3::{S3Entity, S3Event};
 use aws_lambda_events::sqs::SqsEvent;
 use aws_sdk_s3::Client as S3Client;
 use lambda_runtime::{run, service_fn, Error, LambdaEvent};
+use regex::Regex;
 use routefinder::Router;
 use tracing::log::*;
 
@@ -61,6 +62,8 @@ async fn function_handler(
 ) -> Result<(), Error> {
     let input_pattern =
         std::env::var("INPUT_PATTERN").expect("You must define INPUT_PATTERN in the environment");
+    let exclude_regex: Option<Regex> = std::env::var("EXCLUDE_REGEX")
+                            .map(|ex| Regex::new(ex.as_ref()).expect("Failed to compile EXCLUDE_REGEX")).ok();
     let output_template = std::env::var("OUTPUT_TEMPLATE")
         .expect("You must define OUTPUT_TEMPLATE in the environment");
 
@@ -80,6 +83,10 @@ async fn function_handler(
         debug!("Processing {entity:?}");
 
         if let Some(source_key) = entity.object.key {
+            if should_exclude(exclude_regex.as_ref(), &source_key) {
+                continue;
+            }
+
             let parameters = add_builtin_parameters(captured_parameters(&router, &source_key)?);
             let output_key = template.render(&parameters)?;
             info!("Copying {source_key:?} to {output_key:?}");
@@ -110,7 +117,7 @@ async fn main() -> Result<(), Error> {
         .without_time()
         .init();
 
-    let shared_config = aws_config::load_from_env().await;
+    let shared_config = aws_config::from_env().load().await;
     let client = S3Client::new(&shared_config);
     let client_ref = &client;
 
@@ -146,6 +153,15 @@ fn captured_parameters<Handler>(
     Ok(data)
 }
 
+/// Return true if the given key matches the pattern and should be excluded from consideration
+fn should_exclude(pattern: Option<&Regex>, key: &str) -> bool {
+    match pattern {
+        Some(re) => re.is_match(key),
+        None => false,
+    }
+}
+
+/// Introduce the necessary built-in parameters to the `data` for rendering a Handlebars template
 fn add_builtin_parameters(mut data: HashMap<String, String>) -> HashMap<String, String> {
     use chrono::Datelike;
     let now = chrono::Utc::now();
@@ -246,7 +262,7 @@ mod tests {
   ]
 }"#;
 
-        let event: S3Event = serde_json::from_str(&raw_buf)?;
+        let event: S3Event = serde_json::from_str(raw_buf)?;
         Ok(event)
     }
 
@@ -272,4 +288,19 @@ mod tests {
             "databases/oltp/a_table/ds=2023-09-05/some.parquet"
         );
     }
+
+    #[test]
+    fn test_exclude_regex() {
+        let exclude = Some(Regex::new(r#"^path\/to\/table.*"#).expect("Failed to compile regular expression"));
+        let keys = vec![
+            "path/to/alpha",
+            "path/to/bravo/foo.parquet",
+            "path/to/table",
+            "path/to/table/foo.parquet",
+        ];
+
+        let filtered: Vec<_> = keys.iter().filter(|k| !should_exclude(exclude.as_ref(), k)).map(|k| k.clone()).collect();
+        assert_ne!(filtered, keys);
+    }
 }
+