diff --git a/Cargo.toml b/Cargo.toml index 624262c0..38fc6046 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,11 +10,6 @@ readme = "README.md" keywords = ["object-storage", "minio", "s3"] categories = ["api-bindings", "web-programming::http-client"] -[dependencies.reqwest] -version = "0.12.22" -default-features = false -features = ["stream"] - [features] default = ["default-tls", "default-crypto"] default-tls = ["reqwest/default-tls"] @@ -23,21 +18,29 @@ rustls-tls = ["reqwest/rustls-tls"] default-crypto = ["dep:sha2", "dep:hmac"] ring = ["dep:ring"] +[workspace.dependencies] +uuid = "1.18" +futures-util = "0.3" +reqwest = { version = "0.12", default-features = false } +bytes = "1.10" +async-std = "1.13" + + [dependencies] +uuid = { workspace = true, features = ["v4"] } +futures-util = { workspace = true } +bytes = { workspace = true } +async-std = { workspace = true, features = ["attributes"] } +reqwest = { workspace = true, features = ["stream"] } + async-recursion = "1.1.1" -async-std = { version = "1.13.1", features = ["attributes"] } async-stream = "0.3.6" async-trait = "0.1.88" base64 = "0.22.1" -byteorder = "1.5.0" -bytes = "1.10.1" chrono = "0.4.41" crc = "3.3.0" dashmap = "6.1.0" -derivative = "2.2.0" env_logger = "0.11.8" -futures-util = "0.3.31" -hex = "0.4.3" hmac = { version = "0.12.1", optional = true } hyper = { version = "1.6.0", features = ["full"] } lazy_static = "1.5.0" @@ -46,26 +49,25 @@ md5 = "0.8.0" multimap = "0.10.1" percent-encoding = "2.3.1" url = "2.5.4" -rand = { version = "0.8.5", features = ["small_rng"] } regex = "1.11.1" ring = { version = "0.17.14", optional = true, default-features = false, features = ["alloc"] } serde = { version = "1.0.219", features = ["derive"] } -serde_json = "1.0.140" +serde_json = "1.0.142" sha2 = { version = "0.10.9", optional = true } urlencoding = "2.1.3" xmltree = "0.11.0" -futures = "0.3.31" http = "1.3.1" -thiserror = "2.0.12" +thiserror = "2.0.14" [dev-dependencies] +minio-common = { path = "./common" } +minio-macros = { path = "./macros" } tokio = { version = "1.47.1", features = ["full"] } -minio_common = { path = "./common" } async-std = { version = "1.13.1", features = ["attributes", "tokio1"] } clap = { version = "4.5.44", features = ["derive"] } +rand = { version = "0.9.2", features = ["small_rng"] } quickcheck = "1.0.3" criterion = "0.7.0" -minio-macros = { path = "./macros" } [lib] name = "minio" diff --git a/common/Cargo.toml b/common/Cargo.toml index 805523c0..72e89544 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,20 +1,23 @@ [package] -name = "minio_common" +name = "minio-common" version = "0.1.0" edition = "2024" [dependencies] minio = {path = ".." } +uuid = { workspace = true, features = ["v4"] } +reqwest = { workspace = true } +bytes = { workspace = true } +async-std = { workspace = true } + +futures-io = "0.3.31" tokio = { version = "1.47.1", features = ["full"] } -async-std = "1.13.1" -rand = { version = "0.8.5", features = ["small_rng"] } -bytes = "1.10.1" +rand = { version = "0.9.2", features = ["small_rng"] } + log = "0.4.27" chrono = "0.4.41" -reqwest = "0.12.22" + http = "1.3.1" -futures = "0.3.31" -uuid = { version = "1.18.0", features = ["v4"] } [lib] name = "minio_common" diff --git a/common/src/cleanup_guard.rs b/common/src/cleanup_guard.rs index cf9d82ba..2347b987 100644 --- a/common/src/cleanup_guard.rs +++ b/common/src/cleanup_guard.rs @@ -38,7 +38,7 @@ impl CleanupGuard { pub async fn cleanup(client: Client, bucket_name: &str) { tokio::select!( _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { - eprintln!("Cleanup timeout after 60s while removing bucket {}", bucket_name); + eprintln!("Cleanup timeout after 60s while removing bucket {bucket_name}"); }, outcome = client.delete_and_purge_bucket(bucket_name) => { match outcome { @@ -46,7 +46,7 @@ pub async fn cleanup(client: Client, bucket_name: &str) { //eprintln!("Bucket {} removed successfully", bucket_name); } Err(e) => { - eprintln!("Error removing bucket '{}':\n{}", bucket_name, e); + eprintln!("Error removing bucket '{bucket_name}':\n{e}"); } } } diff --git a/common/src/example.rs b/common/src/example.rs index e045dd96..21009431 100644 --- a/common/src/example.rs +++ b/common/src/example.rs @@ -140,7 +140,7 @@ pub fn create_bucket_replication_config_example(dst_bucket: &str) -> Replication rules: vec![ReplicationRule { id: Some(String::from("rule1")), destination: Destination { - bucket_arn: String::from(&format!("arn:aws:s3:::{}", dst_bucket)), + bucket_arn: String::from(&format!("arn:aws:s3:::{dst_bucket}")), ..Default::default() }, filter: Some(Filter { @@ -170,8 +170,8 @@ pub fn create_object_lock_config_example() -> ObjectLockConfig { pub fn create_post_policy_example(bucket_name: &str, object_name: &str) -> PostPolicy { let expiration: DateTime = utc_now() + chrono::Duration::days(5); - let mut policy = PostPolicy::new(&bucket_name, expiration).unwrap(); - policy.add_equals_condition("key", &object_name).unwrap(); + let mut policy = PostPolicy::new(bucket_name, expiration).unwrap(); + policy.add_equals_condition("key", object_name).unwrap(); policy .add_content_length_range_condition(1024 * 1024, 4 * 1024 * 1024) .unwrap(); @@ -189,7 +189,7 @@ pub fn create_select_content_data() -> (String, String) { (body, data) } pub fn create_select_content_request() -> SelectRequest { - let request = SelectRequest::new_csv_input_output( + SelectRequest::new_csv_input_output( "select * from S3Object", CsvInputSerialization { compression_type: None, @@ -209,6 +209,5 @@ pub fn create_select_content_request() -> SelectRequest { record_delimiter: None, }, ) - .unwrap(); - request + .unwrap() } diff --git a/common/src/rand_reader.rs b/common/src/rand_reader.rs index 72f4f36b..c256baf3 100644 --- a/common/src/rand_reader.rs +++ b/common/src/rand_reader.rs @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use futures::AsyncRead; +use futures_io::AsyncRead; use std::io; use std::pin::Pin; use std::task::{Context, Poll}; @@ -34,7 +34,7 @@ impl io::Read for RandReader { let bytes_read = buf.len().min(self.size as usize); if bytes_read > 0 { - let random: &mut dyn rand::RngCore = &mut rand::thread_rng(); + let random: &mut dyn rand::RngCore = &mut rand::rng(); random.fill_bytes(&mut buf[0..bytes_read]); } @@ -53,7 +53,7 @@ impl AsyncRead for RandReader { let bytes_read = buf.len().min(self.size as usize); if bytes_read > 0 { - let random: &mut dyn rand::RngCore = &mut rand::thread_rng(); + let random: &mut dyn rand::RngCore = &mut rand::rng(); random.fill_bytes(&mut buf[0..bytes_read]); } diff --git a/common/src/rand_src.rs b/common/src/rand_src.rs index fef9c2cc..a91941da 100644 --- a/common/src/rand_src.rs +++ b/common/src/rand_src.rs @@ -15,7 +15,7 @@ use async_std::stream::Stream; use bytes::Bytes; -use futures::io::AsyncRead; +use futures_io::AsyncRead; use rand::prelude::SmallRng; use rand::{RngCore, SeedableRng}; use std::io; @@ -30,7 +30,7 @@ pub struct RandSrc { impl RandSrc { #[allow(dead_code)] pub fn new(size: u64) -> RandSrc { - let rng = SmallRng::from_entropy(); + let rng: SmallRng = SmallRng::from_os_rng(); RandSrc { size, rng } } } diff --git a/common/src/test_context.rs b/common/src/test_context.rs index 58b10fee..291ffc51 100644 --- a/common/src/test_context.rs +++ b/common/src/test_context.rs @@ -84,10 +84,10 @@ impl TestContext { let host: String = std::env::var("SERVER_ENDPOINT").unwrap_or(DEFAULT_SERVER_ENDPOINT.to_string()); - log::debug!("SERVER_ENDPOINT={}", host); + log::debug!("SERVER_ENDPOINT={host}"); let access_key: String = std::env::var("ACCESS_KEY").unwrap_or(DEFAULT_ACCESS_KEY.to_string()); - log::debug!("ACCESS_KEY={}", access_key); + log::debug!("ACCESS_KEY={access_key}"); let secret_key: String = std::env::var("SECRET_KEY").unwrap_or(DEFAULT_SECRET_KEY.to_string()); log::debug!("SECRET_KEY=*****"); @@ -95,19 +95,19 @@ impl TestContext { .unwrap_or(DEFAULT_ENABLE_HTTPS.to_string()) .parse() .unwrap_or(false); - log::debug!("ENABLE_HTTPS={}", secure); + log::debug!("ENABLE_HTTPS={secure}"); let ssl_cert: String = std::env::var("MINIO_SSL_CERT_FILE").unwrap_or(DEFAULT_SSL_CERT_FILE.to_string()); - log::debug!("MINIO_SSL_CERT_FILE={}", ssl_cert); + log::debug!("MINIO_SSL_CERT_FILE={ssl_cert}"); let ssl_cert_file: PathBuf = ssl_cert.into(); let ignore_cert_check: bool = std::env::var("IGNORE_CERT_CHECK") .unwrap_or(DEFAULT_IGNORE_CERT_CHECK.to_string()) .parse() .unwrap_or(true); - log::debug!("IGNORE_CERT_CHECK={}", ignore_cert_check); + log::debug!("IGNORE_CERT_CHECK={ignore_cert_check}"); let region: String = std::env::var("SERVER_REGION").unwrap_or(DEFAULT_SERVER_REGION.to_string()); - log::debug!("SERVER_REGION={:?}", region); + log::debug!("SERVER_REGION={region:?}"); let mut base_url: BaseUrl = host.parse().unwrap(); base_url.https = secure; diff --git a/common/src/utils.rs b/common/src/utils.rs index cbc9debe..c9fdbffb 100644 --- a/common/src/utils.rs +++ b/common/src/utils.rs @@ -15,8 +15,9 @@ use http::{Response as HttpResponse, StatusCode}; use minio::s3::error::Error; -use rand::distributions::Standard; -use rand::{Rng, thread_rng}; + +use rand::Rng; +use rand::distr::StandardUniform; use uuid::Uuid; pub fn rand_bucket_name() -> String { @@ -28,9 +29,9 @@ pub fn rand_object_name() -> String { } pub fn rand_object_name_utf8(len: usize) -> String { - let rng = thread_rng(); - rng.sample_iter::(Standard) - .filter(|c| !c.is_control()) + let rng = rand::rng(); + rng.sample_iter(StandardUniform) + .filter(|c: &char| !c.is_control()) .take(len) .collect() } @@ -39,9 +40,9 @@ pub async fn get_bytes_from_response(v: Result) -> byt match v { Ok(r) => match r.bytes().await { Ok(b) => b, - Err(e) => panic!("{:?}", e), + Err(e) => panic!("{e:?}"), }, - Err(e) => panic!("{:?}", e), + Err(e) => panic!("{e:?}"), } } @@ -52,5 +53,5 @@ pub fn get_response_from_bytes(bytes: bytes::Bytes) -> reqwest::Response { .body(bytes) .expect("Failed to build HTTP response"); - reqwest::Response::try_from(http_response).expect("Failed to convert to reqwest::Response") + reqwest::Response::from(http_response) } diff --git a/examples/append_object.rs b/examples/append_object.rs index 132229b7..c24e0079 100644 --- a/examples/append_object.rs +++ b/examples/append_object.rs @@ -22,7 +22,7 @@ use minio::s3::response::{AppendObjectResponse, StatObjectResponse}; use minio::s3::segmented_bytes::SegmentedBytes; use minio::s3::types::S3Api; use rand::Rng; -use rand::distributions::Alphanumeric; +use rand::distr::Alphanumeric; #[tokio::main] async fn main() -> Result<(), Box> { @@ -78,7 +78,7 @@ async fn main() -> Result<(), Box> { } fn random_string(len: usize) -> String { - rand::thread_rng() + rand::rng() .sample_iter(&Alphanumeric) .take(len) .map(char::from) diff --git a/macros/Cargo.toml b/macros/Cargo.toml index 85d0d5a3..9887102d 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -3,18 +3,19 @@ name = "minio-macros" version = "0.1.0" edition = "2024" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[lib] -proc-macro = true - - [dependencies] +uuid = { workspace = true, features = ["v4"] } +futures-util = { workspace = true } + syn = "2.0.104" -proc-macro2 = "1.0.95" +proc-macro2 = "1.0.97" quote = "1.0.40" darling = "0.21.0" darling_core = "0.21.0" -uuid = { version = "1.17.0", features = ["v4"] } [dev-dependencies] -minio_common = { path = "../common" } \ No newline at end of file +minio-common = { path = "../common" } + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +proc-macro = true diff --git a/macros/src/lib.rs b/macros/src/lib.rs index d1cc69ed..2062fa0a 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -104,6 +104,6 @@ pub fn test( // Expand the macro match test_attr::expand_test_macro(args, input_fn) { Ok(expanded) => expanded.into(), - Err(err) => err.into(), + Err(err) => err, } } diff --git a/macros/src/test_attr.rs b/macros/src/test_attr.rs index e9b0ee09..ef1e102a 100644 --- a/macros/src/test_attr.rs +++ b/macros/src/test_attr.rs @@ -55,7 +55,7 @@ impl MacroArgs { } // Validate that the function has exactly two arguments: ctx and bucket_name - if func.sig.inputs.len() != 2 && !self.no_bucket.is_present() { + if (func.sig.inputs.len() != 2) && !self.no_bucket.is_present() { let error_msg = "Minio test function must have exactly two arguments: (ctx: TestContext, bucket_name: String)"; return Err(proc_macro::TokenStream::from( Error::custom(error_msg) @@ -80,18 +80,18 @@ impl MacroArgs { } } - // Check second argument (bucket_name: String) - if !self.no_bucket.is_present() { - if let Some(FnArg::Typed(pat_type)) = iter.next() { - let type_str = pat_type.ty.to_token_stream().to_string(); - if !type_str.contains("String") { - let error_msg = "Second argument must be of type String"; - return Err(proc_macro::TokenStream::from( - Error::custom(error_msg) - .with_span(&pat_type.span()) - .write_errors(), - )); - } + // Check the second argument (bucket_name: String) + if !self.no_bucket.is_present() + && let Some(FnArg::Typed(pat_type)) = iter.next() + { + let type_str = pat_type.ty.to_token_stream().to_string(); + if !type_str.contains("String") { + let error_msg = "Second argument must be of type String"; + return Err(proc_macro::TokenStream::from( + Error::custom(error_msg) + .with_span(&pat_type.span()) + .write_errors(), + )); } } @@ -124,7 +124,7 @@ pub(crate) fn expand_test_macro( // Setup common prelude let prelude = quote!( - use ::futures::FutureExt; + use ::futures_util::FutureExt; use ::std::panic::AssertUnwindSafe; use ::minio::s3::types::S3Api; use ::minio::s3::response::a_response_traits::HasBucket; diff --git a/src/s3/builders/append_object.rs b/src/s3/builders/append_object.rs index 2e647028..75631ba0 100644 --- a/src/s3/builders/append_object.rs +++ b/src/s3/builders/append_object.rs @@ -20,7 +20,7 @@ use crate::s3::builders::{ use crate::s3::error::ValidationErr; use crate::s3::error::{Error, IoError}; use crate::s3::header_constants::*; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::a_response_traits::HasObjectSize; use crate::s3::response::{AppendObjectResponse, StatObjectResponse}; use crate::s3::segmented_bytes::SegmentedBytes; diff --git a/src/s3/builders/bucket_common.rs b/src/s3/builders/bucket_common.rs index 54128e91..d83a18df 100644 --- a/src/s3/builders/bucket_common.rs +++ b/src/s3/builders/bucket_common.rs @@ -14,7 +14,7 @@ // limitations under the License. use crate::s3::client::Client; -use crate::s3::multimap::Multimap; +use crate::s3::multimap_ext::Multimap; use std::marker::PhantomData; /// Common parameters for bucket operations diff --git a/src/s3/builders/copy_object.rs b/src/s3/builders/copy_object.rs index 7bef7bc4..ecdd5e24 100644 --- a/src/s3/builders/copy_object.rs +++ b/src/s3/builders/copy_object.rs @@ -17,7 +17,7 @@ use crate::s3::Client; use crate::s3::client::{MAX_MULTIPART_COUNT, MAX_PART_SIZE}; use crate::s3::error::{Error, ValidationErr}; use crate::s3::header_constants::*; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::a_response_traits::HasEtagFromBody; use crate::s3::response::{ AbortMultipartUploadResponse, CompleteMultipartUploadResponse, ComposeObjectResponse, diff --git a/src/s3/builders/create_bucket.rs b/src/s3/builders/create_bucket.rs index f61b258f..a69455fa 100644 --- a/src/s3/builders/create_bucket.rs +++ b/src/s3/builders/create_bucket.rs @@ -17,7 +17,7 @@ use crate::s3::Client; use crate::s3::client::DEFAULT_REGION; use crate::s3::error::ValidationErr; use crate::s3::header_constants::*; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::CreateBucketResponse; use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::types::{S3Api, S3Request, ToS3Request}; diff --git a/src/s3/builders/delete_object_tagging.rs b/src/s3/builders/delete_object_tagging.rs index 9064f97a..3e9711d2 100644 --- a/src/s3/builders/delete_object_tagging.rs +++ b/src/s3/builders/delete_object_tagging.rs @@ -15,7 +15,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::DeleteObjectTaggingResponse; use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::utils::{check_bucket_name, check_object_name, insert}; diff --git a/src/s3/builders/delete_objects.rs b/src/s3/builders/delete_objects.rs index f947169d..c7cbe8fb 100644 --- a/src/s3/builders/delete_objects.rs +++ b/src/s3/builders/delete_objects.rs @@ -17,7 +17,7 @@ use crate::s3::Client; use crate::s3::client::MAX_MULTIPART_COUNT; use crate::s3::error::{Error, ValidationErr}; use crate::s3::header_constants::*; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::{DeleteError, DeleteObjectResponse, DeleteObjectsResponse}; use crate::s3::types::{ListEntry, S3Api, S3Request, ToS3Request, ToStream}; use crate::s3::utils::{check_bucket_name, check_object_name, insert, md5sum_hash}; diff --git a/src/s3/builders/get_bucket_lifecycle.rs b/src/s3/builders/get_bucket_lifecycle.rs index 095a64b5..516533f7 100644 --- a/src/s3/builders/get_bucket_lifecycle.rs +++ b/src/s3/builders/get_bucket_lifecycle.rs @@ -15,7 +15,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::GetBucketLifecycleResponse; use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::utils::{check_bucket_name, insert}; diff --git a/src/s3/builders/get_bucket_tagging.rs b/src/s3/builders/get_bucket_tagging.rs index 436ab84b..9f8e1ee5 100644 --- a/src/s3/builders/get_bucket_tagging.rs +++ b/src/s3/builders/get_bucket_tagging.rs @@ -15,7 +15,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::Multimap; +use crate::s3::multimap_ext::Multimap; use crate::s3::response::GetBucketTaggingResponse; use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::utils::{check_bucket_name, insert}; diff --git a/src/s3/builders/get_object.rs b/src/s3/builders/get_object.rs index 6dd8d83a..4383cc97 100644 --- a/src/s3/builders/get_object.rs +++ b/src/s3/builders/get_object.rs @@ -16,7 +16,7 @@ use crate::s3::client::Client; use crate::s3::error::ValidationErr; use crate::s3::header_constants::*; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::GetObjectResponse; use crate::s3::sse::{Sse, SseCustomerKey}; use crate::s3::types::{S3Api, S3Request, ToS3Request}; diff --git a/src/s3/builders/get_object_legal_hold.rs b/src/s3/builders/get_object_legal_hold.rs index 685485d7..cf474f4a 100644 --- a/src/s3/builders/get_object_legal_hold.rs +++ b/src/s3/builders/get_object_legal_hold.rs @@ -15,7 +15,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::GetObjectLegalHoldResponse; use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::utils::{check_bucket_name, check_object_name, insert}; diff --git a/src/s3/builders/get_object_prompt.rs b/src/s3/builders/get_object_prompt.rs index c6c6b1f2..185e4984 100644 --- a/src/s3/builders/get_object_prompt.rs +++ b/src/s3/builders/get_object_prompt.rs @@ -15,7 +15,7 @@ use crate::s3::client::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::GetObjectPromptResponse; use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::sse::SseCustomerKey; diff --git a/src/s3/builders/get_object_retention.rs b/src/s3/builders/get_object_retention.rs index 47c446b2..ad02f66c 100644 --- a/src/s3/builders/get_object_retention.rs +++ b/src/s3/builders/get_object_retention.rs @@ -15,7 +15,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::GetObjectRetentionResponse; use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::utils::{check_bucket_name, check_object_name, insert}; diff --git a/src/s3/builders/get_object_tagging.rs b/src/s3/builders/get_object_tagging.rs index 25419cbf..3b557758 100644 --- a/src/s3/builders/get_object_tagging.rs +++ b/src/s3/builders/get_object_tagging.rs @@ -15,7 +15,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::GetObjectTaggingResponse; use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::utils::{check_bucket_name, check_object_name, insert}; diff --git a/src/s3/builders/get_presigned_object_url.rs b/src/s3/builders/get_presigned_object_url.rs index 0f01b04f..62960ff4 100644 --- a/src/s3/builders/get_presigned_object_url.rs +++ b/src/s3/builders/get_presigned_object_url.rs @@ -17,7 +17,7 @@ use crate::s3::Client; use crate::s3::creds::Credentials; use crate::s3::error::Error; use crate::s3::header_constants::*; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::GetPresignedObjectUrlResponse; use crate::s3::signer::presign_v4; use crate::s3::utils::{UtcTime, check_bucket_name, check_object_name, utc_now}; diff --git a/src/s3/builders/get_presigned_policy_form_data.rs b/src/s3/builders/get_presigned_policy_form_data.rs index ec04cdfe..ec769ae9 100644 --- a/src/s3/builders/get_presigned_policy_form_data.rs +++ b/src/s3/builders/get_presigned_policy_form_data.rs @@ -19,7 +19,7 @@ use crate::s3::error::{Error, ValidationErr}; use crate::s3::header_constants::*; use crate::s3::signer::post_presign_v4; use crate::s3::utils::{ - UtcTime, b64encode, check_bucket_name, to_amz_date, to_iso8601utc, to_signer_date, utc_now, + UtcTime, b64_encode, check_bucket_name, to_amz_date, to_iso8601utc, to_signer_date, utc_now, }; use serde_json::{Value, json}; use std::collections::HashMap; @@ -341,7 +341,7 @@ impl PostPolicy { "conditions": conditions, }); - let encoded_policy = b64encode(policy.to_string()); + let encoded_policy = b64_encode(policy.to_string()); let signature = post_presign_v4(&encoded_policy, &secret_key, date, ®ion); let mut data: HashMap = HashMap::new(); diff --git a/src/s3/builders/get_region.rs b/src/s3/builders/get_region.rs index 75634fa7..7932ec35 100644 --- a/src/s3/builders/get_region.rs +++ b/src/s3/builders/get_region.rs @@ -16,7 +16,7 @@ use crate::s3::Client; use crate::s3::client::DEFAULT_REGION; use crate::s3::error::ValidationErr; -use crate::s3::multimap::Multimap; +use crate::s3::multimap_ext::Multimap; use crate::s3::response::GetRegionResponse; use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::utils::{check_bucket_name, insert}; diff --git a/src/s3/builders/list_buckets.rs b/src/s3/builders/list_buckets.rs index c80575e2..a00a4b0f 100644 --- a/src/s3/builders/list_buckets.rs +++ b/src/s3/builders/list_buckets.rs @@ -15,7 +15,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::Multimap; +use crate::s3::multimap_ext::Multimap; use crate::s3::response::ListBucketsResponse; use crate::s3::types::{S3Api, S3Request, ToS3Request}; use http::Method; diff --git a/src/s3/builders/list_objects.rs b/src/s3/builders/list_objects.rs index bf7ec09b..a3e08e26 100644 --- a/src/s3/builders/list_objects.rs +++ b/src/s3/builders/list_objects.rs @@ -14,7 +14,7 @@ use crate::s3::client::Client; use crate::s3::error::{Error, ValidationErr}; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::ListObjectsResponse; use crate::s3::response::list_objects::{ ListObjectVersionsResponse, ListObjectsV1Response, ListObjectsV2Response, diff --git a/src/s3/builders/listen_bucket_notification.rs b/src/s3/builders/listen_bucket_notification.rs index c4f4c2fb..50f21ece 100644 --- a/src/s3/builders/listen_bucket_notification.rs +++ b/src/s3/builders/listen_bucket_notification.rs @@ -15,7 +15,7 @@ use crate::s3::client::Client; use crate::s3::error::{Error, ValidationErr}; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::ListenBucketNotificationResponse; use crate::s3::types::{NotificationRecords, S3Api, S3Request, ToS3Request}; use crate::s3::utils::check_bucket_name; diff --git a/src/s3/builders/put_bucket_encryption.rs b/src/s3/builders/put_bucket_encryption.rs index 2d27bab9..2c37ad5a 100644 --- a/src/s3/builders/put_bucket_encryption.rs +++ b/src/s3/builders/put_bucket_encryption.rs @@ -15,7 +15,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::Multimap; +use crate::s3::multimap_ext::Multimap; use crate::s3::response::PutBucketEncryptionResponse; use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::types::{S3Api, S3Request, SseConfig, ToS3Request}; diff --git a/src/s3/builders/put_bucket_lifecycle.rs b/src/s3/builders/put_bucket_lifecycle.rs index 88875908..e9662871 100644 --- a/src/s3/builders/put_bucket_lifecycle.rs +++ b/src/s3/builders/put_bucket_lifecycle.rs @@ -17,7 +17,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; use crate::s3::header_constants::*; use crate::s3::lifecycle_config::LifecycleConfig; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::PutBucketLifecycleResponse; use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::utils::{check_bucket_name, insert, md5sum_hash}; diff --git a/src/s3/builders/put_bucket_notification.rs b/src/s3/builders/put_bucket_notification.rs index b017dcc6..cc5ffce3 100644 --- a/src/s3/builders/put_bucket_notification.rs +++ b/src/s3/builders/put_bucket_notification.rs @@ -15,7 +15,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::Multimap; +use crate::s3::multimap_ext::Multimap; use crate::s3::response::PutBucketNotificationResponse; use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::types::{NotificationConfig, S3Api, S3Request, ToS3Request}; diff --git a/src/s3/builders/put_bucket_policy.rs b/src/s3/builders/put_bucket_policy.rs index 888ef1ac..186fdd50 100644 --- a/src/s3/builders/put_bucket_policy.rs +++ b/src/s3/builders/put_bucket_policy.rs @@ -15,7 +15,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::Multimap; +use crate::s3::multimap_ext::Multimap; use crate::s3::response::PutBucketPolicyResponse; use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::types::{S3Api, S3Request, ToS3Request}; diff --git a/src/s3/builders/put_bucket_replication.rs b/src/s3/builders/put_bucket_replication.rs index 1f408cf4..40e2859c 100644 --- a/src/s3/builders/put_bucket_replication.rs +++ b/src/s3/builders/put_bucket_replication.rs @@ -15,7 +15,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::Multimap; +use crate::s3::multimap_ext::Multimap; use crate::s3::response::PutBucketReplicationResponse; use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::types::{ReplicationConfig, S3Api, S3Request, ToS3Request}; diff --git a/src/s3/builders/put_bucket_tagging.rs b/src/s3/builders/put_bucket_tagging.rs index 141d51c9..cc15f58a 100644 --- a/src/s3/builders/put_bucket_tagging.rs +++ b/src/s3/builders/put_bucket_tagging.rs @@ -15,7 +15,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::Multimap; +use crate::s3::multimap_ext::Multimap; use crate::s3::response::PutBucketTaggingResponse; use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::types::{S3Api, S3Request, ToS3Request}; diff --git a/src/s3/builders/put_bucket_versioning.rs b/src/s3/builders/put_bucket_versioning.rs index f67a533d..c5a711b4 100644 --- a/src/s3/builders/put_bucket_versioning.rs +++ b/src/s3/builders/put_bucket_versioning.rs @@ -15,7 +15,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::Multimap; +use crate::s3::multimap_ext::Multimap; use crate::s3::response::PutBucketVersioningResponse; use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::types::{S3Api, S3Request, ToS3Request}; diff --git a/src/s3/builders/put_object.rs b/src/s3/builders/put_object.rs index 648c05ba..9fff2323 100644 --- a/src/s3/builders/put_object.rs +++ b/src/s3/builders/put_object.rs @@ -18,7 +18,7 @@ use crate::s3::builders::{ContentStream, Size}; use crate::s3::client::Client; use crate::s3::error::{Error, IoError, ValidationErr}; use crate::s3::header_constants::*; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::a_response_traits::HasEtagFromHeaders; use crate::s3::response::{ AbortMultipartUploadResponse, CompleteMultipartUploadResponse, CreateMultipartUploadResponse, diff --git a/src/s3/builders/put_object_legal_hold.rs b/src/s3/builders/put_object_legal_hold.rs index b6108860..af50c40a 100644 --- a/src/s3/builders/put_object_legal_hold.rs +++ b/src/s3/builders/put_object_legal_hold.rs @@ -16,7 +16,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; use crate::s3::header_constants::*; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::PutObjectLegalHoldResponse; use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::utils::{check_bucket_name, check_object_name, insert, md5sum_hash}; diff --git a/src/s3/builders/put_object_lock_config.rs b/src/s3/builders/put_object_lock_config.rs index e7031879..e713ae29 100644 --- a/src/s3/builders/put_object_lock_config.rs +++ b/src/s3/builders/put_object_lock_config.rs @@ -15,7 +15,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::Multimap; +use crate::s3::multimap_ext::Multimap; use crate::s3::response::PutObjectLockConfigResponse; use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::types::{ObjectLockConfig, S3Api, S3Request, ToS3Request}; diff --git a/src/s3/builders/put_object_retention.rs b/src/s3/builders/put_object_retention.rs index 9da2e1be..e4fe8b6f 100644 --- a/src/s3/builders/put_object_retention.rs +++ b/src/s3/builders/put_object_retention.rs @@ -16,7 +16,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; use crate::s3::header_constants::*; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::PutObjectRetentionResponse; use crate::s3::types::{RetentionMode, S3Api, S3Request, ToS3Request}; use crate::s3::utils::{ diff --git a/src/s3/builders/put_object_tagging.rs b/src/s3/builders/put_object_tagging.rs index 9d806911..ad093e52 100644 --- a/src/s3/builders/put_object_tagging.rs +++ b/src/s3/builders/put_object_tagging.rs @@ -15,7 +15,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::PutObjectTaggingResponse; use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::types::{S3Api, S3Request, ToS3Request}; diff --git a/src/s3/builders/select_object_content.rs b/src/s3/builders/select_object_content.rs index 6b662770..ab2fac38 100644 --- a/src/s3/builders/select_object_content.rs +++ b/src/s3/builders/select_object_content.rs @@ -16,7 +16,7 @@ use crate::s3::Client; use crate::s3::error::ValidationErr; use crate::s3::header_constants::*; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::SelectObjectContentResponse; use crate::s3::sse::SseCustomerKey; use crate::s3::types::{S3Api, S3Request, SelectRequest, ToS3Request}; diff --git a/src/s3/builders/stat_object.rs b/src/s3/builders/stat_object.rs index 3759de4c..63e5b9d4 100644 --- a/src/s3/builders/stat_object.rs +++ b/src/s3/builders/stat_object.rs @@ -16,7 +16,7 @@ use crate::s3::client::Client; use crate::s3::error::ValidationErr; use crate::s3::header_constants::*; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::StatObjectResponse; use crate::s3::sse::{Sse, SseCustomerKey}; use crate::s3::types::{S3Api, S3Request, ToS3Request}; diff --git a/src/s3/client.rs b/src/s3/client.rs index 7a41ea36..29bde11d 100644 --- a/src/s3/client.rs +++ b/src/s3/client.rs @@ -15,32 +15,31 @@ //! S3 client to perform bucket and object operations +use bytes::Bytes; +use dashmap::DashMap; +use http::HeaderMap; +use hyper::http::Method; +use reqwest::{Body, Response}; use std::fs::File; use std::io::prelude::*; use std::mem; use std::path::{Path, PathBuf}; use std::sync::{Arc, OnceLock}; +use uuid::Uuid; use crate::s3::builders::{BucketExists, ComposeSource}; use crate::s3::creds::Provider; +use crate::s3::error::{Error, IoError, NetworkError, S3ServerError, ValidationErr}; use crate::s3::header_constants::*; use crate::s3::http::BaseUrl; use crate::s3::minio_error_response::{MinioErrorCode, MinioErrorResponse}; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::a_response_traits::{HasEtagFromHeaders, HasS3Fields}; use crate::s3::response::*; use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::signer::sign_v4_s3; use crate::s3::utils::{EMPTY_SHA256, check_ssec_with_log, sha256_hash_sb, to_amz_date, utc_now}; -use crate::s3::error::{Error, IoError, NetworkError, S3ServerError, ValidationErr}; -use bytes::Bytes; -use dashmap::DashMap; -use http::HeaderMap; -use hyper::http::Method; -use rand::Rng; -use reqwest::{Body, Response}; - mod append_object; mod bucket_exists; mod copy_object; @@ -276,12 +275,7 @@ impl Client { *val } else { // Create a random bucket name - let bucket_name: String = rand::thread_rng() - .sample_iter(&rand::distributions::Alphanumeric) - .take(20) - .map(char::from) - .collect::() - .to_lowercase(); + let bucket_name: String = Uuid::new_v4().to_string(); let express = match BucketExists::new(self.clone(), bucket_name).send().await { Ok(v) => { diff --git a/src/s3/client/delete_bucket.rs b/src/s3/client/delete_bucket.rs index 8135524c..7f09a7cb 100644 --- a/src/s3/client/delete_bucket.rs +++ b/src/s3/client/delete_bucket.rs @@ -24,7 +24,7 @@ use crate::s3::response::{ }; use crate::s3::types::{S3Api, ToStream}; use bytes::Bytes; -use futures::StreamExt; +use futures_util::StreamExt; impl Client { /// Creates a [`DeleteBucket`] request builder. diff --git a/src/s3/http.rs b/src/s3/http.rs index 0a9ea45e..75c2b740 100644 --- a/src/s3/http.rs +++ b/src/s3/http.rs @@ -19,9 +19,8 @@ use super::utils::urlencode_object_key; use crate::s3::client::DEFAULT_REGION; use crate::s3::error::ValidationErr; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::utils::match_hostname; -use derivative::Derivative; use hyper::Uri; use hyper::http::Method; use lazy_static::lazy_static; @@ -37,11 +36,9 @@ lazy_static! { static ref AWS_S3_PREFIX_REGEX: Regex = Regex::new(AWS_S3_PREFIX).unwrap(); } -#[derive(Derivative)] -#[derivative(Clone, Debug, Default)] +#[derive(Clone, Debug)] /// Represents HTTP URL pub struct Url { - #[derivative(Default(value = "true"))] pub https: bool, pub host: String, pub port: u16, @@ -58,6 +55,18 @@ impl Url { } } +impl Default for Url { + fn default() -> Self { + Self { + https: true, + host: String::default(), + port: u16::default(), + path: String::default(), + query: Multimap::default(), + } + } +} + impl fmt::Display for Url { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { if self.host.is_empty() { @@ -208,11 +217,9 @@ fn get_aws_info( Ok(()) } -#[derive(Derivative)] -#[derivative(Clone, Debug, Default)] +#[derive(Clone, Debug)] /// Represents Base URL of S3 endpoint pub struct BaseUrl { - #[derivative(Default(value = "true"))] pub https: bool, host: String, port: u16, @@ -223,6 +230,21 @@ pub struct BaseUrl { pub virtual_style: bool, } +impl Default for BaseUrl { + fn default() -> Self { + Self { + https: true, + host: "127.0.0.1".to_string(), + port: 9000, + region: "".to_string(), + aws_s3_prefix: "".to_string(), + aws_domain_suffix: "".to_string(), + dualstack: false, + virtual_style: false, + } + } +} + impl FromStr for BaseUrl { type Err = ValidationErr; diff --git a/src/s3/mod.rs b/src/s3/mod.rs index 6d347a76..91f99664 100644 --- a/src/s3/mod.rs +++ b/src/s3/mod.rs @@ -23,7 +23,7 @@ pub mod header_constants; pub mod http; pub mod lifecycle_config; pub mod minio_error_response; -pub mod multimap; +pub mod multimap_ext; mod object_content; pub mod response; pub mod segmented_bytes; diff --git a/src/s3/multimap.rs b/src/s3/multimap_ext.rs similarity index 98% rename from src/s3/multimap.rs rename to src/s3/multimap_ext.rs index 88709b36..ce0f8c87 100644 --- a/src/s3/multimap.rs +++ b/src/s3/multimap_ext.rs @@ -49,9 +49,7 @@ impl MultimapExt for Multimap { } fn add_multimap(&mut self, other: Multimap) { for (key, values) in other.into_iter() { - for value in values { - self.insert(key.clone(), value); - } + self.insert_many(key.clone(), values); } } fn add_version(&mut self, version: Option) { diff --git a/src/s3/object_content.rs b/src/s3/object_content.rs index 2ac86ec6..0135e7a8 100644 --- a/src/s3/object_content.rs +++ b/src/s3/object_content.rs @@ -13,14 +13,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::s3::segmented_bytes::SegmentedBytes; use async_std::io::{ReadExt, WriteExt}; use bytes::Bytes; -use futures::stream::{self, Stream, StreamExt}; -use rand::prelude::random; +use futures_util::stream::{self, Stream, StreamExt}; use std::path::PathBuf; -use std::{ffi::OsString, fs, path::Path, pin::Pin}; +use std::{fs, path::Path, pin::Pin}; +use uuid::Uuid; -use crate::s3::segmented_bytes::SegmentedBytes; #[cfg(test)] use quickcheck::Arbitrary; @@ -216,13 +216,11 @@ impl ObjectContent { async_std::fs::create_dir_all(parent_dir).await?; } let file_name = file_path.file_name().ok_or(std::io::Error::other( - "could not get filename component of path", + "could not get filename-component of path", ))?; - let mut tmp_file_name: OsString = file_name.to_os_string(); - tmp_file_name.push(format!("_{}", random::())); - let tmp_file_path = parent_dir - .to_path_buf() - .join(Path::new(tmp_file_name.as_os_str())); + let mut tmp_file_name = file_name.to_os_string(); + tmp_file_name.push(format!("_{}", Uuid::new_v4().to_string().replace('-', "_"))); + let tmp_file_path = parent_dir.join(tmp_file_name); let mut total_bytes_written = 0; let mut fp = async_std::fs::OpenOptions::new() diff --git a/src/s3/response/select_object_content.rs b/src/s3/response/select_object_content.rs index 81aa88a9..956272d7 100644 --- a/src/s3/response/select_object_content.rs +++ b/src/s3/response/select_object_content.rs @@ -15,7 +15,7 @@ use crate::impl_has_s3fields; use crate::s3::error::{Error, ValidationErr}; -use crate::s3::multimap::{Multimap, MultimapExt}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; use crate::s3::response::a_response_traits::{HasBucket, HasObject, HasRegion, HasS3Fields}; use crate::s3::types::{FromS3Response, S3Request, SelectProgress}; use crate::s3::utils::{copy_slice, crc32, get_text_result, uint32}; diff --git a/src/s3/signer.rs b/src/s3/signer.rs index 6226d113..8a5d1c6d 100644 --- a/src/s3/signer.rs +++ b/src/s3/signer.rs @@ -16,9 +16,8 @@ //! Signature V4 for S3 API use crate::s3::header_constants::*; -use crate::s3::multimap::{Multimap, MultimapExt}; -use crate::s3::utils::{UtcTime, sha256_hash, to_amz_date, to_signer_date}; -use hex::encode as hexencode; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; +use crate::s3::utils::{UtcTime, hex_encode, sha256_hash, to_amz_date, to_signer_date}; #[cfg(not(feature = "ring"))] use hmac::{Hmac, Mac}; use hyper::http::Method; @@ -45,16 +44,14 @@ fn hmac_hash(key: &[u8], data: &[u8]) -> Vec { /// Returns hex encoded HMAC hash for given key and data fn hmac_hash_hex(key: &[u8], data: &[u8]) -> String { - hexencode(hmac_hash(key, data)) + hex_encode(hmac_hash(key, data).as_slice()) } /// Returns scope value of given date, region and service name fn get_scope(date: UtcTime, region: &str, service_name: &str) -> String { format!( - "{}/{}/{}/aws4_request", - to_signer_date(date), - region, - service_name + "{}/{region}/{service_name}/aws4_request", + to_signer_date(date) ) } @@ -76,10 +73,8 @@ fn get_canonical_request_hash( /// Returns string-to-sign value of given date, scope and canonical request hash fn get_string_to_sign(date: UtcTime, scope: &str, canonical_request_hash: &str) -> String { format!( - "AWS4-HMAC-SHA256\n{}\n{}\n{}", - to_amz_date(date), - scope, - canonical_request_hash + "AWS4-HMAC-SHA256\n{}\n{scope}\n{canonical_request_hash}", + to_amz_date(date) ) } diff --git a/src/s3/sse.rs b/src/s3/sse.rs index 95a86521..7f0209bc 100644 --- a/src/s3/sse.rs +++ b/src/s3/sse.rs @@ -16,8 +16,8 @@ //! Server side encryption definitions use crate::s3::header_constants::*; -use crate::s3::multimap::{Multimap, MultimapExt}; -use crate::s3::utils::{b64encode, md5sum_hash}; +use crate::s3::multimap_ext::{Multimap, MultimapExt}; +use crate::s3::utils::{b64_encode, md5sum_hash}; use std::any::Any; /// Base server side encryption @@ -39,7 +39,7 @@ pub struct SseCustomerKey { impl SseCustomerKey { pub fn new(key: &str) -> Self { - let b64key: String = b64encode(key); + let b64key: String = b64_encode(key); let md5key: String = md5sum_hash(key.as_bytes()); let mut headers = Multimap::with_capacity(3); @@ -102,7 +102,7 @@ impl SseKms { headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, key); headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION, "aws:kms"); if let Some(v) = context { - headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, b64encode(v)); + headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, b64_encode(v)); } SseKms { headers } diff --git a/src/s3/types.rs b/src/s3/types.rs index f384b978..e3e65450 100644 --- a/src/s3/types.rs +++ b/src/s3/types.rs @@ -18,7 +18,7 @@ use super::client::{Client, DEFAULT_REGION}; use crate::s3::error::{Error, ValidationErr}; use crate::s3::header_constants::*; -use crate::s3::multimap::Multimap; +use crate::s3::multimap_ext::Multimap; use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::utils::{UtcTime, get_text_option, get_text_result}; use async_trait::async_trait; diff --git a/src/s3/utils.rs b/src/s3/utils.rs index f8793fa2..f9763ed4 100644 --- a/src/s3/utils.rs +++ b/src/s3/utils.rs @@ -13,18 +13,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Various utility and helper functions - -use crate::s3::multimap::Multimap; +use crate::s3::Client; +use crate::s3::error::ValidationErr; +use crate::s3::multimap_ext::Multimap; use crate::s3::segmented_bytes::SegmentedBytes; +use crate::s3::sse::{Sse, SseCustomerKey}; use base64::engine::Engine as _; -use base64::engine::general_purpose::STANDARD as BASE64; -use byteorder::{BigEndian, ReadBytesExt}; use chrono::{DateTime, Datelike, NaiveDateTime, Utc}; use crc::{CRC_32_ISO_HDLC, Crc}; -use hex::ToHex; use lazy_static::lazy_static; -use md5::compute as md5compute; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode}; use regex::Regex; #[cfg(feature = "ring")] @@ -34,12 +31,9 @@ use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::sync::Arc; use xmltree::Element; + /// Date and time with UTC timezone pub type UtcTime = DateTime; -use crate::s3::Client; -use crate::s3::error::ValidationErr; -use crate::s3::sse::{Sse, SseCustomerKey}; -use url::form_urlencoded; // Great stuff to get confused about. // String "a b+c" in Percent-Encoding (RFC 3986) becomes "a%20b%2Bc". @@ -50,7 +44,7 @@ use url::form_urlencoded; /// Decodes a URL-encoded string in the application/x-www-form-urlencoded syntax into a string. /// Note that "+" is decoded to a space character, and "%2B" is decoded to a plus sign. pub fn url_decode(s: &str) -> String { - form_urlencoded::parse(s.as_bytes()) + url::form_urlencoded::parse(s.as_bytes()) .map(|(k, _)| k) .collect() } @@ -62,8 +56,8 @@ pub fn url_encode(s: &str) -> String { } /// Encodes data using base64 algorithm -pub fn b64encode(input: impl AsRef<[u8]>) -> String { - BASE64.encode(input) +pub fn b64_encode(input: impl AsRef<[u8]>) -> String { + base64::engine::general_purpose::STANDARD.encode(input) } /// Computes CRC32 of given data. @@ -73,12 +67,17 @@ pub fn crc32(data: &[u8]) -> u32 { } /// Converts data array into 32 bit BigEndian unsigned int -pub fn uint32(mut data: &[u8]) -> Result { - data.read_u32::() - .map_err(|e| ValidationErr::InvalidIntegerValue { +pub fn uint32(data: &[u8]) -> Result { + if data.len() < 4 { + return Err(ValidationErr::InvalidIntegerValue { message: "data is not a valid 32-bit BigEndian unsigned integer".into(), - source: Box::new(e), - }) + source: Box::new(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "not enough bytes", + )), + }); + } + Ok(u32::from_be_bytes(data[..4].try_into().unwrap())) } /// sha256 hash of empty data @@ -88,12 +87,66 @@ pub const EMPTY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934c pub fn sha256_hash(data: &[u8]) -> String { #[cfg(feature = "ring")] { - ring::digest::digest(&SHA256, data).encode_hex() + hex_encode(ring::digest::digest(&SHA256, data).as_ref()) } #[cfg(not(feature = "ring"))] { - Sha256::new_with_prefix(data).finalize().encode_hex() + hex_encode(Sha256::new_with_prefix(data).finalize().as_slice()) + } +} + +/// Hex-encode a byte slice into a lowercase ASCII string. +/// +/// # Safety +/// This implementation uses `unsafe` code for performance reasons: +/// - We call [`String::as_mut_vec`] to get direct access to the +/// underlying `Vec` backing the `String`. +/// - We then use [`set_len`] to pre-allocate the final length without +/// initializing the contents first. +/// - Finally, we use [`get_unchecked`] and [`get_unchecked_mut`] to +/// avoid bounds checking inside the tight encoding loop. +/// +/// # Why unsafe is needed +/// Normally, writing this function with safe Rust requires: +/// - Pushing each hex digit one-by-one into the string (extra bounds checks). +/// - Or allocating and copying temporary buffers. +/// +/// Using `unsafe` avoids redundant checks and makes this implementation +/// significantly faster, especially for large inputs. +/// +/// # Why this is correct +/// - `s` is allocated with exactly `len * 2` capacity, and we immediately +/// set its length to that value. Every byte in the string buffer will be +/// initialized before being read or used. +/// - The loop index `i` is always in `0..len`, so `bytes.get_unchecked(i)` +/// is safe. +/// - Each write goes to positions `j` and `j + 1`, where `j = i * 2`. +/// Since `i < len`, the maximum write index is `2*len - 1`, which is +/// within the allocated range. +/// - All written bytes come from the `LUT` table, which has exactly 16 +/// elements, and indices are masked into the 0–15 range. +/// +/// Therefore, although `unsafe` is used to skip bounds checking, +/// the logic ensures all memory accesses remain in-bounds and initialized. +pub fn hex_encode(bytes: &[u8]) -> String { + const LUT: &[u8; 16] = b"0123456789abcdef"; + let len = bytes.len(); + let mut s = String::with_capacity(len * 2); + + unsafe { + let v = s.as_mut_vec(); + v.set_len(len * 2); + for i in 0..len { + let b = bytes.get_unchecked(i); + let hi = LUT.get_unchecked((b >> 4) as usize); + let lo = LUT.get_unchecked((b & 0xF) as usize); + let j = i * 2; + *v.get_unchecked_mut(j) = *hi; + *v.get_unchecked_mut(j + 1) = *lo; + } } + + s } pub fn sha256_hash_sb(sb: Arc) -> String { @@ -101,19 +154,17 @@ pub fn sha256_hash_sb(sb: Arc) -> String { { let mut context = Context::new(&SHA256); for data in sb.iter() { - // Note: &SegmentedBytes.iter yields clones of Bytes, but those clones are cheap context.update(data.as_ref()); } - context.finish().encode_hex() + hex_encode(context.finish().as_ref()) } #[cfg(not(feature = "ring"))] { let mut hasher = Sha256::new(); for data in sb.iter() { - // Note: &SegmentedBytes.iter yields clones of Bytes, but those clones are cheap hasher.update(data); } - hasher.finalize().encode_hex() + hex_encode(hasher.finalize().as_slice()) } } @@ -134,7 +185,7 @@ mod tests { /// Gets bas64 encoded MD5 hash of given data pub fn md5sum_hash(data: &[u8]) -> String { - b64encode(md5compute(data).as_slice()) + b64_encode(md5::compute(data).as_slice()) } /// Gets current UTC time @@ -480,7 +531,7 @@ fn escape(s: &str) -> String { // // Handles escaping same as MinIO server - needed for ensuring compatibility. pub fn encode_tags(h: &HashMap) -> String { - let mut tags = Vec::new(); + let mut tags = Vec::with_capacity(h.len()); for (k, v) in h { tags.push(format!("{}={}", escape(k), escape(v))); } diff --git a/tests/run-tests-windows.ps1 b/tests/run-tests-windows.ps1 index ff865240..f1a01ae2 100644 --- a/tests/run-tests-windows.ps1 +++ b/tests/run-tests-windows.ps1 @@ -11,4 +11,8 @@ $Env:SERVER_REGION = "" cargo test -- --nocapture # run one specific test and show stdout -# cargo test --test test_bucket_exists -- --nocapture \ No newline at end of file +# cargo test --test test_bucket_exists -- --nocapture + +# run tests with ring instead of default-crypto +# cargo test --no-default-features --features "default-tls,ring" -- --nocapture + diff --git a/tests/test_upload_download_object.rs b/tests/test_upload_download_object.rs index ea6db331..9831d262 100644 --- a/tests/test_upload_download_object.rs +++ b/tests/test_upload_download_object.rs @@ -14,11 +14,11 @@ // limitations under the License. use async_std::io::ReadExt; -use hex::ToHex; use minio::s3::builders::ObjectContent; use minio::s3::response::a_response_traits::{HasBucket, HasObject}; use minio::s3::response::{GetObjectResponse, PutObjectContentResponse}; use minio::s3::types::S3Api; +use minio::s3::utils::hex_encode; use minio_common::rand_reader::RandReader; use minio_common::test_context::TestContext; use minio_common::utils::rand_object_name_utf8; @@ -36,7 +36,7 @@ async fn get_hash(filename: &str) -> String { let mut buf = Vec::new(); file.read_to_end(&mut buf).await.unwrap(); context.update(&buf); - context.finish().encode_hex() + hex_encode(context.finish().as_ref()) } #[cfg(not(feature = "ring"))] { @@ -45,7 +45,7 @@ async fn get_hash(filename: &str) -> String { let mut buf = Vec::new(); file.read_to_end(&mut buf).await.unwrap(); hasher.update(&buf); - hasher.finalize().encode_hex() + hex_encode(hasher.finalize().as_slice()) } }