From 58e9ae7b396bd592060ba02a410302f47f7c6463 Mon Sep 17 00:00:00 2001 From: Henk-Jan Lebbink Date: Fri, 11 Jul 2025 11:08:22 +0200 Subject: [PATCH] bugfix: proper handing of whitespace char in url with Form-decoding instead of Percent-decoding --- common/src/cleanup_guard.rs | 24 +++++------ src/s3/builders/copy_object.rs | 14 +++---- src/s3/builders/put_object.rs | 8 ++-- src/s3/client/delete_bucket.rs | 13 +++++- src/s3/multimap.rs | 11 +++--- src/s3/response/list_objects.rs | 29 +++++++------- src/s3/utils.rs | 27 +++++++++++-- tests/test_bucket_create_delete.rs | 46 +++++++++++----------- tests/test_get_object.rs | 23 +++++++---- tests/test_list_objects.rs | 59 ++++++++++++++++++++++++---- tests/test_object_copy.rs | 39 +++++++++++++----- tests/test_object_delete.rs | 35 +++++++---------- tests/test_upload_download_object.rs | 40 +++++++++++++------ 13 files changed, 242 insertions(+), 126 deletions(-) diff --git a/common/src/cleanup_guard.rs b/common/src/cleanup_guard.rs index a705895d..93105e35 100644 --- a/common/src/cleanup_guard.rs +++ b/common/src/cleanup_guard.rs @@ -37,18 +37,18 @@ impl CleanupGuard { pub async fn cleanup(client: Client, bucket_name: &str) { tokio::select!( - _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { - eprintln!("Cleanup timeout after 60s while removing bucket {}", bucket_name); - }, - outcome = client.delete_and_purge_bucket(bucket_name) => { - match outcome { - Ok(_) => { - eprintln!("Bucket {} removed successfully", bucket_name); - } - Err(e) => { - eprintln!("Error removing bucket {}: {:?}", bucket_name, e); - } - } + _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { + eprintln!("Cleanup timeout after 60s while removing bucket {}", bucket_name); + }, + outcome = client.delete_and_purge_bucket(bucket_name) => { + match outcome { + Ok(_) => { + //eprintln!("Bucket {} removed successfully", bucket_name); } + Err(e) => { + eprintln!("Error removing bucket {}: {:?}", bucket_name, e); + } + } + } ); } diff --git a/src/s3/builders/copy_object.rs b/src/s3/builders/copy_object.rs index 60c6bdd6..8f8962b1 100644 --- a/src/s3/builders/copy_object.rs +++ b/src/s3/builders/copy_object.rs @@ -26,7 +26,7 @@ use crate::s3::response::{ use crate::s3::sse::{Sse, SseCustomerKey}; use crate::s3::types::{Directive, PartInfo, Retention, S3Api, S3Request, ToS3Request}; use crate::s3::utils::{ - UtcTime, check_bucket_name, check_object_name, to_http_header_value, to_iso8601utc, urlencode, + UtcTime, check_bucket_name, check_object_name, to_http_header_value, to_iso8601utc, url_encode, }; use async_recursion::async_recursion; use http::Method; @@ -254,9 +254,9 @@ impl ToS3Request for CopyObjectInternal { if !tagging.is_empty() { tagging.push('&'); } - tagging.push_str(&urlencode(key)); + tagging.push_str(&url_encode(key)); tagging.push('='); - tagging.push_str(&urlencode(value)); + tagging.push_str(&url_encode(value)); } if !tagging.is_empty() { headers.add("x-amz-tagging", tagging); @@ -285,7 +285,7 @@ impl ToS3Request for CopyObjectInternal { copy_source.push_str(&self.source.object); if let Some(v) = &self.source.version_id { copy_source.push_str("?versionId="); - copy_source.push_str(&urlencode(v)); + copy_source.push_str(&url_encode(v)); } headers.add("x-amz-copy-source", copy_source); @@ -1032,7 +1032,7 @@ impl ComposeSource { copy_source.push_str(&self.object); if let Some(v) = &self.version_id { copy_source.push_str("?versionId="); - copy_source.push_str(&urlencode(v)); + copy_source.push_str(&url_encode(v)); } headers.add("x-amz-copy-source", copy_source); @@ -1155,9 +1155,9 @@ fn into_headers_copy_object( if !tagging.is_empty() { tagging.push('&'); } - tagging.push_str(&urlencode(key)); + tagging.push_str(&url_encode(key)); tagging.push('='); - tagging.push_str(&urlencode(value)); + tagging.push_str(&url_encode(value)); } if !tagging.is_empty() { diff --git a/src/s3/builders/put_object.rs b/src/s3/builders/put_object.rs index 9a5b9b7b..e7a3b051 100644 --- a/src/s3/builders/put_object.rs +++ b/src/s3/builders/put_object.rs @@ -29,7 +29,7 @@ use crate::s3::{ }, sse::Sse, types::{PartInfo, Retention, S3Api, S3Request, ToS3Request}, - utils::{check_bucket_name, md5sum_hash, to_iso8601utc, urlencode}, + utils::{check_bucket_name, md5sum_hash, to_iso8601utc, url_encode}, }; use bytes::{Bytes, BytesMut}; use http::Method; @@ -201,7 +201,7 @@ impl ToS3Request for AbortMultipartUpload { let headers: Multimap = self.extra_headers.unwrap_or_default(); let mut query_params: Multimap = self.extra_query_params.unwrap_or_default(); - query_params.add("uploadId", urlencode(&self.upload_id).to_string()); + query_params.add("uploadId", url_encode(&self.upload_id).to_string()); Ok(S3Request::new(self.client, Method::DELETE) .region(self.region) @@ -885,9 +885,9 @@ fn into_headers_put_object( if !tagging.is_empty() { tagging.push('&'); } - tagging.push_str(&urlencode(key)); + tagging.push_str(&url_encode(key)); tagging.push('='); - tagging.push_str(&urlencode(value)); + tagging.push_str(&url_encode(value)); } if !tagging.is_empty() { diff --git a/src/s3/client/delete_bucket.rs b/src/s3/client/delete_bucket.rs index e627177d..a6b4e1dd 100644 --- a/src/s3/client/delete_bucket.rs +++ b/src/s3/client/delete_bucket.rs @@ -16,7 +16,7 @@ use super::Client; use crate::s3::builders::{DeleteBucket, DeleteObject, ObjectToDelete}; use crate::s3::error::{Error, ErrorCode}; -use crate::s3::response::DeleteResult; +use crate::s3::response::{BucketExistsResponse, DeleteResult}; use crate::s3::response::{ DeleteBucketResponse, DeleteObjectResponse, DeleteObjectsResponse, PutObjectLegalHoldResponse, }; @@ -57,6 +57,17 @@ impl Client { bucket: S, ) -> Result { let bucket: String = bucket.into(); + + let resp: BucketExistsResponse = self.bucket_exists(&bucket).send().await?; + if !resp.exists { + // if the bucket does not exist, we can return early + return Ok(DeleteBucketResponse { + request: Default::default(), //TODO consider how to handle this + body: Bytes::new(), + headers: Default::default(), + }); + } + let is_express = self.is_minio_express().await; let mut stream = self diff --git a/src/s3/multimap.rs b/src/s3/multimap.rs index 3dd0e501..7bb1f767 100644 --- a/src/s3/multimap.rs +++ b/src/s3/multimap.rs @@ -13,12 +13,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::s3::utils::urlencode; +use crate::s3::utils::url_encode; use lazy_static::lazy_static; use multimap::MultiMap; use regex::Regex; use std::collections::BTreeMap; -pub use urlencoding::decode as urldecode; /// Multimap for string key and string value pub type Multimap = MultiMap; @@ -71,9 +70,9 @@ impl MultimapExt for Multimap { if !query.is_empty() { query.push('&'); } - query.push_str(&urlencode(key)); + query.push_str(&url_encode(key)); query.push('='); - query.push_str(&urlencode(value)); + query.push_str(&url_encode(value)); } } query @@ -94,9 +93,9 @@ impl MultimapExt for Multimap { if !query.is_empty() { query.push('&'); } - query.push_str(&urlencode(key.as_str())); + query.push_str(&url_encode(key.as_str())); query.push('='); - query.push_str(&urlencode(value)); + query.push_str(&url_encode(value)); } } None => todo!(), // This never happens. diff --git a/src/s3/response/list_objects.rs b/src/s3/response/list_objects.rs index 78c59b56..28b55f9c 100644 --- a/src/s3/response/list_objects.rs +++ b/src/s3/response/list_objects.rs @@ -15,26 +15,26 @@ use crate::s3::error::Error; use crate::s3::response::a_response_traits::HasS3Fields; use crate::s3::types::{FromS3Response, ListEntry, S3Request}; use crate::s3::utils::xml::{Element, MergeXmlElements}; -use crate::s3::utils::{from_iso8601utc, parse_tags, urldecode}; +use crate::s3::utils::{from_iso8601utc, parse_tags, url_decode}; use async_trait::async_trait; use bytes::{Buf, Bytes}; use reqwest::header::HeaderMap; use std::collections::HashMap; use std::mem; -fn url_decode( +fn url_decode_w_enc( encoding_type: &Option, - prefix: Option, + s: Option, ) -> Result, Error> { if let Some(v) = encoding_type.as_ref() { if v == "url" { - if let Some(v) = prefix { - return Ok(Some(urldecode(&v)?.to_string())); + if let Some(raw) = s { + return Ok(Some(url_decode(&raw).to_string())); } } } - if let Some(v) = prefix.as_ref() { + if let Some(v) = s.as_ref() { return Ok(Some(v.to_string())); } @@ -56,7 +56,7 @@ fn parse_common_list_objects_response( Error, > { let encoding_type = root.get_child_text("EncodingType"); - let prefix = url_decode( + let prefix = url_decode_w_enc( &encoding_type, Some(root.get_child_text("Prefix").unwrap_or_default()), )?; @@ -90,7 +90,7 @@ fn parse_list_objects_contents( let merged = MergeXmlElements::new(&children1, &children2); for content in merged { let etype = encoding_type.as_ref().cloned(); - let key = url_decode(&etype, Some(content.get_child_text_or_error("Key")?))?.unwrap(); + let key = url_decode_w_enc(&etype, Some(content.get_child_text_or_error("Key")?))?.unwrap(); let last_modified = Some(from_iso8601utc( &content.get_child_text_or_error("LastModified")?, )?); @@ -156,7 +156,7 @@ fn parse_list_objects_common_prefixes( ) -> Result<(), Error> { for (_, common_prefix) in root.get_matching_children("CommonPrefixes") { contents.push(ListEntry { - name: url_decode( + name: url_decode_w_enc( encoding_type, Some(common_prefix.get_child_text_or_error("Prefix")?), )? @@ -214,8 +214,8 @@ impl FromS3Response for ListObjectsV1Response { let root = Element::from(&xmltree_root); let (name, encoding_type, prefix, delimiter, is_truncated, max_keys) = parse_common_list_objects_response(&root)?; - let marker = url_decode(&encoding_type, root.get_child_text("Marker"))?; - let mut next_marker = url_decode(&encoding_type, root.get_child_text("NextMarker"))?; + let marker = url_decode_w_enc(&encoding_type, root.get_child_text("Marker"))?; + let mut next_marker = url_decode_w_enc(&encoding_type, root.get_child_text("NextMarker"))?; let mut contents: Vec = Vec::new(); parse_list_objects_contents(&mut contents, &root, "Contents", &encoding_type, false)?; if is_truncated && next_marker.is_none() { @@ -281,7 +281,7 @@ impl FromS3Response for ListObjectsV2Response { .get_child_text("KeyCount") .map(|x| x.parse::()) .transpose()?; - let start_after = url_decode(&encoding_type, root.get_child_text("StartAfter"))?; + let start_after = url_decode_w_enc(&encoding_type, root.get_child_text("StartAfter"))?; let continuation_token = root.get_child_text("ContinuationToken"); let next_continuation_token = root.get_child_text("NextContinuationToken"); let mut contents: Vec = Vec::new(); @@ -344,8 +344,9 @@ impl FromS3Response for ListObjectVersionsResponse { let root = Element::from(&xmltree_root); let (name, encoding_type, prefix, delimiter, is_truncated, max_keys) = parse_common_list_objects_response(&root)?; - let key_marker = url_decode(&encoding_type, root.get_child_text("KeyMarker"))?; - let next_key_marker = url_decode(&encoding_type, root.get_child_text("NextKeyMarker"))?; + let key_marker = url_decode_w_enc(&encoding_type, root.get_child_text("KeyMarker"))?; + let next_key_marker = + url_decode_w_enc(&encoding_type, root.get_child_text("NextKeyMarker"))?; let version_id_marker = root.get_child_text("VersionIdMarker"); let next_version_id_marker = root.get_child_text("NextVersionIdMarker"); let mut contents: Vec = Vec::new(); diff --git a/src/s3/utils.rs b/src/s3/utils.rs index 186337fd..acac48a4 100644 --- a/src/s3/utils.rs +++ b/src/s3/utils.rs @@ -34,13 +34,33 @@ use ring::digest::{Context, SHA256}; use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::sync::Arc; -pub use urlencoding::decode as urldecode; -pub use urlencoding::encode as urlencode; use xmltree::Element; /// Date and time with UTC timezone pub type UtcTime = DateTime; +use url::form_urlencoded; + +// Great stuff to get confused about. +// String "a b+c" in Percent-Encoding (RFC 3986) becomes "a%20b%2Bc". +// S3 sometimes returns Form-Encoding (application/x-www-form-urlencoded) rendering string "a%20b%2Bc" into "a+b%2Bc" +// If you were to do Percent-Decoding on "a+b%2Bc" you would get "a+b+c", which is wrong. +// If you use Form-Decoding on "a+b%2Bc" you would get "a b+c", which is correct. + +/// Decodes a URL-encoded string in the application/x-www-form-urlencoded syntax into a string. +/// Note that "+" is decoded to a space character, and "%2B" is decoded to a plus sign. +pub fn url_decode(s: &str) -> String { + form_urlencoded::parse(s.as_bytes()) + .map(|(k, _)| k) + .collect() +} + +/// Encodes a string using URL encoding. Note that a whitespace is encoded as "%20" and plus +/// sign is encoded as "%2B". +pub fn url_encode(s: &str) -> String { + urlencoding::encode(s).into_owned() +} + /// Encodes data using base64 algorithm pub fn b64encode(input: impl AsRef<[u8]>) -> String { BASE64.encode(input) @@ -245,7 +265,7 @@ pub fn match_region(value: &str) -> bool { || value.ends_with('_') } -/// Validates given bucket name +/// Validates given bucket name. TODO S3Express has slightly different rules for bucket names pub fn check_bucket_name(bucket_name: impl AsRef, strict: bool) -> Result<(), Error> { let bucket_name: &str = bucket_name.as_ref().trim(); let bucket_name_len = bucket_name.len(); @@ -302,6 +322,7 @@ pub fn check_bucket_name(bucket_name: impl AsRef, strict: bool) -> Result<( Ok(()) } +/// Validates given object name. TODO S3Express has slightly different rules for object names pub fn check_object_name(object_name: impl AsRef) -> Result<(), Error> { let object_name: &str = object_name.as_ref(); let object_name_n_bytes = object_name.len(); diff --git a/tests/test_bucket_create_delete.rs b/tests/test_bucket_create_delete.rs index 341771f1..e18efc24 100644 --- a/tests/test_bucket_create_delete.rs +++ b/tests/test_bucket_create_delete.rs @@ -21,7 +21,7 @@ use minio::s3::response::{ }; use minio::s3::types::S3Api; use minio_common::test_context::TestContext; -use minio_common::utils::{rand_bucket_name, rand_object_name}; +use minio_common::utils::{rand_bucket_name, rand_object_name_utf8}; #[minio_macros::test(no_bucket)] async fn bucket_create(ctx: TestContext) { @@ -88,39 +88,39 @@ async fn bucket_delete(ctx: TestContext) { assert_eq!(resp.region(), ""); } -#[minio_macros::test(no_bucket)] -async fn bucket_delete_and_purge_1(ctx: TestContext) { - let bucket_name = rand_bucket_name(); - - // create a new bucket - let resp: CreateBucketResponse = ctx.client.create_bucket(&bucket_name).send().await.unwrap(); +async fn test_bucket_delete_and_purge(ctx: &TestContext, bucket_name: &str, object_name: &str) { + let resp: PutObjectContentResponse = ctx + .client + .put_object_content(bucket_name, object_name, "Hello, World!") + .send() + .await + .unwrap(); assert_eq!(resp.bucket(), bucket_name); - assert_eq!(resp.region(), DEFAULT_REGION); - - // add some objects to the bucket - for _ in 0..5 { - let object_name = rand_object_name(); - let resp: PutObjectContentResponse = ctx - .client - .put_object_content(&bucket_name, &object_name, "Hello, World!") - .send() - .await - .unwrap(); - assert_eq!(resp.bucket(), bucket_name); - assert_eq!(resp.object(), object_name); - } + assert_eq!(resp.object(), object_name); // try to remove the bucket without purging, this should fail because the bucket is not empty let resp: Result = - ctx.client.delete_bucket(&bucket_name).send().await; + ctx.client.delete_bucket(bucket_name).send().await; assert!(resp.is_err()); // try to remove the bucket with purging, this should succeed let resp: DeleteBucketResponse = ctx .client - .delete_and_purge_bucket(&bucket_name) + .delete_and_purge_bucket(bucket_name) .await .unwrap(); assert_eq!(resp.bucket(), bucket_name); } + +/// Test purging a bucket with an object that contains utf8 characters. +#[minio_macros::test] +async fn bucket_delete_and_purge_1(ctx: TestContext, bucket_name: String) { + test_bucket_delete_and_purge(&ctx, &bucket_name, &rand_object_name_utf8(20)).await; +} + +/// Test purging a bucket with an object that contains white space characters. +#[minio_macros::test] +async fn bucket_delete_and_purge_2(ctx: TestContext, bucket_name: String) { + test_bucket_delete_and_purge(&ctx, &bucket_name, "a b+c").await; +} diff --git a/tests/test_get_object.rs b/tests/test_get_object.rs index e1edac25..f970245b 100644 --- a/tests/test_get_object.rs +++ b/tests/test_get_object.rs @@ -18,16 +18,13 @@ use minio::s3::response::a_response_traits::{HasBucket, HasObject}; use minio::s3::response::{GetObjectResponse, PutObjectContentResponse}; use minio::s3::types::S3Api; use minio_common::test_context::TestContext; -use minio_common::utils::rand_object_name; - -#[minio_macros::test] -async fn get_object(ctx: TestContext, bucket_name: String) { - let object_name = rand_object_name(); +use minio_common::utils::rand_object_name_utf8; +async fn test_get_object(ctx: &TestContext, bucket_name: &str, object_name: &str) { let data: Bytes = Bytes::from("hello, world".to_string().into_bytes()); let resp: PutObjectContentResponse = ctx .client - .put_object_content(&bucket_name, &object_name, data.clone()) + .put_object_content(bucket_name, object_name, data.clone()) .send() .await .unwrap(); @@ -37,7 +34,7 @@ async fn get_object(ctx: TestContext, bucket_name: String) { let resp: GetObjectResponse = ctx .client - .get_object(&bucket_name, &object_name) + .get_object(bucket_name, object_name) .send() .await .unwrap(); @@ -54,3 +51,15 @@ async fn get_object(ctx: TestContext, bucket_name: String) { .to_bytes(); assert_eq!(got, data); } + +/// Test getting an object with a name that contains utf-8 characters. +#[minio_macros::test] +async fn get_object_1(ctx: TestContext, bucket_name: String) { + test_get_object(&ctx, &bucket_name, &rand_object_name_utf8(20)).await; +} + +/// Test getting an object with a name that contains white space characters. +#[minio_macros::test] +async fn get_object_2(ctx: TestContext, bucket_name: String) { + test_get_object(&ctx, &bucket_name, "a b+c").await; +} diff --git a/tests/test_list_objects.rs b/tests/test_list_objects.rs index 78be1a58..456834c0 100644 --- a/tests/test_list_objects.rs +++ b/tests/test_list_objects.rs @@ -14,14 +14,14 @@ // limitations under the License. use async_std::stream::StreamExt; -use minio::s3::response::PutObjectContentResponse; use minio::s3::response::a_response_traits::{HasBucket, HasObject}; +use minio::s3::response::{ListObjectsResponse, PutObjectContentResponse}; use minio::s3::types::ToStream; use minio_common::test_context::TestContext; -use minio_common::utils::rand_object_name; +use minio_common::utils::{rand_object_name, rand_object_name_utf8}; use std::collections::HashSet; -async fn list_objects( +async fn test_list_objects( use_api_v1: bool, include_versions: bool, express: bool, @@ -99,27 +99,70 @@ async fn list_objects( #[minio_macros::test(skip_if_express)] async fn list_objects_v1_no_versions(ctx: TestContext, bucket_name: String) { - list_objects(true, false, false, 5, 5, ctx, bucket_name).await; + test_list_objects(true, false, false, 5, 5, ctx, bucket_name).await; } #[minio_macros::test(skip_if_express)] async fn list_objects_v1_with_versions(ctx: TestContext, bucket_name: String) { - list_objects(true, true, false, 5, 5, ctx, bucket_name).await; + test_list_objects(true, true, false, 5, 5, ctx, bucket_name).await; } #[minio_macros::test(skip_if_express)] async fn list_objects_v2_no_versions(ctx: TestContext, bucket_name: String) { - list_objects(false, false, false, 5, 5, ctx, bucket_name).await; + test_list_objects(false, false, false, 5, 5, ctx, bucket_name).await; } #[minio_macros::test(skip_if_express)] async fn list_objects_v2_with_versions(ctx: TestContext, bucket_name: String) { - list_objects(false, true, false, 5, 5, ctx, bucket_name).await; + test_list_objects(false, true, false, 5, 5, ctx, bucket_name).await; } /// Test for S3-Express: List objects with S3-Express are only supported with V2 API, without /// versions, and yield results that need not be sorted. #[minio_macros::test(skip_if_not_express)] async fn list_objects_express(ctx: TestContext, bucket_name: String) { - list_objects(false, false, true, 5, 5, ctx, bucket_name).await; + test_list_objects(false, false, true, 5, 5, ctx, bucket_name).await; +} + +async fn test_list_one_object(ctx: &TestContext, bucket_name: &str, object_name: &str) { + let resp: PutObjectContentResponse = ctx + .client + .put_object_content(bucket_name, object_name, "Hello, World!") + .send() + .await + .unwrap(); + assert_eq!(resp.bucket(), bucket_name); + assert_eq!(resp.object(), object_name); + + let mut stream = ctx + .client + .list_objects(bucket_name) + .use_api_v1(false) // S3-Express does not support V1 API + .include_versions(false) // S3-Express does not support versions + .to_stream() + .await; + + let mut result: Vec = Vec::new(); + while let Some(items) = stream.next().await { + result.push(items.unwrap()); + } + + assert_eq!(result.len(), 1); + assert_eq!(result[0].contents[0].name, object_name); +} + +/// Test listing an object with a name that contains utf-8 characters. +#[minio_macros::test] +async fn list_object_1(ctx: TestContext, bucket_name: String) { + test_list_one_object(&ctx, &bucket_name, &rand_object_name_utf8(20)).await; +} + +/// Test getting an object with a name that contains white space characters. +/// +/// In percent-encoding, "a b+c" becomes "a%20b%2Bc", but some S3 implementations may do +/// form-encoding, yielding "a+b2Bc", which will result in "a+b+c" is percent-decoding is +/// used. This test checks that form-decoding is used to retrieve "a b+c". +#[minio_macros::test] +async fn list_object_2(ctx: TestContext, bucket_name: String) { + test_list_one_object(&ctx, &bucket_name, "a b+c").await; } diff --git a/tests/test_object_copy.rs b/tests/test_object_copy.rs index 8853a02b..05f161b7 100644 --- a/tests/test_object_copy.rs +++ b/tests/test_object_copy.rs @@ -19,19 +19,20 @@ use minio::s3::response::{CopyObjectResponse, PutObjectContentResponse, StatObje use minio::s3::types::S3Api; use minio_common::rand_src::RandSrc; use minio_common::test_context::TestContext; -use minio_common::utils::rand_object_name; - -#[minio_macros::test(skip_if_express)] -async fn copy_object(ctx: TestContext, bucket_name: String) { - let object_name_src: String = rand_object_name(); - let object_name_dst: String = rand_object_name(); +use minio_common::utils::rand_object_name_utf8; +async fn test_copy_object( + ctx: &TestContext, + bucket_name: &str, + object_name_src: &str, + object_name_dst: &str, +) { let size = 16_u64; let content = ObjectContent::new_from_stream(RandSrc::new(size), Some(size)); let resp: PutObjectContentResponse = ctx .client - .put_object_content(&bucket_name, &object_name_src, content) + .put_object_content(bucket_name, object_name_src, content) .send() .await .unwrap(); @@ -40,8 +41,8 @@ async fn copy_object(ctx: TestContext, bucket_name: String) { let resp: CopyObjectResponse = ctx .client - .copy_object(&bucket_name, &object_name_dst) - .source(CopySource::new(&bucket_name, &object_name_src).unwrap()) + .copy_object(bucket_name, object_name_dst) + .source(CopySource::new(bucket_name, object_name_src).unwrap()) .send() .await .unwrap(); @@ -50,10 +51,28 @@ async fn copy_object(ctx: TestContext, bucket_name: String) { let resp: StatObjectResponse = ctx .client - .stat_object(&bucket_name, &object_name_dst) + .stat_object(bucket_name, object_name_dst) .send() .await .unwrap(); assert_eq!(resp.size().unwrap(), size); assert_eq!(resp.bucket(), bucket_name); } + +/// Test copying an object with a name that contains utf8 characters. +#[minio_macros::test(skip_if_express)] +async fn copy_object_1(ctx: TestContext, bucket_name: String) { + test_copy_object( + &ctx, + &bucket_name, + &rand_object_name_utf8(20), + &rand_object_name_utf8(20), + ) + .await; +} + +/// Test copying an object with a name that contains white space characters. +#[minio_macros::test(skip_if_express)] +async fn copy_object_2(ctx: TestContext, bucket_name: String) { + test_copy_object(&ctx, &bucket_name, "a b+c", "a b+c2").await; +} diff --git a/tests/test_object_delete.rs b/tests/test_object_delete.rs index 5aadcf22..7269ddb0 100644 --- a/tests/test_object_delete.rs +++ b/tests/test_object_delete.rs @@ -23,7 +23,7 @@ use minio::s3::types::{S3Api, ToStream}; use minio_common::test_context::TestContext; use minio_common::utils::rand_object_name_utf8; -async fn create_object( +async fn create_object_helper( ctx: &TestContext, bucket_name: &str, object_name: &str, @@ -39,14 +39,12 @@ async fn create_object( resp } -#[minio_macros::test] -async fn delete_object(ctx: TestContext, bucket_name: String) { - let object_name = rand_object_name_utf8(20); - let _resp = create_object(&ctx, &bucket_name, &object_name).await; +async fn test_delete_object(ctx: &TestContext, bucket_name: &str, object_name: &str) { + let _resp = create_object_helper(ctx, bucket_name, object_name).await; let resp: DeleteObjectResponse = ctx .client - .delete_object(&bucket_name, &object_name) + .delete_object(bucket_name, object_name) .send() .await .unwrap(); @@ -54,19 +52,16 @@ async fn delete_object(ctx: TestContext, bucket_name: String) { assert_eq!(resp.bucket(), bucket_name); } +/// Test deleting an object with a name that contains utf-8 characters. #[minio_macros::test] -async fn delete_object_with_whitespace(ctx: TestContext, bucket_name: String) { - let object_name = format!(" {}", rand_object_name_utf8(20)); - let _resp = create_object(&ctx, &bucket_name, &object_name).await; - - let resp: DeleteObjectResponse = ctx - .client - .delete_object(&bucket_name, &object_name) - .send() - .await - .unwrap(); +async fn delete_object_1(ctx: TestContext, bucket_name: String) { + test_delete_object(&ctx, &bucket_name, &rand_object_name_utf8(20)).await; +} - assert_eq!(resp.bucket(), bucket_name); +/// Test deleting an object with a name that contains white space characters. +#[minio_macros::test] +async fn delete_object_2(ctx: TestContext, bucket_name: String) { + test_delete_object(&ctx, &bucket_name, "a b+c").await; } #[minio_macros::test] @@ -75,7 +70,7 @@ async fn delete_objects(ctx: TestContext, bucket_name: String) { let mut names: Vec = Vec::new(); for _ in 1..=OBJECT_COUNT { let object_name = rand_object_name_utf8(20); - let _resp = create_object(&ctx, &bucket_name, &object_name).await; + let _resp = create_object_helper(&ctx, &bucket_name, &object_name).await; names.push(object_name); } let del_items: Vec = names @@ -104,7 +99,7 @@ async fn delete_objects_streaming(ctx: TestContext, bucket_name: String) { let mut names: Vec = Vec::new(); for _ in 1..=OBJECT_COUNT { let object_name = rand_object_name_utf8(20); - let _resp = create_object(&ctx, &bucket_name, &object_name).await; + let _resp = create_object_helper(&ctx, &bucket_name, &object_name).await; names.push(object_name); } let del_items: Vec = names @@ -129,5 +124,5 @@ async fn delete_objects_streaming(ctx: TestContext, bucket_name: String) { assert!(obj.is_deleted()); } } - assert_eq!(del_count, 3); + assert_eq!(del_count, OBJECT_COUNT); } diff --git a/tests/test_upload_download_object.rs b/tests/test_upload_download_object.rs index e289fcb5..ea6db331 100644 --- a/tests/test_upload_download_object.rs +++ b/tests/test_upload_download_object.rs @@ -21,14 +21,14 @@ use minio::s3::response::{GetObjectResponse, PutObjectContentResponse}; use minio::s3::types::S3Api; use minio_common::rand_reader::RandReader; use minio_common::test_context::TestContext; -use minio_common::utils::rand_object_name; +use minio_common::utils::rand_object_name_utf8; #[cfg(feature = "ring")] use ring::digest::{Context, SHA256}; #[cfg(not(feature = "ring"))] use sha2::{Digest, Sha256}; use std::path::PathBuf; -async fn get_hash(filename: &String) -> String { +async fn get_hash(filename: &str) -> String { #[cfg(feature = "ring")] { let mut context = Context::new(&SHA256); @@ -49,8 +49,12 @@ async fn get_hash(filename: &String) -> String { } } -async fn upload_download_object(size: u64, ctx: TestContext, bucket_name: String) { - let object_name: String = rand_object_name(); +async fn test_upload_download_object( + ctx: &TestContext, + bucket_name: &str, + object_name: &str, + size: u64, +) { let mut file = async_std::fs::File::create(&object_name).await.unwrap(); async_std::io::copy(&mut RandReader::new(size), &mut file) @@ -59,11 +63,11 @@ async fn upload_download_object(size: u64, ctx: TestContext, bucket_name: String file.sync_all().await.unwrap(); - let obj: ObjectContent = PathBuf::from(&object_name).as_path().into(); + let obj: ObjectContent = PathBuf::from(object_name).as_path().into(); let resp: PutObjectContentResponse = ctx .client - .put_object_content(&bucket_name, &object_name, obj) + .put_object_content(bucket_name, object_name, obj) .send() .await .unwrap(); @@ -71,10 +75,10 @@ async fn upload_download_object(size: u64, ctx: TestContext, bucket_name: String assert_eq!(resp.object(), object_name); assert_eq!(resp.object_size(), size); - let filename: String = rand_object_name(); + let filename: String = rand_object_name_utf8(20); let resp: GetObjectResponse = ctx .client - .get_object(&bucket_name, &object_name) + .get_object(bucket_name, object_name) .send() .await .unwrap(); @@ -87,18 +91,32 @@ async fn upload_download_object(size: u64, ctx: TestContext, bucket_name: String .to_file(PathBuf::from(&filename).as_path()) .await .unwrap(); - assert_eq!(get_hash(&object_name).await, get_hash(&filename).await); + assert_eq!(get_hash(object_name).await, get_hash(&filename).await); async_std::fs::remove_file(&object_name).await.unwrap(); async_std::fs::remove_file(&filename).await.unwrap(); } +/// Test uploading and downloading an object with a size that fits in a single part #[minio_macros::test] async fn upload_download_object_1(ctx: TestContext, bucket_name: String) { - upload_download_object(16, ctx, bucket_name).await; + test_upload_download_object(&ctx, &bucket_name, &rand_object_name_utf8(20), 16).await; } +/// Test uploading and downloading an object with a name that contains white space characters. #[minio_macros::test] async fn upload_download_object_2(ctx: TestContext, bucket_name: String) { - upload_download_object(16 + 5 * 1024 * 1024, ctx, bucket_name).await; + test_upload_download_object(&ctx, &bucket_name, "a b+c", 16).await; +} + +/// Test uploading and downloading an object with a size that needs multiple parts. +#[minio_macros::test] +async fn upload_download_object_3(ctx: TestContext, bucket_name: String) { + test_upload_download_object( + &ctx, + &bucket_name, + &rand_object_name_utf8(20), + 16 + 5 * 1024 * 1024, + ) + .await; }