diff --git a/Cargo.lock b/Cargo.lock index 82964573b..64ac49f6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3916,7 +3916,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "e2e_test" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "anyhow", "astral-tokio-tar", @@ -9386,7 +9386,7 @@ dependencies = [ [[package]] name = "rustfs" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "aes-gcm", "anyhow", @@ -9527,7 +9527,7 @@ dependencies = [ [[package]] name = "rustfs-audit" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "const-str", "futures", @@ -9549,7 +9549,7 @@ dependencies = [ [[package]] name = "rustfs-checksums" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "base64-simd", "bytes", @@ -9565,7 +9565,7 @@ dependencies = [ [[package]] name = "rustfs-common" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "hotpath", "metrics", @@ -9578,7 +9578,7 @@ dependencies = [ [[package]] name = "rustfs-concurrency" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "hotpath", "insta", @@ -9591,7 +9591,7 @@ dependencies = [ [[package]] name = "rustfs-config" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "const-str", "hotpath", @@ -9601,7 +9601,7 @@ dependencies = [ [[package]] name = "rustfs-credentials" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "base64-simd", "hmac 0.13.0", @@ -9615,7 +9615,7 @@ dependencies = [ [[package]] name = "rustfs-crypto" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "aes-gcm", "argon2", @@ -9636,7 +9636,7 @@ dependencies = [ [[package]] name = "rustfs-data-usage" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "hotpath", "rmp-serde", @@ -9646,7 +9646,7 @@ dependencies = [ [[package]] name = "rustfs-ecstore" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "arc-swap", "async-channel", @@ -9781,7 +9781,7 @@ dependencies = [ [[package]] name = "rustfs-extension-schema" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "hotpath", "serde", @@ -9791,7 +9791,7 @@ dependencies = [ [[package]] name = "rustfs-filemeta" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "arc-swap", "byteorder", @@ -9818,7 +9818,7 @@ dependencies = [ [[package]] name = "rustfs-heal" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "async-trait", "base64-simd", @@ -9854,7 +9854,7 @@ dependencies = [ [[package]] name = "rustfs-heal-contracts" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "serde", "serde_json", @@ -9864,7 +9864,7 @@ dependencies = [ [[package]] name = "rustfs-iam" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "arc-swap", "async-trait", @@ -9912,7 +9912,7 @@ dependencies = [ [[package]] name = "rustfs-io-core" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "bytes", "hotpath", @@ -9924,7 +9924,7 @@ dependencies = [ [[package]] name = "rustfs-io-metrics" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "criterion", "hotpath", @@ -9988,7 +9988,7 @@ dependencies = [ [[package]] name = "rustfs-keystone" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "bytes", "futures", @@ -10015,7 +10015,7 @@ dependencies = [ [[package]] name = "rustfs-kms" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "aes-gcm", "anyhow", @@ -10065,7 +10065,7 @@ dependencies = [ [[package]] name = "rustfs-lifecycle" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "async-trait", "hotpath", @@ -10088,7 +10088,7 @@ dependencies = [ [[package]] name = "rustfs-lock" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "async-trait", "compact_str", @@ -10111,7 +10111,7 @@ dependencies = [ [[package]] name = "rustfs-log-analyzer" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "chrono", "flate2", @@ -10130,7 +10130,7 @@ dependencies = [ [[package]] name = "rustfs-madmin" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "hotpath", "http 1.5.0", @@ -10168,7 +10168,7 @@ dependencies = [ [[package]] name = "rustfs-notify" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "arc-swap", "async-trait", @@ -10203,7 +10203,7 @@ dependencies = [ [[package]] name = "rustfs-object-capacity" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "criterion", "futures", @@ -10222,7 +10222,7 @@ dependencies = [ [[package]] name = "rustfs-object-data-cache" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "bytes", "criterion", @@ -10239,7 +10239,7 @@ dependencies = [ [[package]] name = "rustfs-obs" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "chrono", "crossbeam-channel", @@ -10297,7 +10297,7 @@ dependencies = [ [[package]] name = "rustfs-policy" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "async-trait", "base64-simd", @@ -10328,7 +10328,7 @@ dependencies = [ [[package]] name = "rustfs-protocols" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "astral-tokio-tar", "async-compression", @@ -10390,7 +10390,7 @@ dependencies = [ [[package]] name = "rustfs-protos" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "flatbuffers", "hotpath", @@ -10415,7 +10415,7 @@ dependencies = [ [[package]] name = "rustfs-replication" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "byteorder", "bytes", @@ -10433,7 +10433,7 @@ dependencies = [ [[package]] name = "rustfs-rio" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "aes-gcm", "arc-swap", @@ -10474,7 +10474,7 @@ dependencies = [ [[package]] name = "rustfs-rio-v2" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "aes-gcm", "bytes", @@ -10497,7 +10497,7 @@ dependencies = [ [[package]] name = "rustfs-s3-client" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "base64-simd", "bytes", @@ -10541,7 +10541,7 @@ dependencies = [ [[package]] name = "rustfs-s3-ops" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "hotpath", "rustfs-s3-types", @@ -10549,7 +10549,7 @@ dependencies = [ [[package]] name = "rustfs-s3-types" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "hotpath", "serde", @@ -10558,7 +10558,7 @@ dependencies = [ [[package]] name = "rustfs-s3select-api" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "arc-swap", "async-compression", @@ -10593,7 +10593,7 @@ dependencies = [ [[package]] name = "rustfs-s3select-query" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "async-recursion", "async-trait", @@ -10612,7 +10612,7 @@ dependencies = [ [[package]] name = "rustfs-scanner" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "async-trait", "bytes", @@ -10655,7 +10655,7 @@ dependencies = [ [[package]] name = "rustfs-scanner-contracts" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "chrono", "jiff", @@ -10670,7 +10670,7 @@ dependencies = [ [[package]] name = "rustfs-security-governance" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "hotpath", "thiserror 2.0.20", @@ -10678,7 +10678,7 @@ dependencies = [ [[package]] name = "rustfs-signer" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "base64-simd", "bytes", @@ -10696,7 +10696,7 @@ dependencies = [ [[package]] name = "rustfs-storage-api" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "async-trait", "hotpath", @@ -10711,7 +10711,7 @@ dependencies = [ [[package]] name = "rustfs-targets" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "arc-swap", "async-nats", @@ -10765,7 +10765,7 @@ dependencies = [ [[package]] name = "rustfs-test-utils" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "hotpath", "rustfs-data-usage", @@ -10781,7 +10781,7 @@ dependencies = [ [[package]] name = "rustfs-tls-runtime" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "arc-swap", "hotpath", @@ -10802,7 +10802,7 @@ dependencies = [ [[package]] name = "rustfs-trusted-proxies" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "async-trait", "axum", @@ -10839,7 +10839,7 @@ dependencies = [ [[package]] name = "rustfs-utils" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "base64-simd", "blake2", @@ -10881,7 +10881,7 @@ dependencies = [ [[package]] name = "rustfs-zip" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" dependencies = [ "async-compression", "hotpath", diff --git a/Cargo.toml b/Cargo.toml index 2c7f42ce8..3b3235a92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,7 @@ edition = "2024" license = "Apache-2.0" repository = "https://github.com/rustfs/rustfs" rust-version = "1.97.1" -version = "1.0.0-rc.4" +version = "1.0.0-rc.5" homepage = "https://rustfs.com" description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. " keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"] @@ -89,55 +89,55 @@ redundant_clone = "warn" [workspace.dependencies] # RustFS Internal Crates -rustfs = { path = "./rustfs", version = "1.0.0-rc.4" } -rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.4" } -rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.4" } -rustfs-scanner-contracts = { path = "crates/scanner-contracts", version = "1.0.0-rc.4" } -rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.4" } -rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.4" } -rustfs-common = { path = "crates/common", version = "1.0.0-rc.4" } -rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.4" } -rustfs-config = { path = "./crates/config", version = "1.0.0-rc.4" } -rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.4" } -rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.4" } -rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.4" } -rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.4" } -rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.4" } -rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.4" } -rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.4" } -rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.4" } -rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.4" } -rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.4" } -rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.4" } -rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.4" } -rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.4" } -rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.4" } -rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.4" } -rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.4", default-features = false } -rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.4" } -rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.4" } -rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.4" } -rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.4" } -rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.4" } -rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.4" } -rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.4" } -rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.4" } -rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.4" } -rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.4" } -rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.4" } -rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.4" } -rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.4" } -rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.4" } -rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.4" } -rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.4" } -rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.4" } -rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.4" } -rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.4" } -rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.4" } -rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.4" } -rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.4" } -rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.4" } -rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.4" } +rustfs = { path = "./rustfs", version = "1.0.0-rc.5" } +rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.5" } +rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.5" } +rustfs-scanner-contracts = { path = "crates/scanner-contracts", version = "1.0.0-rc.5" } +rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.5" } +rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.5" } +rustfs-common = { path = "crates/common", version = "1.0.0-rc.5" } +rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.5" } +rustfs-config = { path = "./crates/config", version = "1.0.0-rc.5" } +rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.5" } +rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.5" } +rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.5" } +rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.5" } +rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.5" } +rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.5" } +rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.5" } +rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.5" } +rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.5" } +rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.5" } +rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.5" } +rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.5" } +rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.5" } +rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.5" } +rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.5" } +rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.5", default-features = false } +rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.5" } +rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.5" } +rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.5" } +rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.5" } +rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.5" } +rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.5" } +rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.5" } +rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.5" } +rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.5" } +rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.5" } +rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.5" } +rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.5" } +rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.5" } +rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.5" } +rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.5" } +rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.5" } +rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.5" } +rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.5" } +rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.5" } +rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.5" } +rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.5" } +rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.5" } +rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.5" } +rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" } # Async Runtime and Networking async-channel = "2.5.0" diff --git a/README.md b/README.md index ffa8423c8..4b08457cc 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ chown -R 10001:10001 data logs docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest # Using specific version -docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.4 +docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5 ``` If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command diff --git a/README_ZH.md b/README_ZH.md index c97c13391..dcf6fcfcc 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -112,7 +112,7 @@ chown -R 10001:10001 data logs docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest # 使用指定版本运行 -docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.4 +docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5 ``` 如果您通过绑定挂载启用 TLS 证书目录，也请用同样方式准备该目录： diff --git a/crates/e2e_test/src/multipart_auth_test.rs b/crates/e2e_test/src/multipart_auth_test.rs index d2d45600a..d4606a8ff 100644 --- a/crates/e2e_test/src/multipart_auth_test.rs +++ b/crates/e2e_test/src/multipart_auth_test.rs @@ -4467,9 +4467,15 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent let context_archive_resources = [ format!("arn:aws:s3:::{bucket}/tag-context.tar"), format!("arn:aws:s3:::{bucket}/lock-context.tar"), + format!("arn:aws:s3:::{bucket}/legal-hold-context.tar"), + format!("arn:aws:s3:::{bucket}/user-agent-bypass.tar"), + format!("arn:aws:s3:::{bucket}/sse-bypass.tar"), ]; let tag_entry_resource = format!("arn:aws:s3:::{bucket}/tag-context-entry.txt"); let lock_entry_resource = format!("arn:aws:s3:::{bucket}/lock-context-entry.txt"); + let legal_hold_entry_resource = format!("arn:aws:s3:::{bucket}/legal-hold-context-entry.txt"); + let user_agent_entry_resource = format!("arn:aws:s3:::{bucket}/user-agent-bypass-entry.txt"); + let sse_entry_resource = format!("arn:aws:s3:::{bucket}/sse-bypass-entry.txt"); let policy = serde_json::json!({ "Version": "2012-10-17", "Statement": [ @@ -4529,7 +4535,7 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent "Sid": "PaxContextArchives", "Effect": "Allow", "Principal": { "AWS": [pax_context_user] }, - "Action": ["s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectTagging"], + "Action": ["s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectLegalHold", "s3:PutObjectTagging"], "Resource": context_archive_resources }, { @@ -4569,6 +4575,49 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent "Principal": { "AWS": [pax_context_user] }, "Action": ["s3:PutObjectRetention"], "Resource": [lock_entry_resource] + }, + { + "Sid": "PaxLegalHoldContextPut", + "Effect": "Allow", + "Principal": { "AWS": [pax_context_user] }, + "Action": ["s3:PutObject"], + "Resource": [legal_hold_entry_resource.clone()] + }, + { + "Sid": "PaxLegalHoldContextAction", + "Effect": "Allow", + "Principal": { "AWS": [pax_context_user] }, + "Action": ["s3:PutObjectLegalHold"], + "Resource": [legal_hold_entry_resource], + "Condition": { + "StringEquals": { + "s3:object-lock-legal-hold": "OFF" + } + } + }, + { + "Sid": "MemberUserAgentCondition", + "Effect": "Allow", + "Principal": { "AWS": [pax_context_user] }, + "Action": ["s3:PutObject"], + "Resource": [user_agent_entry_resource], + "Condition": { + "StringEquals": { + "aws:UserAgent": "trusted" + } + } + }, + { + "Sid": "MemberSseCondition", + "Effect": "Allow", + "Principal": { "AWS": [pax_context_user] }, + "Action": ["s3:PutObject"], + "Resource": [sse_entry_resource], + "Condition": { + "StringEquals": { + "s3:x-amz-server-side-encryption": "AES256" + } + } } ] }) @@ -4581,9 +4630,14 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent let cases = [ ( "legal-hold.tar", - put_only_client, + put_only_client.clone(), HashMap::from([("minio.metadata.x-amz-object-lock-legal-hold", "ON".to_string())]), ), + ( + "tagging.tar", + put_only_client, + HashMap::from([("minio.metadata.x-amz-tagging", "classification=restricted".to_string())]), + ), ( "retention-condition.tar", conditional_client, @@ -4670,6 +4724,57 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), b"condition-body"); let pax_context_client = restricted_user_client(&env, pax_context_user, pax_context_secret); + for (archive_key, entry_key, pax_key, injected_value, outer_user_agent) in [ + ( + "user-agent-bypass.tar", + "user-agent-bypass-entry.txt", + "minio.metadata.user-agent", + "trusted", + Some("untrusted"), + ), + ( + "sse-bypass.tar", + "sse-bypass-entry.txt", + "minio.metadata.x-amz-server-side-encryption", + "AES256", + None, + ), + ] { + let pax = HashMap::from([(pax_key, injected_value.to_string())]); + let archive = make_tar_with_pax_entry(entry_key, b"must-not-write", None, &pax).await; + let err = pax_context_client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(archive)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + if let Some(user_agent) = outer_user_agent { + req.headers_mut().insert("user-agent", user_agent); + } + }) + .send() + .await + .expect_err("PAX metadata must not satisfy unrelated IAM request conditions"); + assert_eq!( + err.as_service_error().and_then(|error| error.meta().code()), + Some("AccessDenied"), + "{archive_key}" + ); + let err = admin_client + .head_object() + .bucket(bucket) + .key(entry_key) + .send() + .await + .expect_err("a denied PAX member must not be written"); + assert!(matches!( + err.as_service_error().and_then(|error| error.meta().code()), + Some("NoSuchKey" | "NotFound") + )); + } + let tag_pax = HashMap::from([("minio.metadata.x-amz-tagging", "classification=public".to_string())]); let archive = make_tar_with_pax_entry("tag-context-entry.txt", b"tag-context-body", None, &tag_pax).await; pax_context_client @@ -4733,6 +4838,34 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent pax_retain_until ); + let legal_hold_pax = HashMap::from([("minio.metadata.x-amz-object-lock-legal-hold", "ON".to_string())]); + let archive = make_tar_with_pax_entry("legal-hold-context-entry.txt", b"must-not-write", None, &legal_hold_pax).await; + let err = pax_context_client + .put_object() + .bucket(bucket) + .key("legal-hold-context.tar") + .object_lock_legal_hold_status(aws_sdk_s3::types::ObjectLockLegalHoldStatus::Off) + .body(ByteStream::from(archive)) + .customize() + .mutate_request(|req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + }) + .send() + .await + .expect_err("PAX legal hold must replace the outer value in the member IAM condition context"); + assert_eq!(err.as_service_error().and_then(|error| error.meta().code()), Some("AccessDenied")); + let err = admin_client + .head_object() + .bucket(bucket) + .key("legal-hold-context-entry.txt") + .send() + .await + .expect_err("a denied PAX legal-hold member must not be written"); + assert!(matches!( + err.as_service_error().and_then(|error| error.meta().code()), + Some("NoSuchKey" | "NotFound") + )); + Ok(()) } diff --git a/crates/e2e_test/src/snowball_auto_extract_test.rs b/crates/e2e_test/src/snowball_auto_extract_test.rs index 5e705cc8b..4bc12156e 100644 --- a/crates/e2e_test/src/snowball_auto_extract_test.rs +++ b/crates/e2e_test/src/snowball_auto_extract_test.rs @@ -21,6 +21,53 @@ mod tests { use std::error::Error; use std::io::{Cursor, Write}; + fn pax_record(key: &str, value: &str) -> Vec { + let payload = format!("{key}={value}\n"); + let mut len = payload.len() + 3; + loop { + let record = format!("{len} {payload}"); + if record.len() == len { + return record.into_bytes(); + } + len = record.len(); + } + } + + async fn append_pax_header( + builder: &mut tokio_tar::Builder>>, + entry_type: tokio_tar::EntryType, + records: &[(&str, &str)], + ) -> Result<(), Box> { + let mut payload = Vec::new(); + for (key, value) in records { + payload.extend(pax_record(key, value)); + } + let mut header = tokio_tar::Header::new_ustar(); + header.set_entry_type(entry_type); + header.set_size(u64::try_from(payload.len()).expect("PAX payload length should fit in u64")); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "PaxHeaders.X/snowball", Cursor::new(payload)) + .await?; + Ok(()) + } + + async fn append_typed_entry( + builder: &mut tokio_tar::Builder>>, + path: &str, + entry_type: tokio_tar::EntryType, + body: &[u8], + ) -> Result<(), Box> { + let mut header = tokio_tar::Header::new_gnu(); + header.set_entry_type(entry_type); + header.set_size(u64::try_from(body.len()).expect("TAR member length should fit in u64")); + header.set_mode(0o644); + header.set_cksum(); + builder.append_data(&mut header, path, Cursor::new(body)).await?; + Ok(()) + } + async fn build_test_archive() -> Result, Box> { let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new())); @@ -147,6 +194,57 @@ mod tests { archive } + async fn build_member_semantics_archive() -> Result, Box> { + let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new())); + append_pax_header( + &mut builder, + tokio_tar::EntryType::XGlobalHeader, + &[ + ("minio.metadata.x-amz-meta-owner", "global"), + ("minio.metadata.x-amz-meta-snowball-auto-extract", "true"), + ], + ) + .await?; + append_pax_header( + &mut builder, + tokio_tar::EntryType::XHeader, + &[("minio.metadata.x-amz-meta-owner", "local")], + ) + .await?; + append_typed_entry(&mut builder, "regular.txt", tokio_tar::EntryType::Regular, b"regular-body").await?; + for (path, entry_type) in [ + ("char", tokio_tar::EntryType::Char), + ("block", tokio_tar::EntryType::Block), + ("fifo", tokio_tar::EntryType::Fifo), + ] { + append_typed_entry(&mut builder, path, entry_type, b"").await?; + } + let mut directory = tokio_tar::Header::new_gnu(); + directory.set_entry_type(tokio_tar::EntryType::Directory); + directory.set_size(0); + directory.set_mode(0o755); + directory.set_cksum(); + builder + .append_data(&mut directory, "directory/", Cursor::new(Vec::new())) + .await?; + for (path, entry_type) in [ + ("hard-link", tokio_tar::EntryType::Link), + ("symlink", tokio_tar::EntryType::Symlink), + ("continuous", tokio_tar::EntryType::Continuous), + ("unknown", tokio_tar::EntryType::Other(b'9')), + ] { + append_typed_entry(&mut builder, path, entry_type, b"").await?; + } + Ok(builder.into_inner().await?.into_inner()) + } + + async fn build_versioned_member_archive(path: &str, version_id: &str) -> Result, Box> { + let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new())); + append_pax_header(&mut builder, tokio_tar::EntryType::XHeader, &[("minio.versionId", version_id)]).await?; + append_typed_entry(&mut builder, path, tokio_tar::EntryType::Regular, b"versioned-body").await?; + Ok(builder.into_inner().await?.into_inner()) + } + fn build_archive_with_invalid_utf8_entry() -> Vec { let mut archive = Vec::new(); append_raw_tar_entry(&mut archive, b"invalid-\xff.txt", b"ignored-body"); @@ -199,6 +297,147 @@ mod tests { Ok(()) } + #[tokio::test] + async fn snowball_auto_extract_applies_member_semantics_and_metadata_precedence() -> Result<(), Box> + { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let client = env.create_s3_client(); + let bucket = "snowball-member-semantics"; + client.create_bucket().bucket(bucket).send().await?; + client + .put_object() + .bucket(bucket) + .key("fixture.tar") + .metadata("Snowball-Auto-Extract", "true") + .metadata("Minio-Snowball-Prefix", "members") + .metadata("owner", "outer") + .body(ByteStream::from(build_member_semantics_archive().await?)) + .send() + .await?; + + let regular = client.head_object().bucket(bucket).key("members/regular.txt").send().await?; + let regular_metadata = regular.metadata().expect("regular member should expose metadata"); + assert_eq!(regular_metadata.get("owner").map(String::as_str), Some("local")); + assert!(!regular_metadata.contains_key("snowball-auto-extract")); + assert!(!regular_metadata.contains_key("minio-snowball-prefix")); + + for key in ["char", "block", "fifo"] { + let head = client + .head_object() + .bucket(bucket) + .key(format!("members/{key}")) + .send() + .await?; + assert_eq!(head.content_length(), Some(0), "{key} should be materialized as an empty object"); + assert_eq!( + head.metadata().and_then(|metadata| metadata.get("owner")).map(String::as_str), + Some("outer"), + "{key} should not inherit global PAX metadata" + ); + } + let directory = client.head_object().bucket(bucket).key("members/directory/").send().await?; + assert_eq!(directory.content_length(), Some(0)); + + for key in ["hard-link", "symlink", "continuous", "unknown"] { + let error = client + .head_object() + .bucket(bucket) + .key(format!("members/{key}")) + .send() + .await + .expect_err("unsupported TAR entry type must be skipped"); + assert_eq!(error.into_service_error().code(), Some("NotFound"), "{key}"); + } + + env.stop_server(); + Ok(()) + } + + #[tokio::test] + async fn snowball_auto_extract_validates_pax_version_id_against_bucket_state() -> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let client = env.create_s3_client(); + let bucket = "snowball-version-semantics"; + client.create_bucket().bucket(bucket).send().await?; + + client + .put_object() + .bucket(bucket) + .key("null.tar") + .metadata("Snowball-Auto-Extract", "true") + .body(ByteStream::from(build_versioned_member_archive("null.txt", "null").await?)) + .send() + .await?; + let null_member = client.get_object().bucket(bucket).key("null.txt").send().await?; + assert_eq!(null_member.body.collect().await?.into_bytes().as_ref(), b"versioned-body"); + + for (archive_key, member_key, version_id) in [ + ("uuid.tar", "uuid.txt", uuid::Uuid::new_v4().to_string()), + ("uppercase-null.tar", "uppercase-null.txt", "NULL".to_string()), + ] { + let error = client + .put_object() + .bucket(bucket) + .key(archive_key) + .metadata("Snowball-Auto-Extract", "true") + .body(ByteStream::from(build_versioned_member_archive(member_key, &version_id).await?)) + .send() + .await + .expect_err("invalid or unversioned UUID import must be rejected"); + assert_eq!(error.into_service_error().code(), Some("InvalidArgument"), "{archive_key}"); + let missing = client + .head_object() + .bucket(bucket) + .key(member_key) + .send() + .await + .expect_err("rejected version import must not create an object"); + assert_eq!(missing.into_service_error().code(), Some("NotFound"), "{member_key}"); + } + + client + .put_bucket_versioning() + .bucket(bucket) + .versioning_configuration( + aws_sdk_s3::types::VersioningConfiguration::builder() + .status(aws_sdk_s3::types::BucketVersioningStatus::Enabled) + .build(), + ) + .send() + .await?; + let imported_version_id = uuid::Uuid::new_v4().to_string(); + client + .put_object() + .bucket(bucket) + .key("versioned-uuid.tar") + .metadata("Snowball-Auto-Extract", "true") + .body(ByteStream::from( + build_versioned_member_archive("versioned-uuid.txt", &imported_version_id).await?, + )) + .send() + .await?; + let imported = client + .get_object() + .bucket(bucket) + .key("versioned-uuid.txt") + .version_id(&imported_version_id) + .send() + .await?; + assert_eq!(imported.version_id(), Some(imported_version_id.as_str())); + assert_eq!(imported.body.collect().await?.into_bytes().as_ref(), b"versioned-body"); + + env.stop_server(); + Ok(()) + } + #[tokio::test] async fn snowball_auto_extract_supports_standard_headers_with_combined_extract_options() -> Result<(), Box> { diff --git a/crates/policy/src/policy/function/key_name.rs b/crates/policy/src/policy/function/key_name.rs index 5b99ecaa5..eeb054866 100644 --- a/crates/policy/src/policy/function/key_name.rs +++ b/crates/policy/src/policy/function/key_name.rs @@ -198,6 +198,9 @@ pub enum S3KeyName { #[strum(serialize = "s3:object-lock-retain-until-date")] S3ObjectLockRetainUntilDate, + #[strum(serialize = "s3:object-lock-legal-hold")] + S3ObjectLockLegalHold, + #[strum(serialize = "s3:object-lock-mode")] S3ObjectLockMode, @@ -389,6 +392,7 @@ mod tests { #[test_case("s3:VersionId", KeyName::S3(S3KeyName::S3VersionId) ; "aws_version_id")] #[test_case("s3:versionid", KeyName::S3(S3KeyName::S3VersionId) ; "minio_version_id")] #[test_case("s3:object-lock-mode", KeyName::S3(S3KeyName::S3ObjectLockMode))] + #[test_case("s3:object-lock-legal-hold", KeyName::S3(S3KeyName::S3ObjectLockLegalHold))] #[test_case("aws:SecureTransport", KeyName::Aws(AwsKeyName::AWSSecureTransport))] #[test_case("jwt:sub", KeyName::Jwt(JwtKeyName::JWTSub))] #[test_case("ldap:user", KeyName::Ldap(LdapKeyName::User))] @@ -412,6 +416,7 @@ mod tests { #[test_case("s3:VersionId", KeyName::S3(S3KeyName::S3VersionId) ; "aws_version_id")] #[test_case("s3:versionid", KeyName::S3(S3KeyName::S3VersionId) ; "minio_version_id")] #[test_case("s3:object-lock-mode", KeyName::S3(S3KeyName::S3ObjectLockMode))] + #[test_case("s3:object-lock-legal-hold", KeyName::S3(S3KeyName::S3ObjectLockLegalHold))] #[test_case("aws:SecureTransport", KeyName::Aws(AwsKeyName::AWSSecureTransport))] #[test_case("jwt:sub", KeyName::Jwt(JwtKeyName::JWTSub))] #[test_case("ldap:user", KeyName::Ldap(LdapKeyName::User))] @@ -431,6 +436,7 @@ mod tests { #[test_case("s3:x-amz-copy-source", KeyName::S3(S3KeyName::S3XAmzCopySource))] #[test_case("s3:versionid", KeyName::S3(S3KeyName::S3VersionId))] #[test_case("s3:object-lock-mode", KeyName::S3(S3KeyName::S3ObjectLockMode))] + #[test_case("s3:object-lock-legal-hold", KeyName::S3(S3KeyName::S3ObjectLockLegalHold))] #[test_case("aws:SecureTransport", KeyName::Aws(AwsKeyName::AWSSecureTransport))] #[test_case("jwt:sub", KeyName::Jwt(JwtKeyName::JWTSub))] #[test_case("ldap:user", KeyName::Ldap(LdapKeyName::User))] diff --git a/crates/policy/src/policy/function/string.rs b/crates/policy/src/policy/function/string.rs index cba2e6b3b..983f79d13 100644 --- a/crates/policy/src/policy/function/string.rs +++ b/crates/policy/src/policy/function/string.rs @@ -287,7 +287,7 @@ mod tests { }; use std::collections::HashMap; - use crate::policy::function::key_name::S3KeyName::{S3LocationConstraint, S3ObjectLockMode}; + use crate::policy::function::key_name::S3KeyName::{S3LocationConstraint, S3ObjectLockLegalHold, S3ObjectLockMode}; use test_case::test_case; fn new_func(name: KeyName, variable: Option, values: Vec<&str>) -> StringFunc { @@ -309,6 +309,7 @@ mod tests { #[test_case(r#"{"aws:username/value": ["johndoe", "aaa"]}"#, new_func(Aws(AWSUsername), Some("value".into()), vec!["johndoe", "aaa"] ))] #[test_case(r#"{"s3:object-lock-mode": "COMPLIANCE"}"#, new_func(S3(S3ObjectLockMode), None, vec!["COMPLIANCE"]))] + #[test_case(r#"{"s3:object-lock-legal-hold": "ON"}"#, new_func(S3(S3ObjectLockLegalHold), None, vec!["ON"]))] fn test_deser(input: &str, expect: StringFunc) -> Result<(), serde_json::Error> { let v: StringFunc = serde_json::from_str(input)?; assert_eq!(v, expect); diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 9d72f72f2..35229ba6b 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -131,6 +131,39 @@ pub(crate) async fn read_config_with_revision( } } +pub(crate) fn usage_floor_primary_read_error_allows_backup(err: &Error) -> bool { + match err { + Error::FileCorrupt + | Error::CorruptedFormat + | Error::CorruptedBackend + | Error::PartMissingOrCorrupt + | Error::LessData + | Error::MoreData => true, + Error::Io(io_error) => { + matches!(io_error.kind(), std::io::ErrorKind::InvalidData | std::io::ErrorKind::UnexpectedEof) + || error_chain_has_usage_floor_corruption_signature(io_error) + } + _ => false, + } +} + +fn error_chain_has_usage_floor_corruption_signature(error: &(dyn std::error::Error + 'static)) -> bool { + let mut current = Some(error); + while let Some(err) = current { + let message = err.to_string(); + if message.contains("InlineData value out of range") + || message.contains("InlineData key out of range") + || message.contains("insufficient data for metadata") + || message.contains("insufficient data for meta length") + || message.contains("insufficient data for CRC") + { + return true; + } + current = err.source(); + } + false +} + /// Read only the object revision without materializing its body. pub(crate) async fn read_config_revision(store: Arc, path: &str) -> StorageResult { match store diff --git a/crates/scanner/src/scanner/cycle_state.rs b/crates/scanner/src/scanner/cycle_state.rs index f8a11c6f3..99d2958d5 100644 --- a/crates/scanner/src/scanner/cycle_state.rs +++ b/crates/scanner/src/scanner/cycle_state.rs @@ -14,7 +14,9 @@ /// Scanner cycle-state codec, persisted usage floors, and cycle-state persistence. use super::*; use crate::ScannerGetObjectReader; -use crate::data_usage_define::{DATA_USAGE_BLOOM_RECOVERY_PATH, DATA_USAGE_RECOVERY_PATH}; +use crate::data_usage_define::{ + DATA_USAGE_BLOOM_RECOVERY_PATH, DATA_USAGE_RECOVERY_PATH, usage_floor_primary_read_error_allows_backup, +}; use crate::storage_api::owner::ObjectIO as _; use tokio::io::AsyncReadExt as _; @@ -1717,6 +1719,7 @@ pub(super) async fn persisted_usage_floor_for_startup( let backup_path = format!("{primary_path}.bkp"); let is_v2_path = primary_path == DATA_USAGE_OBJ_NAME_PATH.as_str(); let mut recovered_primary_companion_epoch = None; + let mut primary_read_error = None; let primary_epoch = match read_config_with_revision(storeapi.clone(), primary_path).await { Ok((Some(data), revision)) => { let usage = serde_json::from_slice::(&data).map_err(|err| { @@ -1776,6 +1779,12 @@ pub(super) async fn persisted_usage_floor_for_startup( } } Ok((None, _)) => None, + Err(err) if !is_v2_path && usage_floor_primary_read_error_allows_backup(&err) => { + primary_read_error = Some(format!("failed to read scanner usage epoch floor from {primary_path}: {err}")); + invalid_baseline_path.get_or_insert_with(|| primary_path.to_string()); + unrecoverable_baseline_path.get_or_insert_with(|| primary_path.to_string()); + None + } Err(err) => { return Err(ScannerError::Other(format!( "failed to read scanner usage epoch floor from {primary_path}: {err}" @@ -1855,6 +1864,14 @@ pub(super) async fn persisted_usage_floor_for_startup( ))); } } + if let Some(primary_read_error) = primary_read_error + && !any_found + { + return Err(ScannerError::Other(format!( + "{}; no valid scanner usage floor backup was available at {backup_path}", + primary_read_error + ))); + } if any_found { if bootstrap_pending { return Err(ScannerError::Other( diff --git a/crates/scanner/src/scanner/leadership.rs b/crates/scanner/src/scanner/leadership.rs index e728e7420..d273d133a 100644 --- a/crates/scanner/src/scanner/leadership.rs +++ b/crates/scanner/src/scanner/leadership.rs @@ -13,6 +13,7 @@ // limitations under the License. /// Leader-lock claiming, usage-epoch fencing, and lock-loss handling. use super::*; +use crate::data_usage_define::usage_floor_primary_read_error_allows_backup; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum ScannerLeadershipClaimReconcile { @@ -142,20 +143,34 @@ pub(super) async fn usage_snapshot_for_epoch_fence( } } - for path in [ - LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(), - format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), - ] { - let (legacy, _) = read_config_with_revision(storeapi.clone(), &path) - .await - .map_err(|err| ScannerError::Other(format!("failed to read legacy scanner usage epoch fence: {err}")))?; + let legacy_primary_path = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(); + let legacy_backup_path = format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()); + let mut legacy_primary_read_error = None; + for path in [&legacy_primary_path, &legacy_backup_path] { + let legacy = match read_config_with_revision(storeapi.clone(), path).await { + Ok((legacy, _)) => legacy, + Err(err) if path == &legacy_primary_path && usage_floor_primary_read_error_allows_backup(&err) => { + legacy_primary_read_error = Some(format!("failed to read legacy scanner usage epoch fence from {path}: {err}")); + continue; + } + Err(err) => { + return Err(ScannerError::Other(format!( + "failed to read legacy scanner usage epoch fence from {path}: {err}" + ))); + } + }; if let Some(legacy) = legacy.as_deref() { - let usage = decode_usage_snapshot_for_epoch_fence(legacy, &path, false)?; + let usage = decode_usage_snapshot_for_epoch_fence(legacy, path, false)?; if invalid_primary_epoch.is_none_or(|epoch| usage.scanner_epoch.unwrap_or_default() >= epoch) { return Ok(Some(usage)); } } } + if let Some(legacy_primary_read_error) = legacy_primary_read_error { + return Err(ScannerError::Other(format!( + "{legacy_primary_read_error}; no valid legacy scanner usage epoch fence backup was available at {legacy_backup_path}" + ))); + } // A missing usage snapshot is an uninitialized state, not an empty // snapshot. Leadership fencing may proceed without creating a plausible // default; the first authoritative scanner publication will create it. @@ -258,6 +273,12 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch( Some(epoch) if epoch == claimed_epoch => return Ok(()), Some(_) | None => {} } + // A validated pre-marker legacy baseline needs an explicit complete + // identity before acquiring an epoch. Otherwise the v2 reader would + // reject the fenced value on its next startup. + if !usage.usage_snapshot_bootstrap_pending { + usage.usage_snapshot_complete = true; + } usage.scanner_epoch = Some(claimed_epoch); let data = serde_json::to_vec(&usage) .map_err(|err| ScannerError::Other(format!("failed to encode scanner usage epoch fence: {err}")))?; diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index fcf815395..29a65a0e0 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -574,6 +574,7 @@ struct MemoryConfigStore { objects: Mutex>>, revisions: Mutex>, insert_after_gets: Mutex>>, + read_errors: Mutex>, delayed_gets: Mutex>, non_regular_objects: Mutex>, fail_put_number: Mutex>, @@ -623,6 +624,9 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore { _opts: &ObjectOptions, ) -> EcstoreResult { let key = memory_config_key(bucket, object); + if let Some(error) = self.read_errors.lock().await.get(&key).cloned() { + return Err(error); + } if let Some(delay) = self.delayed_gets.lock().await.remove(&key) { tokio::time::sleep(delay).await; } @@ -2902,6 +2906,147 @@ async fn scanner_usage_floor_recovers_from_incomplete_v2_primary_using_fenced_ba ); } +async fn seed_legacy_primary_read_error_with_backup(store: &Arc, error: EcstoreError, epoch: u64, cycle: u64) { + let legacy_primary = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(); + let legacy_backup = format!("{legacy_primary}.bkp"); + let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + backup.scanner_epoch = Some(epoch); + backup.scanner_cycle = Some(cycle); + + store + .read_errors + .lock() + .await + .insert(memory_config_key(RUSTFS_META_BUCKET, legacy_primary), error); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, &legacy_backup), + serde_json::to_vec(&backup).expect("legacy backup usage snapshot should encode"), + ); +} + +#[tokio::test] +async fn scanner_usage_floor_recovers_legacy_backup_after_primary_decode_error() { + let store = Arc::new(MemoryConfigStore::default()); + seed_legacy_primary_read_error_with_backup(&store, EcstoreError::other("InlineData value out of range"), 19, 41).await; + + let (floor, state) = persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect("valid legacy backup should recover the startup floor"); + assert_eq!(state, PersistedUsageFloorStartup::Authoritative); + assert_eq!( + floor, + PersistedUsageFloor { + next_cycle: 42, + leader_epoch: 19, + } + ); + assert_eq!( + persisted_usage_floor(store) + .await + .expect("valid legacy backup should recover the authoritative floor"), + floor + ); +} + +#[tokio::test] +async fn scanner_usage_floor_does_not_bootstrap_over_corrupt_legacy_primary_without_backup() { + let store = Arc::new(MemoryConfigStore::default()); + let legacy_primary = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(); + store + .read_errors + .lock() + .await + .insert(memory_config_key(RUSTFS_META_BUCKET, legacy_primary), EcstoreError::FileCorrupt); + + let err = persisted_usage_floor_for_startup(store, true) + .await + .expect_err("corrupt legacy primary without a valid backup must remain fail-closed"); + assert!(err.to_string().contains("no valid scanner usage floor backup"), "unexpected error: {err}"); +} + +#[tokio::test] +async fn scanner_usage_floor_does_not_fallback_to_legacy_after_corrupt_v2_primary() { + let store = Arc::new(MemoryConfigStore::default()); + let mut v2_backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + v2_backup.scanner_epoch = Some(8); + v2_backup.scanner_cycle = Some(11); + let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + legacy.scanner_epoch = Some(3); + legacy.scanner_cycle = Some(7); + store.read_errors.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), + EcstoreError::FileCorrupt, + ); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, &format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str())), + serde_json::to_vec(&v2_backup).expect("v2 backup usage snapshot should encode"), + ); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), + serde_json::to_vec(&legacy).expect("legacy usage snapshot should encode"), + ); + + let err = persisted_usage_floor_for_startup(store, true) + .await + .expect_err("corrupt v2 primary must not recover without a primary revision"); + assert!( + err.to_string().contains(&format!( + "failed to read scanner usage epoch floor from {}", + DATA_USAGE_OBJ_NAME_PATH.as_str() + )), + "unexpected error: {err}" + ); +} + +#[tokio::test] +async fn scanner_usage_floor_keeps_transient_primary_read_error_fail_closed() { + let store = Arc::new(MemoryConfigStore::default()); + let legacy_primary = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(); + seed_legacy_primary_read_error_with_backup( + &store, + EcstoreError::Io(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "connection reset while reading usage primary", + )), + 19, + 41, + ) + .await; + + let err = persisted_usage_floor_for_startup(store, true) + .await + .expect_err("transient primary errors must not be converted into backup recovery"); + assert!( + err.to_string() + .contains(&format!("failed to read scanner usage epoch floor from {legacy_primary}")), + "unexpected error: {err}" + ); + assert!( + !err.to_string().contains("no valid scanner usage floor backup"), + "transient error should not enter corrupt-primary fallback: {err}" + ); +} + +#[tokio::test] +async fn scanner_usage_floor_keeps_outdated_primary_metadata_fail_closed() { + let store = Arc::new(MemoryConfigStore::default()); + let legacy_primary = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(); + seed_legacy_primary_read_error_with_backup(&store, EcstoreError::OutdatedXLMeta, 19, 41).await; + + let err = persisted_usage_floor_for_startup(store, true) + .await + .expect_err("outdated primary metadata must not be converted into backup recovery"); + assert!( + err.to_string() + .contains(&format!("failed to read scanner usage epoch floor from {legacy_primary}")), + "unexpected error: {err}" + ); + assert!( + !err.to_string().contains("no valid scanner usage floor backup"), + "outdated metadata should not enter corrupt-primary fallback: {err}" + ); +} + #[tokio::test] async fn scanner_usage_floor_does_not_bootstrap_over_incomplete_v2_primary() { let store = Arc::new(MemoryConfigStore::default()); @@ -3004,6 +3149,19 @@ async fn scanner_leadership_fencing_recovers_incomplete_v2_primary_from_backup() assert_eq!(recovered.scanner_cycle, Some(103)); } +#[tokio::test] +async fn scanner_usage_floor_leadership_fencing_recovers_legacy_backup_after_primary_decode_error() { + let store = Arc::new(MemoryConfigStore::default()); + seed_legacy_primary_read_error_with_backup(&store, EcstoreError::other("InlineData value out of range"), 19, 41).await; + + let recovered = usage_snapshot_for_epoch_fence(store, None, false) + .await + .expect("a valid legacy backup should provide the fencing baseline") + .expect("the fencing baseline should be present"); + assert_eq!(recovered.scanner_epoch, Some(19)); + assert_eq!(recovered.scanner_cycle, Some(41)); +} + #[tokio::test] async fn scanner_usage_floor_ignores_older_backup_after_primary_epoch_fence() { let store = Arc::new(MemoryConfigStore::default()); @@ -3955,6 +4113,162 @@ async fn scanner_defers_leadership_when_usage_snapshots_are_stably_absent() { assert!(read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()).await.is_err()); } +#[tokio::test] +async fn scanner_usage_floor_leadership_claim_recovers_legacy_backup_after_primary_decode_error() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + seed_legacy_primary_read_error_with_backup(&store, EcstoreError::other("InlineData value out of range"), 19, 41).await; + + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle::default(); + let mut persisted_epoch = 19; + assert!( + claim_scanner_leadership( + &ctx, + store.clone(), + &mut cycle, + &mut revision, + &mut persisted_epoch, + false, + ScannerCycleResetPolicy::None, + ) + .await + ); + + let state = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("leadership claim should persist after legacy backup recovery"); + let (_, claimed_epoch) = decode_scanner_cycle_state(&state).expect("leadership claim should decode"); + assert_eq!(claimed_epoch, 20); + assert_eq!(persisted_epoch, 20); + + let usage = read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("legacy backup recovery should publish a fenced v2 usage primary"); + let usage = serde_json::from_slice::(&usage).expect("fenced v2 usage primary should decode"); + assert_eq!(usage.scanner_epoch, Some(20)); + assert_eq!(usage.scanner_cycle, Some(41)); +} + +#[tokio::test] +#[serial_test::serial] +async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_metadata_truncation() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let (temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await; + let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + usage.usage_snapshot_complete = false; + usage.scanner_cycle = Some(41); + let mut data = serde_json::to_vec(&usage).expect("legacy usage should encode"); + data.resize(data.len() + 16 * 1024, b' '); + let legacy_path = LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(); + let backup_path = format!("{legacy_path}.bkp"); + for path in [legacy_path, backup_path.as_str()] { + save_config(store.clone(), path, data.clone()) + .await + .expect("legacy usage fixture should persist"); + } + let mut truncated_files = Vec::new(); + for disk_index in 0..4 { + let path = temp_dir + .path() + .join(format!("pool0/disk{disk_index}")) + .join(RUSTFS_META_BUCKET) + .join(legacy_path) + .join("xl.meta"); + let file = tokio::fs::OpenOptions::new() + .write(true) + .open(&path) + .await + .expect("legacy inline metadata should exist"); + assert!(file.metadata().await.expect("metadata should be readable").len() > 4096); + file.set_len(4096).await.expect("fixture should truncate at a page boundary"); + truncated_files.push(( + path.clone(), + tokio::fs::read(&path) + .await + .expect("truncated evidence should remain readable"), + )); + } + + let store = restart_scanner_cycle_store_from(&store).await; + let error = read_config_with_revision(store.clone(), legacy_path) + .await + .expect_err("truncated primary must fail in the real object reader"); + assert!( + error.to_string().contains("InlineData value out of range"), + "unexpected truncated-primary error: {error}" + ); + assert_eq!( + read_config_with_revision(store.clone(), &backup_path) + .await + .expect("backup should remain readable") + .0, + Some(data.clone()), + ); + let (floor, state) = persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect("intact legacy backup must recover startup despite truncated primary"); + assert_eq!( + floor, + PersistedUsageFloor { + next_cycle: 42, + leader_epoch: 0 + } + ); + assert_eq!(state, PersistedUsageFloorStartup::Authoritative); + + let baseline = read_data_usage_persist_baseline(store.clone()) + .await + .expect("publication must also read the intact backup"); + assert_eq!(baseline.data.as_deref(), Some(data.as_slice())); + assert_eq!(baseline.revision, DataUsageCacheRevision::Missing); + fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false) + .await + .expect("legacy backup must be fenced into v2"); + let fenced = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("fencing must publish a v2 usage primary"); + let fenced = serde_json::from_slice::(&fenced).expect("fenced v2 usage primary should decode"); + assert!( + fenced.usage_snapshot_complete, + "the fenced pre-marker baseline must become a complete v2 identity" + ); + + let store = restart_scanner_cycle_store_from(&store).await; + let (floor, state) = persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect("a restart after fencing must preserve the recovered floor"); + assert_eq!( + floor, + PersistedUsageFloor { + next_cycle: 42, + leader_epoch: 7 + } + ); + assert_eq!(state, PersistedUsageFloorStartup::Authoritative); + let restarted = restart_scanner_cycle_store_from(&store).await; + assert_eq!( + persisted_usage_floor(restarted) + .await + .expect("fenced floor must survive another restart"), + PersistedUsageFloor { + next_cycle: 42, + leader_epoch: 7 + } + ); + for (path, bytes) in truncated_files { + assert_eq!(tokio::fs::read(path).await.expect("legacy evidence must not be removed"), bytes); + } + assert_eq!( + read_config(store, &backup_path) + .await + .expect("legacy backup must remain intact"), + data + ); + global_metrics().set_cycle(None).await; + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); +} + #[tokio::test] async fn usage_bootstrap_pending_unblocks_first_leadership_claim() { let store = Arc::new(MemoryConfigStore::default()); diff --git a/crates/scanner/src/scanner/usage_store.rs b/crates/scanner/src/scanner/usage_store.rs index b9dd53271..af7c616ee 100644 --- a/crates/scanner/src/scanner/usage_store.rs +++ b/crates/scanner/src/scanner/usage_store.rs @@ -13,6 +13,7 @@ // limitations under the License. /// Data-usage snapshot persistence: CAS store pipeline, epoch baselines, and observed-snapshot cleanup. use super::*; +use crate::data_usage_define::usage_floor_primary_read_error_allows_backup; use crate::storage_api::owner::ScannerPublicationCommitState; use std::collections::HashMap; use std::sync::atomic::AtomicBool; @@ -53,6 +54,30 @@ pub(super) struct DataUsagePersistBaseline { pub(super) revision: DataUsageCacheRevision, } +async fn read_usage_persist_candidate( + storeapi: Arc, + path: &str, +) -> Result<(Option>, DataUsageCacheRevision), EcstoreError> { + let primary = read_config_with_revision(storeapi.clone(), path).await; + if path != LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str() { + return primary; + } + let Err(primary_error) = &primary else { + return primary; + }; + if !usage_floor_primary_read_error_allows_backup(primary_error) { + return primary; + } + let backup_path = format!("{path}.bkp"); + let backup = read_config_with_revision(storeapi, &backup_path).await?; + if backup.0.as_deref().is_some_and(|data| { + serde_json::from_slice::(data).is_ok_and(|usage| data_usage_info_has_persisted_baseline_identity(&usage)) + }) { + return Ok(backup); + } + primary +} + /// Read the bytes used as the baseline for a usage publication while keeping /// the v2 primary revision as the CAS fence. During an interrupted upgrade the /// primary can be valid JSON without a baseline identity; in that case a @@ -68,7 +93,7 @@ pub(super) async fn read_data_usage_persist_baseline( LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(), format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), ] { - let (candidate, _) = read_config_with_revision(storeapi.clone(), &path).await?; + let (candidate, _) = read_usage_persist_candidate(storeapi.clone(), &path).await?; let Some(candidate) = candidate else { continue; }; @@ -107,7 +132,7 @@ pub(super) async fn read_data_usage_persist_baseline( LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(), format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), ] { - let (candidate, _) = read_config_with_revision(storeapi.clone(), &path).await?; + let (candidate, _) = read_usage_persist_candidate(storeapi.clone(), &path).await?; let Some(candidate) = candidate else { continue; }; diff --git a/flake.nix b/flake.nix index 6fbe3240a..719094479 100644 --- a/flake.nix +++ b/flake.nix @@ -77,7 +77,7 @@ rustfs = rustPlatform.buildRustPackage { pname = "rustfs"; - version = "1.0.0-rc.4"; + version = "1.0.0-rc.5"; src = ./.; diff --git a/helm/rustfs/Chart.yaml b/helm/rustfs/Chart.yaml index c7b43885a..fa017d7d3 100644 --- a/helm/rustfs/Chart.yaml +++ b/helm/rustfs/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: rustfs description: RustFS helm chart to deploy RustFS on kubernetes cluster. type: application -version: "1.0.0-rc.4" -appVersion: "1.0.0-rc.4" +version: "1.0.0-rc.5" +appVersion: "1.0.0-rc.5" home: https://rustfs.com icon: https://media.sys.truenas.net/apps/rustfs/icons/icon.svg maintainers: diff --git a/rustfs.spec b/rustfs.spec index 76aff52b7..78e7ad503 100644 --- a/rustfs.spec +++ b/rustfs.spec @@ -1,9 +1,9 @@ %global _enable_debug_packages 0 %global _empty_manifest_terminate_build 0 -%global prerelease rc.4 +%global prerelease rc.5 Name: rustfs Version: 1.0.0 -Release: rc.4 +Release: rc.5 Summary: High-performance distributed object storage for MinIO alternative License: Apache-2.0 @@ -58,6 +58,9 @@ install %_builddir/%{name}-%{version}-%{prerelease}/target/%_arch/%_arch-unknown %_bindir/rustfs %changelog +* Mon Aug 31 2026 overtrue +- Update RPM package to RustFS 1.0.0-rc.5 + * Thu Aug 27 2026 overtrue - Update RPM package to RustFS 1.0.0-rc.4 diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index c8ddd39db..bca762fb3 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -32,7 +32,9 @@ use crate::admin::storage_api::bucket::replication::{ }; use crate::admin::storage_api::bucket::target::{BucketTarget, BucketTargetType, BucketTargets}; use crate::admin::storage_api::bucket::utils::{deserialize, serialize}; -use crate::admin::storage_api::bucket::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _}; +use crate::admin::storage_api::bucket::{ + AdminObjectLockConfigExt as _, AdminReplicationConfigExt as _, AdminVersioningConfigExt as _, +}; use crate::admin::storage_api::contract::bucket::{ BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp, }; @@ -61,18 +63,18 @@ use rustfs_madmin::{ BucketBandwidth, GroupStatus, IDPSettings, InProgressMetric, InQueueMetric, LDAPConfigSettings, LDAPSettings, OpenIDProviderSettings, PeerInfo, PeerSite, QStat, ReplProxyMetric, ReplicateAddStatus, ReplicateEditStatus, ReplicateRemoveStatus, ResyncBucketStatus, SITE_REPL_API_VERSION, SR_IAM_ITEM_STS_ACC, SR_IAM_ITEM_STS_ACC_LEGACY, - SRBucketMeta, SRBucketStatsSummary, SRGroupInfo, SRGroupStatsSummary, SRIAMItem, SRIAMUser, SRILMExpiryStatsSummary, SRInfo, - SRMetric, SRMetricsSummary, SRPeerError, SRPeerJoinReq, SRPendingOperation, SRPolicyMapping, SRPolicyStatsSummary, - SRRemoveReq, SRResyncOpStatus, SRSTSCredential, SRSessionPolicy, SRSiteSummary, SRStateEditReq, SRStateInfo, SRStatusInfo, - SRSvcAccChange, SRSvcAccCreate, SRUserStatsSummary, SiteReplicationInfo, SyncStatus, WorkerStat, + SRBucketInfo, SRBucketMeta, SRBucketStatsSummary, SRGroupInfo, SRGroupStatsSummary, SRIAMItem, SRIAMUser, + SRILMExpiryStatsSummary, SRInfo, SRMetric, SRMetricsSummary, SRPeerError, SRPeerJoinReq, SRPendingOperation, SRPolicyMapping, + SRPolicyStatsSummary, SRRemoveReq, SRResyncOpStatus, SRSTSCredential, SRSessionPolicy, SRSiteSummary, SRStateEditReq, + SRStateInfo, SRStatusInfo, SRSvcAccChange, SRSvcAccCreate, SRUserStatsSummary, SiteReplicationInfo, SyncStatus, WorkerStat, }; use rustfs_policy::policy::{ Policy, action::{Action, AdminAction}, }; use s3s::dto::{ - DeleteMarkerReplicationStatus, DeleteReplicationStatus, ExistingObjectReplicationStatus, ReplicaModificationsStatus, - ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, + DeleteMarkerReplicationStatus, DeleteReplicationStatus, ExistingObjectReplicationStatus, ObjectLockConfiguration, + ReplicaModificationsStatus, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, VersioningConfiguration, }; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use serde::Deserialize; @@ -287,12 +289,21 @@ struct SiteReplicationAddPreflightInfo { endpoint: String, deployment_id: String, enabled: bool, - bucket_count: usize, - bucket_names: HashSet, + buckets: BTreeMap, peer_deployment_ids: BTreeSet, idp_settings: serde_json::Value, } +/// The per-bucket facts the add preflight compares across sites when more +/// than one requested site holds data (rustfs/backlog#2070). Only properties +/// that cannot converge after the add belong here — everything else is +/// reconciled by the bucket-metadata sync. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct AddPreflightBucketCompat { + versioning_enabled: bool, + object_lock_enabled: bool, +} + #[derive(Debug, Clone, Serialize, Deserialize, Default)] struct SRPeerJoinResponse { peer: PeerInfo, @@ -935,19 +946,65 @@ fn idp_settings_value(settings: &IDPSettings) -> S3Result { .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize IDP settings failed: {e}"))) } +/// The merge-critical facts of one bucket a site reported in its add +/// preflight metainfo. The configs arrive as the `build_sr_info` wire form +/// (base64-encoded XML); an undecodable config fails the preflight instead of +/// defaulting, so corruption cannot admit an unsafe merge. +fn add_preflight_bucket_compat(endpoint: &str, bucket: &str, info: &SRBucketInfo) -> S3Result { + let versioning_enabled = info + .versioning + .as_deref() + .map(|raw| { + deserialize::(&decode_bucket_meta_wire_value(raw)) + .map(|config| config.enabled()) + .map_err(|e| { + s3_error!( + InvalidRequest, + "site `{endpoint}` reported an unreadable versioning config for bucket `{bucket}`: {e}" + ) + }) + }) + .transpose()? + .unwrap_or(false); + let object_lock_enabled = info + .object_lock_config + .as_deref() + .map(|raw| { + deserialize::(&decode_bucket_meta_wire_value(raw)) + .map(|config| config.enabled()) + .map_err(|e| { + s3_error!( + InvalidRequest, + "site `{endpoint}` reported an unreadable object-lock config for bucket `{bucket}`: {e}" + ) + }) + }) + .transpose()? + .unwrap_or(false); + Ok(AddPreflightBucketCompat { + versioning_enabled, + object_lock_enabled, + }) +} + fn add_preflight_info_from_sr_info( site: &PeerSite, info: SRInfo, idp_settings: IDPSettings, ) -> S3Result { - let bucket_names = info.buckets.keys().cloned().collect(); + let buckets = info + .buckets + .iter() + .map(|(bucket, bucket_info)| { + add_preflight_bucket_compat(&site.endpoint, bucket, bucket_info).map(|compat| (bucket.clone(), compat)) + }) + .collect::>>()?; Ok(SiteReplicationAddPreflightInfo { name: if info.name.is_empty() { site.name.clone() } else { info.name }, endpoint: site.endpoint.clone(), deployment_id: info.deployment_id, enabled: info.enabled, - bucket_count: info.buckets.len(), - bucket_names, + buckets, peer_deployment_ids: info.state.peers.keys().cloned().collect(), idp_settings: idp_settings_value(&idp_settings)?, }) @@ -1045,7 +1102,7 @@ fn validate_add_preflight_topology(infos: &[SiteReplicationAddPreflightInfo], lo if info.deployment_id == local_peer.deployment_id { local_seen = true; } - if info.bucket_count > 0 { + if !info.buckets.is_empty() { non_empty_sites.push(info.name.clone()); } } @@ -1070,11 +1127,15 @@ fn validate_add_preflight_topology(infos: &[SiteReplicationAddPreflightInfo], lo } if non_empty_sites.len() > 1 { - return Err(s3_error!( - InvalidRequest, - "site replication can be initialized with data on only one site; non-empty sites: {}", - non_empty_sites.join(", ") - )); + validate_nonempty_add_bucket_compatibility(infos)?; + info!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "nonempty_sites_admitted", + non_empty_sites = %non_empty_sites.join(", "), + "admin site replication state" + ); } let requested: BTreeSet = infos.iter().map(|info| info.deployment_id.clone()).collect(); @@ -1091,6 +1152,76 @@ fn validate_add_preflight_topology(infos: &[SiteReplicationAddPreflightInfo], lo Ok(()) } +/// Operator recovery guidance for a rejected add between sites that both hold +/// data — the only supported path is to empty one side and let a resync copy +/// the objects back (rustfs/backlog#2070). +const NONEMPTY_ADD_RECOVERY_HINT: &str = "to pair these sites, delete the conflicting bucket (or its data) on all but one \ + site, re-run `replicate add`, then run `replicate resync` from the surviving site to restore the objects"; + +/// Admission check for an add in which more than one requested site holds +/// data — the DR re-pair case: two sites that were unpaired (or never +/// finished a removal) both keep their buckets, and the historical +/// unconditional "only one site may hold data" rejection made `replicate +/// remove` a one-way door (rustfs/backlog#2070). +/// +/// The add is admitted when every bucket name held by MORE than one requested +/// site is provably safe to merge through the existing backfill/resync +/// convergence: +/// +/// - versioning must be Enabled on every holder: replication into a versioned +/// bucket lands as another version, so a same-key object from the peer +/// never destroys the local copy — while on an unversioned holder it would +/// silently replace the only copy; +/// - object-lock enablement must match across holders: lock cannot be toggled +/// after bucket creation, so a mismatch never converges, and replicating +/// locked objects into a lock-less bucket would strip their WORM guarantee. +/// +/// A bucket held by a single site carries no merge risk — the post-add +/// backfill creates it on the peers exactly as the historical +/// one-non-empty-site path always has. +fn validate_nonempty_add_bucket_compatibility(infos: &[SiteReplicationAddPreflightInfo]) -> S3Result<()> { + let mut holders: BTreeMap<&str, Vec<(&SiteReplicationAddPreflightInfo, AddPreflightBucketCompat)>> = BTreeMap::new(); + for info in infos { + for (bucket, compat) in &info.buckets { + holders.entry(bucket.as_str()).or_default().push((info, *compat)); + } + } + + for (bucket, holders) in holders { + let [(first, first_compat), rest @ ..] = holders.as_slice() else { + continue; + }; + if rest.is_empty() { + continue; + } + if let Some((conflicting, _)) = rest + .iter() + .find(|(_, compat)| compat.object_lock_enabled != first_compat.object_lock_enabled) + { + let (enabled_on, disabled_on) = if first_compat.object_lock_enabled { + (&first.name, &conflicting.name) + } else { + (&conflicting.name, &first.name) + }; + return Err(s3_error!( + InvalidRequest, + "bucket `{bucket}` has object lock enabled on site `{enabled_on}` but not on site `{disabled_on}`, and \ + object lock cannot be changed after bucket creation; {NONEMPTY_ADD_RECOVERY_HINT}" + )); + } + if let Some((unversioned, _)) = holders.iter().find(|(_, compat)| !compat.versioning_enabled) { + return Err(s3_error!( + InvalidRequest, + "bucket `{bucket}` exists on more than one site but does not have versioning enabled on site `{}`, so \ + merging could silently overwrite objects; enable versioning on every site holding it, or {NONEMPTY_ADD_RECOVERY_HINT}", + unversioned.name + )); + } + } + + Ok(()) +} + fn site_replication_bootstrap_token(uri: &Uri) -> Option { query_pairs(uri).get("bootstrapToken").cloned() } @@ -5701,6 +5832,53 @@ fn sts_replication_compatibility_policy<'a>(claims: &HashMap, par (!claims.contains_key(OIDC_VIRTUAL_PARENT_CLAIM) && !parent_policy_mapping.is_empty()).then_some(parent_policy_mapping) } +/// Adopt only the fields a committed add computed onto the freshly loaded +/// transaction state. Everything else is owned by writers that commit without +/// touching `updated_at` (retry events, peer-edit generations, resync +/// progress, the acks/clears of an already pending rotation), so the add's +/// `updated_at` CAS cannot vouch for them — they keep the freshly loaded +/// value, except `pending_remove`: +/// +/// A committed add supersedes a half-finished removal THIS site started, +/// exactly as an accepted join does on the receiving side (`apply_peer_join`, +/// rustfs/rustfs#5963): the adopted topology IS the new membership, while the +/// pending record only exists to keep notifying peers about the old one. Left +/// in place, the reconcile tick would replay the stale `SRRemoveReq` against +/// a freshly re-paired peer — `SRPeerRemoveHandler` applies it +/// unconditionally — and dismantle the pairing this add just created +/// (rustfs/backlog#2070). A removal that started AFTER the add's preflight +/// snapshot moved `updated_at`, so the CAS refuses the commit before this +/// runs. +/// +/// The exhaustive destructure makes adding a state field a compile error here +/// until it is classified. +fn adopt_add_commit_state(state: &mut SiteReplicationState, next_state: SiteReplicationState) { + let SiteReplicationState { + name, + service_account_access_key, + service_account_secret_key: _, + service_account_parent, + peers, + updated_at, + resync_status: _, + pending_rotation: _, + pending_remove: _, + pending_endpoint_refresh: _, + retry_queue: _, + iam_deletion_replays: _, + sync_state_initialized, + edit_generation: _, + applied_edit_generations: _, + } = next_state; + state.name = name; + state.service_account_access_key = service_account_access_key; + state.service_account_parent = service_account_parent; + state.peers = peers; + state.updated_at = updated_at; + state.sync_state_initialized = sync_state_initialized; + state.pending_remove = None; +} + pub struct SiteReplicationAddHandler {} /// MinIO's `SRPeerJoin` replies with an empty body on success; synthesize the @@ -5756,7 +5934,7 @@ impl Operation for SiteReplicationAddHandler { let bootstrap_buckets = preflight_infos .iter() .filter(|info| !same_identity_endpoint(&info.endpoint, &local_peer.endpoint)) - .flat_map(|info| info.bucket_names.iter().cloned()) + .flat_map(|info| info.buckets.keys().cloned()) .collect(); let add_in_progress_guard = SiteReplicationAddInProgressGuard::start(lifecycle_guard, bootstrap_buckets)?; let mut state = merge_add_sites( @@ -5853,36 +6031,7 @@ impl Operation for SiteReplicationAddHandler { "site replication state changed during peer join; the peers may already be joined — re-run replicate add" )); } - // Adopt only the fields this add computed. Everything else is - // owned by writers that commit without touching `updated_at` - // (retry events, peer-edit generations, resync progress, the - // acks/clears of an already pending rotation or removal), so the - // CAS above cannot vouch for them — they keep the freshly loaded - // value. The exhaustive destructure makes adding a state field a - // compile error here until it is classified. - let SiteReplicationState { - name, - service_account_access_key, - service_account_secret_key: _, - service_account_parent, - peers, - updated_at, - resync_status: _, - pending_rotation: _, - pending_remove: _, - pending_endpoint_refresh: _, - retry_queue: _, - iam_deletion_replays: _, - sync_state_initialized, - edit_generation: _, - applied_edit_generations: _, - } = next_state; - state.name = name; - state.service_account_access_key = service_account_access_key; - state.service_account_parent = service_account_parent; - state.peers = peers; - state.updated_at = updated_at; - state.sync_state_initialized = sync_state_initialized; + adopt_add_commit_state(state, next_state); let edit_generation = next_peer_edit_generation(state); Ok((state.clone(), edit_generation)) }) @@ -9421,18 +9570,29 @@ mod tests { } fn preflight_site(name: &str, endpoint: &str, deployment_id: &str, bucket_count: usize) -> SiteReplicationAddPreflightInfo { + // Site-prefixed names keep the generated buckets disjoint across + // sites; tests exercising shared-bucket merges insert their own. + let buckets = (0..bucket_count) + .map(|i| (format!("{name}-bucket-{i}"), versioned_bucket())) + .collect(); SiteReplicationAddPreflightInfo { name: name.to_string(), endpoint: endpoint.to_string(), deployment_id: deployment_id.to_string(), enabled: false, - bucket_count, - bucket_names: HashSet::new(), + buckets, peer_deployment_ids: BTreeSet::new(), idp_settings: serde_json::json!({"provider": "same"}), } } + fn versioned_bucket() -> AddPreflightBucketCompat { + AddPreflightBucketCompat { + versioning_enabled: true, + object_lock_enabled: false, + } + } + #[test] fn test_validate_add_preflight_topology_accepts_matching_sites() { let local_peer = PeerInfo { @@ -9491,20 +9651,120 @@ mod tests { assert!(err.to_string().contains("IDP settings mismatch")); } + // rustfs/backlog#2070: two sites that both hold data (the DR re-pair + // case) must be admitted when their bucket sets are merge-safe, instead + // of the historical unconditional "only one site may hold data" rejection + // that made `replicate remove` a one-way door. #[test] - fn test_validate_add_preflight_topology_rejects_multiple_non_empty_sites() { + fn test_validate_add_preflight_topology_accepts_compatible_non_empty_sites() { + let local_peer = PeerInfo { + deployment_id: "local-dep".to_string(), + ..peer("local", "https://local.example.com") + }; + let mut local = preflight_site("local", "https://local.example.com", "local-dep", 1); + let mut remote = preflight_site("remote", "https://remote.example.com", "remote-dep", 1); + // The same bucket on both sites, versioning enabled on both: the + // exact shape a formerly paired cluster is left in after a remove. + local.buckets.insert("shared".to_string(), versioned_bucket()); + remote.buckets.insert("shared".to_string(), versioned_bucket()); + let infos = vec![local, remote]; + + validate_add_preflight_topology(&infos, &local_peer).expect("compatible non-empty sites should be admitted"); + } + + #[test] + fn test_validate_add_preflight_topology_accepts_disjoint_non_empty_sites() { let local_peer = PeerInfo { deployment_id: "local-dep".to_string(), ..peer("local", "https://local.example.com") }; let infos = vec![ - preflight_site("local", "https://local.example.com", "local-dep", 1), - preflight_site("remote", "https://remote.example.com", "remote-dep", 1), + preflight_site("local", "https://local.example.com", "local-dep", 2), + preflight_site("remote", "https://remote.example.com", "remote-dep", 2), ]; - let err = validate_add_preflight_topology(&infos, &local_peer).expect_err("multiple non-empty sites should fail"); + validate_add_preflight_topology(&infos, &local_peer).expect("disjoint non-empty sites should be admitted"); + } - assert!(err.to_string().contains("only one site")); + #[test] + fn test_validate_add_preflight_topology_rejects_shared_bucket_object_lock_mismatch() { + let local_peer = PeerInfo { + deployment_id: "local-dep".to_string(), + ..peer("local", "https://local.example.com") + }; + let mut local = preflight_site("local", "https://local.example.com", "local-dep", 0); + let mut remote = preflight_site("remote", "https://remote.example.com", "remote-dep", 0); + local.buckets.insert( + "shared".to_string(), + AddPreflightBucketCompat { + versioning_enabled: true, + object_lock_enabled: true, + }, + ); + remote.buckets.insert("shared".to_string(), versioned_bucket()); + let infos = vec![local, remote]; + + let err = validate_add_preflight_topology(&infos, &local_peer).expect_err("object-lock mismatch should fail"); + + let message = err.to_string(); + assert!( + message.contains("bucket `shared` has object lock enabled on site `local`"), + "got: {message}" + ); + // The rejection must carry the operator recovery steps, not a bare no. + assert!(message.contains("re-run `replicate add`"), "got: {message}"); + assert!(message.contains("`replicate resync`"), "got: {message}"); + } + + #[test] + fn test_validate_add_preflight_topology_rejects_shared_unversioned_bucket() { + let local_peer = PeerInfo { + deployment_id: "local-dep".to_string(), + ..peer("local", "https://local.example.com") + }; + let mut local = preflight_site("local", "https://local.example.com", "local-dep", 0); + let mut remote = preflight_site("remote", "https://remote.example.com", "remote-dep", 0); + local.buckets.insert("shared".to_string(), versioned_bucket()); + remote.buckets.insert( + "shared".to_string(), + AddPreflightBucketCompat { + versioning_enabled: false, + object_lock_enabled: false, + }, + ); + let infos = vec![local, remote]; + + let err = validate_add_preflight_topology(&infos, &local_peer).expect_err("shared unversioned bucket should fail"); + + let message = err.to_string(); + assert!( + message.contains("bucket `shared`") && message.contains("versioning enabled on site `remote`"), + "got: {message}" + ); + assert!(message.contains("re-run `replicate add`"), "got: {message}"); + } + + // A bucket held by a single site never blocks the add, whatever its + // configs: the backfill creates it on the peers exactly like the + // historical one-non-empty-site path. + #[test] + fn test_validate_add_preflight_topology_ignores_unshared_bucket_configs() { + let local_peer = PeerInfo { + deployment_id: "local-dep".to_string(), + ..peer("local", "https://local.example.com") + }; + let mut local = preflight_site("local", "https://local.example.com", "local-dep", 1); + let remote = preflight_site("remote", "https://remote.example.com", "remote-dep", 1); + local.buckets.insert( + "local-only".to_string(), + AddPreflightBucketCompat { + versioning_enabled: false, + object_lock_enabled: true, + }, + ); + let infos = vec![local, remote]; + + validate_add_preflight_topology(&infos, &local_peer).expect("unshared buckets should not block the add"); } #[test] @@ -9524,6 +9784,91 @@ mod tests { assert!(err.to_string().contains("different site replication peer set")); } + // add_preflight_bucket_compat reads the build_sr_info wire form: + // base64-encoded XML for both the versioning and the object-lock config. + #[test] + fn test_add_preflight_bucket_compat_parses_wire_configs() { + let info = SRBucketInfo { + versioning: Some( + BASE64_STANDARD.encode_to_string(b"Enabled"), + ), + object_lock_config: Some(BASE64_STANDARD.encode_to_string( + b"Enabled", + )), + ..Default::default() + }; + + let compat = add_preflight_bucket_compat("https://a.example.com", "b", &info).expect("wire configs should parse"); + + assert!(compat.versioning_enabled); + assert!(compat.object_lock_enabled); + } + + #[test] + fn test_add_preflight_bucket_compat_absent_and_suspended_configs_are_disabled() { + let absent = add_preflight_bucket_compat("https://a.example.com", "b", &SRBucketInfo::default()) + .expect("absent configs should parse"); + assert!(!absent.versioning_enabled); + assert!(!absent.object_lock_enabled); + + let suspended = SRBucketInfo { + versioning: Some( + BASE64_STANDARD + .encode_to_string(b"Suspended"), + ), + ..Default::default() + }; + let compat = + add_preflight_bucket_compat("https://a.example.com", "b", &suspended).expect("suspended config should parse"); + assert!(!compat.versioning_enabled, "suspended versioning is not merge-safe"); + } + + // rustfs/backlog#2070: a committed add must supersede this site's own + // half-finished removal (mirroring the join side, rustfs/rustfs#5963) — + // otherwise the reconcile tick replays the stale removal against the + // freshly re-paired peer and dismantles the new pairing. + #[test] + fn test_adopt_add_commit_state_clears_pending_remove() { + let mut state = SiteReplicationState { + pending_remove: Some(PendingRemove { + id: "remove-1".to_string(), + ..Default::default() + }), + edit_generation: 7, + ..Default::default() + }; + let next_state = SiteReplicationState { + name: "local".to_string(), + peers: BTreeMap::from([ + ("local-dep".to_string(), peer("local", "https://local.example.com")), + ("remote-dep".to_string(), peer("remote", "https://remote.example.com")), + ]), + updated_at: Some(OffsetDateTime::now_utc()), + sync_state_initialized: true, + ..Default::default() + }; + + adopt_add_commit_state(&mut state, next_state); + + assert!(state.pending_remove.is_none(), "the committed add supersedes the removal"); + assert_eq!(state.peers.len(), 2, "the add's topology is adopted"); + assert_eq!(state.edit_generation, 7, "commit-owned fields keep the loaded value"); + } + + // Fail closed: a config this site cannot read must fail the preflight + // instead of defaulting into an unsafe admission. + #[test] + fn test_add_preflight_bucket_compat_rejects_undecodable_config() { + let info = SRBucketInfo { + versioning: Some(BASE64_STANDARD.encode_to_string(b" bool; +} + +impl AdminObjectLockConfigExt for s3s::dto::ObjectLockConfiguration { + fn enabled(&self) -> bool { + ::enabled(self) + } +} + pub(crate) mod bandwidth { pub(crate) mod monitor { pub(crate) type BandwidthDetails = super::super::ecstore_bucket::bandwidth::monitor::BandwidthDetails; @@ -863,7 +873,9 @@ pub(crate) mod bucket { pub(crate) use super::replication; pub(crate) use super::target; pub(crate) use super::versioning_sys; - pub(crate) use super::{AdminReplicationConfigExt, AdminVersioningConfigExt, is_reserved_or_invalid_bucket}; + pub(crate) use super::{ + AdminObjectLockConfigExt, AdminReplicationConfigExt, AdminVersioningConfigExt, is_reserved_or_invalid_bucket, + }; pub(crate) mod utils { pub(crate) use super::super::ecstore_utils::{deserialize, is_valid_object_prefix, serialize}; diff --git a/rustfs/src/app/object/extract.rs b/rustfs/src/app/object/extract.rs index e55231b0d..12f367a13 100644 --- a/rustfs/src/app/object/extract.rs +++ b/rustfs/src/app/object/extract.rs @@ -15,6 +15,12 @@ //! Snowball auto-extract (PutObject x-amz-meta-snowball-auto-extract) path. use super::*; +use crate::app::storage_api::object_usecase::bucket::replication::ReplicateDecision; +#[cfg(test)] +use crate::app::storage_api::object_usecase::concurrency::SNOWBALL_MEMBER_COMMIT_LIMIT; +use crate::app::storage_api::object_usecase::concurrency::SNOWBALL_STAGING_BYTES_LIMIT; +use futures::stream::FuturesUnordered; +use std::collections::HashSet; // One logical member can be preceded by local PAX, GNU long-name, and GNU // long-link records. Count all four physical headers without rejecting that @@ -240,8 +246,12 @@ fn track_extract_member_read_errors(reader: HashReader) -> std::io::Result<(Hash Ok((tracked, failed)) } -fn should_ignore_extract_member_write_error(ignore_errors: bool, member_read_failed: &AtomicBool) -> bool { - ignore_errors && !member_read_failed.load(Ordering::Acquire) +fn classify_extract_member_write_error(error: S3Error, member_read_failed: &AtomicBool) -> ExtractCommitError { + if member_read_failed.load(Ordering::Acquire) { + ExtractCommitError::Fatal(error) + } else { + ExtractCommitError::StorageWrite(error) + } } pin_project! { @@ -309,6 +319,19 @@ const SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER: &str = "snowball-ignore-dirs"; const SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER: &str = "snowball-ignore-errors"; +const SNOWBALL_STORED_TRANSPORT_KEYS_LOWER: &[&str] = &[ + "snowball-auto-extract", + "snowball-prefix", + "snowball-ignore-dirs", + "snowball-ignore-errors", + "minio-snowball-prefix", + "minio-snowball-ignore-dirs", + "minio-snowball-ignore-errors", + "rustfs-snowball-prefix", + "rustfs-snowball-ignore-dirs", + "rustfs-snowball-ignore-errors", +]; + const SNOWBALL_PREFIX_HEADER_KEYS: &[&str] = &[AMZ_MINIO_SNOWBALL_PREFIX, AMZ_SNOWBALL_PREFIX, AMZ_RUSTFS_SNOWBALL_PREFIX]; const SNOWBALL_IGNORE_DIRS_HEADER_KEYS: &[&str] = &[ @@ -323,6 +346,583 @@ const SNOWBALL_IGNORE_ERRORS_HEADER_KEYS: &[&str] = &[ AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS, ]; +const EXTRACT_MAX_EFFECTIVE_PAX_HEADER_BYTES: usize = 8 * 1024; +const EXTRACT_MAX_EFFECTIVE_PAX_USER_METADATA_BYTES: usize = 2 * 1024; +const EXTRACT_MAX_EFFECTIVE_PAX_FIELDS: usize = 4096; +const EXTRACT_MAX_EXPANDED_PAX_METADATA_BYTES: u64 = 128 * 1024 * 1024; +const EXTRACT_SMALL_MEMBER_MAX_BYTES: usize = 64 * 1024; +const EXTRACT_DEFAULT_MAX_INFLIGHT: usize = 1; +const EXTRACT_BATCH_MAX_MEMBERS: usize = 16; +const EXTRACT_BATCH_MAX_STAGING_BYTES: usize = 2 * 1024 * 1024; +const EXTRACT_MEMBER_CONTEXT_OVERHEAD_BYTES: usize = 512; +const EXTRACT_METADATA_ENTRY_OVERHEAD_BYTES: usize = 64; +const ENV_RUSTFS_SNOWBALL_EXTRACT_MAX_INFLIGHT: &str = "RUSTFS_SNOWBALL_EXTRACT_MAX_INFLIGHT"; +const TAR_TYPEFLAG_OFFSET: usize = 156; + +fn put_object_extract_max_inflight() -> usize { + static MAX_INFLIGHT: OnceLock = OnceLock::new(); + *MAX_INFLIGHT.get_or_init(|| { + normalize_put_object_extract_max_inflight(rustfs_utils::get_env_usize( + ENV_RUSTFS_SNOWBALL_EXTRACT_MAX_INFLIGHT, + EXTRACT_DEFAULT_MAX_INFLIGHT, + )) + }) +} + +fn normalize_put_object_extract_max_inflight(value: usize) -> usize { + value.clamp(1, EXTRACT_BATCH_MAX_MEMBERS) +} + +fn select_put_object_extract_max_inflight(configured: usize, ignore_errors: bool, quota_enabled: bool) -> usize { + if ignore_errors && !quota_enabled { configured } else { 1 } +} + +struct ExtractPutRequestGuard { + inner: PutObjectGuard, + succeeded: bool, +} + +impl ExtractPutRequestGuard { + fn new() -> Self { + Self { + inner: PutObjectGuard::new(), + succeeded: false, + } + } + + fn finish_ok(&mut self) { + self.succeeded = true; + } +} + +impl Drop for ExtractPutRequestGuard { + fn drop(&mut self) { + if self.succeeded { + self.inner.finish_ok(); + } else { + self.inner.finish_err(); + } + } +} + +struct ExtractStagedBody { + reader: Option, +} + +impl ExtractStagedBody { + fn empty() -> Self { + Self { reader: None } + } +} + +impl AsyncRead for ExtractStagedBody { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, target: &mut ReadBuf<'_>) -> Poll> { + let Some(reader) = self.reader.as_mut() else { + return Poll::Ready(Ok(())); + }; + Pin::new(reader).poll_read(cx, target) + } +} + +async fn stage_extract_member_body(body: &mut R, size: usize) -> S3Result +where + R: AsyncRead + Unpin, +{ + if size == 0 { + return Ok(ExtractStagedBody::empty()); + } + + let pool = get_concurrency_manager().bytes_pool(); + let mut buffer = pool.acquire_buffer(size).await; + super::put::read_small_put_body_into(body, &mut *buffer, size).await?; + Ok(ExtractStagedBody { + reader: Some(super::put::PooledBufferReader::new(buffer, size)), + }) +} + +fn checked_extract_hash_map_allocation(map: &HashMap) -> Option { + // HashMap does not expose its raw bucket allocation. Twice the advertised + // element capacity conservatively covers buckets plus control bytes while + // keeping the accounting independent of the current hashbrown layout. + map.capacity() + .checked_mul(2)? + .checked_mul(std::mem::size_of::<(K, V)>().checked_add(1)?) +} + +fn checked_extract_member_staging_weight( + path: &str, + size: usize, + opts: &ObjectOptions, + replication: &ReplicateDecision, +) -> S3Result { + let body_reservation = if size == 0 { 0 } else { EXTRACT_SMALL_MEMBER_MAX_BYTES }; + // The authorization request is dropped before staging. Account the two + // retained key copies plus all dynamic maps frozen into ObjectOptions; + // the fixed allowance covers the remaining options/write-plan handles. + let mut total = body_reservation + .checked_add( + path.len() + .checked_mul(2) + .ok_or_else(|| object_s3_error(S3ErrorCode::InvalidArgument, "Snowball prepared member path size overflowed"))?, + ) + .and_then(|bytes| bytes.checked_add(EXTRACT_MEMBER_CONTEXT_OVERHEAD_BYTES)) + .ok_or_else(|| object_s3_error(S3ErrorCode::InvalidArgument, "Snowball prepared member size overflowed"))?; + for metadata in std::iter::once(&opts.user_defined).chain(opts.eval_metadata.iter()) { + for (name, value) in metadata { + total = total + .checked_add(name.len()) + .and_then(|bytes| bytes.checked_add(value.len())) + .and_then(|bytes| bytes.checked_add(EXTRACT_METADATA_ENTRY_OVERHEAD_BYTES)) + .ok_or_else(|| object_s3_error(S3ErrorCode::InvalidArgument, "Snowball prepared member size overflowed"))?; + } + } + total = total + .checked_add( + checked_extract_hash_map_allocation(&replication.targets_map) + .ok_or_else(|| object_s3_error(S3ErrorCode::InvalidArgument, "Snowball replication decision size overflowed"))?, + ) + .ok_or_else(|| object_s3_error(S3ErrorCode::InvalidArgument, "Snowball prepared member size overflowed"))?; + for (target_name, target) in &replication.targets_map { + total = total + .checked_add(target_name.capacity()) + .and_then(|bytes| bytes.checked_add(target.arn.capacity())) + .and_then(|bytes| bytes.checked_add(target.id.capacity())) + .ok_or_else(|| object_s3_error(S3ErrorCode::InvalidArgument, "Snowball replication decision size overflowed"))?; + } + Ok(total) +} + +fn try_acquire_extract_staging_permit(manager: &ConcurrencyManager, staging_weight: usize) -> S3Result { + if staging_weight > SNOWBALL_STAGING_BYTES_LIMIT { + return Err(object_s3_error( + S3ErrorCode::SlowDown, + "Snowball member retained state exceeds the global staging budget", + )); + } + let permits = u32::try_from(staging_weight).map_err(|_| { + object_s3_error(S3ErrorCode::SlowDown, "Snowball member retained state exceeds the global staging budget") + })?; + manager.try_acquire_snowball_staging_bytes(permits).ok_or_else(|| { + object_s3_error( + S3ErrorCode::SlowDown, + "Snowball staging budget is exhausted, please reduce your request rate", + ) + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ExtractBatchAction { + Stage, + FlushThenStage, + SerialBarrier, +} + +#[derive(Debug, Default)] +struct ExtractBatchState { + keys: HashSet, + staging_bytes: usize, +} + +impl ExtractBatchState { + fn action( + &self, + key: &str, + member_size: usize, + staging_weight: usize, + max_inflight: usize, + durable_quota: bool, + ) -> ExtractBatchAction { + if durable_quota + || max_inflight <= 1 + || member_size > EXTRACT_SMALL_MEMBER_MAX_BYTES + || staging_weight > EXTRACT_BATCH_MAX_STAGING_BYTES + { + return ExtractBatchAction::SerialBarrier; + } + + if self.keys.contains(key) + || self.keys.len() >= max_inflight + || self + .staging_bytes + .checked_add(staging_weight) + .is_none_or(|bytes| bytes > EXTRACT_BATCH_MAX_STAGING_BYTES) + { + ExtractBatchAction::FlushThenStage + } else { + ExtractBatchAction::Stage + } + } + + fn record(&mut self, key: &str, staging_weight: usize) { + debug_assert!(!self.keys.contains(key)); + self.keys.insert(key.to_string()); + self.staging_bytes = self.staging_bytes.saturating_add(staging_weight); + } + + fn clear(&mut self) { + self.keys.clear(); + self.staging_bytes = 0; + } +} + +struct ExtractPreparedMember { + archive_seq: usize, + key: String, + size: i64, + actual_size: i64, + body: R, + write_plan: WritePlan, + opts: ObjectOptions, + replication: ReplicateDecision, + staging_permit: OwnedSemaphorePermit, + member_permit: OwnedSemaphorePermit, +} + +struct ExtractCommitContext { + store: Arc, + cache_adapter: Arc, + notify: Arc, + bucket: String, + quota_enabled: bool, + request_context: request_context::RequestContext, + req_params: hashbrown::HashMap, + host: String, + port: u16, + user_agent: String, + wrote_any_entry: AtomicBool, +} + +struct ExtractCommitSuccess { + event: rustfs_notify::EventArgs, + post_commit_error: Option, +} + +enum ExtractCommitError { + StorageWrite(S3Error), + Fatal(S3Error), +} + +impl From for ExtractCommitError { + fn from(error: ApiError) -> Self { + Self::Fatal(error.into()) + } +} + +impl ExtractCommitError { + fn into_unignored(self, ignore_errors: bool) -> Option { + match self { + Self::StorageWrite(error) if ignore_errors => { + warn!(error = %error, "Archive object write skipped due to ignore-errors"); + None + } + Self::StorageWrite(error) | Self::Fatal(error) => Some(error), + } + } +} + +struct ExtractCommitOutcome { + archive_seq: usize, + event: Option, + error: Option, +} + +async fn run_extract_owned_task(task: F) -> S3Result +where + T: Send + 'static, + F: std::future::Future + Send + 'static, +{ + spawn_traced_join(task) + .await + .map_err(|error| S3Error::with_message(S3ErrorCode::InternalError, format!("Snowball commit owner task failed: {error}"))) +} + +async fn complete_extract_member_post_commit( + context: Arc, + key: String, + opts: ObjectOptions, + replication: ReplicateDecision, + obj_info: ObjectInfo, + backfilled_old_current_size: Option, +) -> ExtractCommitSuccess { + let extract_versioned = opts.versioned; + let post_commit_error = match quota_accounting_object_size(&obj_info, context.quota_enabled) { + Ok(committed_size) => { + match previous_current_size_from_backfill(backfilled_old_current_size) { + Some(previous_current_size) => { + if extract_versioned { + record_bucket_object_version_write_memory(&context.bucket, previous_current_size, committed_size).await; + } else { + record_bucket_object_write_memory(&context.bucket, previous_current_size, committed_size).await; + } + } + None => { + record_bucket_object_write_unknown_previous_memory(&context.bucket, committed_size, extract_versioned).await; + } + } + None + } + Err(err) => Some(err), + }; + let _ = invalidate_object_data_cache_after_put_success(&context.cache_adapter, &context.bucket, &key).await; + + if replication.replicate_any() { + schedule_object_replication(obj_info.clone(), context.store.clone(), replication).await; + } + + let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)); + let output = PutObjectOutput { + e_tag, + ..Default::default() + }; + let actual_version_id = extract_notification_version_id(obj_info.version_id, opts.versioned, opts.version_suspended); + let mut event_object = convert_ecstore_object_info(obj_info); + event_object.version_id = (!actual_version_id.is_empty()).then_some(actual_version_id.clone()); + let event = rustfs_notify::EventArgs { + event_name: put_event_name_for_post_object(false), + bucket_name: context.bucket.clone(), + object: event_object, + req_params: context.req_params.clone(), + resp_elements: build_event_resp_elements(&S3Response::new(output), &context.request_context.request_id), + version_id: actual_version_id, + host: context.host.clone(), + port: context.port, + user_agent: context.user_agent.clone(), + }; + + ExtractCommitSuccess { + event, + post_commit_error, + } +} + +async fn commit_extract_member_inner( + context: Arc, + member: ExtractPreparedMember, + foreground_permit: Option, +) -> Result +where + R: AsyncRead + Send + Sync + Unpin + 'static, +{ + let ExtractPreparedMember { + archive_seq: _, + key, + size, + actual_size, + body, + write_plan, + opts, + replication, + staging_permit, + member_permit, + } = member; + let hrd = HashReader::from_stream(body, size, actual_size, None, None, false).map_err(ApiError::from)?; + let hrd = write_plan.apply(hrd, actual_size).map_err(ApiError::from)?; + let (hrd, member_read_failed) = track_extract_member_read_errors(hrd).map_err(ApiError::from)?; + let mut reader = PutObjReader::new(hrd); + let _ = invalidate_object_data_cache_before_mutation(&context.cache_adapter, &context.bucket, &key).await; + + let (obj_info, backfilled_old_current_size) = match context + .store + .put_object_with_old_current_size(&context.bucket, &key, &mut reader, &opts) + .await + { + Ok(result) => result, + Err(error) => { + let error: S3Error = ApiError::from(error).into(); + return Err(classify_extract_member_write_error(error, &member_read_failed)); + } + }; + drop(reader); + drop(staging_permit); + drop(foreground_permit); + + // The independently owned commit publishes the authoritative mutation to + // the scanner before its post-store awaits, then retains the lifecycle slot + // through quota, cache, replication, and event construction. + if !context.wrote_any_entry.swap(true, Ordering::AcqRel) { + rustfs_scanner::record_dirty_usage_bucket(&context.bucket); + } + let success = + complete_extract_member_post_commit(context, key, opts, replication, obj_info, backfilled_old_current_size).await; + drop(member_permit); + Ok(success) +} + +async fn commit_extract_member(context: Arc, member: ExtractPreparedMember) -> ExtractCommitOutcome +where + R: AsyncRead + Send + Sync + Unpin + 'static, +{ + let archive_seq = member.archive_seq; + let manager = get_concurrency_manager(); + let foreground_permit = match manager.admit_snowball_foreground_write(member.actual_size).await { + Ok(ForegroundWriteAdmission::Disabled) => None, + Ok(ForegroundWriteAdmission::Admitted(permit)) => Some(permit), + Ok(ForegroundWriteAdmission::Rejected) => { + return ExtractCommitOutcome { + archive_seq, + event: None, + error: Some(ExtractCommitError::StorageWrite(object_s3_error( + S3ErrorCode::SlowDown, + "foreground write concurrency limit reached, please reduce your request rate", + ))), + }; + } + Err(_) => { + return ExtractCommitOutcome { + archive_seq, + event: None, + error: Some(ExtractCommitError::Fatal(object_s3_error( + S3ErrorCode::InternalError, + "Snowball foreground write admission closed", + ))), + }; + } + }; + match commit_extract_member_inner(context, member, foreground_permit).await { + Ok(success) => ExtractCommitOutcome { + archive_seq, + event: Some(success.event), + error: success.post_commit_error.map(ExtractCommitError::Fatal), + }, + Err(error) => ExtractCommitOutcome { + archive_seq, + event: None, + error: Some(error), + }, + } +} + +fn ordered_extract_outcomes( + mut outcomes: Vec, + ignore_errors: bool, +) -> (Vec, Option) { + outcomes.sort_by_key(|outcome| outcome.archive_seq); + let mut events = Vec::with_capacity(outcomes.len()); + let mut earliest_error = None; + for outcome in outcomes { + if let Some(event) = outcome.event { + events.push(event); + } + let error = outcome.error.and_then(|error| error.into_unignored(ignore_errors)); + if earliest_error.is_none() { + earliest_error = error; + } + } + (events, earliest_error) +} + +fn spawn_extract_notification_batch( + request_context: Option, + events: Vec, + notify: F, +) where + F: Fn(rustfs_notify::EventArgs) -> Fut + Send + Sync + 'static, + Fut: std::future::Future + Send + 'static, +{ + spawn_background_with_context(request_context, async move { + for event in events { + notify(event).await; + } + }); +} + +fn finish_extract_outcomes( + outcomes: Vec, + context: &Arc, + ignore_errors: bool, +) -> S3Result<()> { + let (events, earliest_error) = ordered_extract_outcomes(outcomes, ignore_errors); + if !events.is_empty() { + let notify = context.notify.clone(); + spawn_extract_notification_batch(Some(context.request_context.clone()), events, move |event| { + let notify = notify.clone(); + async move { + notify.notify(event).await; + } + }); + } + match earliest_error { + Some(error) => Err(error), + None => Ok(()), + } +} + +async fn run_extract_outcomes_owner(context: Arc, ignore_errors: bool, commits: F) -> S3Result<()> +where + F: std::future::Future> + Send + 'static, +{ + run_extract_owned_task(async move { + let outcomes = commits.await; + finish_extract_outcomes(outcomes, &context, ignore_errors) + }) + .await? +} + +async fn drain_extract_commits(mut commits: FuturesUnordered) -> Vec +where + F: std::future::Future, +{ + let mut outcomes = Vec::with_capacity(commits.len()); + while let Some(outcome) = commits.next().await { + outcomes.push(outcome); + } + outcomes +} + +async fn run_extract_commits(members: impl IntoIterator, mut commit: F) -> Vec +where + F: FnMut(T) -> Fut, + Fut: std::future::Future, +{ + let commits = FuturesUnordered::new(); + for member in members { + commits.push(commit(member)); + } + drain_extract_commits(commits).await +} + +async fn flush_extract_batch( + batch: &mut Vec, + batch_state: &mut ExtractBatchState, + context: &Arc, + ignore_errors: bool, +) -> S3Result<()> { + if batch.is_empty() { + batch_state.clear(); + return Ok(()); + } + + let members = batch.split_off(0); + batch_state.clear(); + let commit_context = context.clone(); + run_extract_outcomes_owner(context.clone(), ignore_errors, async move { + run_extract_commits(members, |member| commit_extract_member(commit_context.clone(), member)).await + }) + .await +} + +async fn acquire_extract_member_lifecycle_permit( + batch: &mut Vec, + batch_state: &mut ExtractBatchState, + context: &Arc, + ignore_errors: bool, +) -> S3Result { + let manager = get_concurrency_manager(); + if let Some(permit) = manager.try_acquire_snowball_member_commit() { + return Ok(permit); + } + if !batch.is_empty() { + flush_extract_batch(batch, batch_state, context, ignore_errors).await?; + if let Some(permit) = manager.try_acquire_snowball_member_commit() { + return Ok(permit); + } + } + manager + .acquire_snowball_member_commit() + .await + .map_err(|_| object_s3_error(S3ErrorCode::InternalError, "Snowball member lifecycle admission closed")) +} + #[derive(Debug, Clone, Default, PartialEq, Eq)] struct PutObjectExtractOptions { prefix: Option, @@ -330,6 +930,28 @@ struct PutObjectExtractOptions { ignore_errors: bool, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ExtractEntryKind { + Directory, + Object, + Skip, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ExtractNormalizedVersion { + storage_id: String, + authorization_id: String, + requires_versioning: bool, +} + +#[derive(Debug, Clone, Default)] +struct ExtractPaxOverrides { + headers: HeaderMap, + header_bytes: usize, + user_metadata_bytes: usize, + version_id: Option, +} + fn header_value_is_true(headers: &HeaderMap, key: &str) -> bool { headers .get(key) @@ -337,6 +959,121 @@ fn header_value_is_true(headers: &HeaderMap, key: &str) -> bool { .is_some_and(|value| value.trim().eq_ignore_ascii_case("true")) } +fn classify_extract_entry_type(entry_type: tokio_tar::EntryType) -> ExtractEntryKind { + if entry_type.is_dir() { + ExtractEntryKind::Directory + } else if entry_type.is_file() + || entry_type.is_character_special() + || entry_type.is_block_special() + || entry_type.is_fifo() + || entry_type.is_gnu_sparse() + { + // `EntryType::Regular` covers both POSIX TypeReg and the legacy NUL + // TypeRegA marker. MinIO materializes zero-sized device and FIFO + // members as empty objects instead of recreating filesystem nodes. + ExtractEntryKind::Object + } else { + // Links, contiguous files, PAX extension carrier records, and unknown + // typeflags are archive control/filesystem semantics, not S3 objects. + // In particular, MinIO does not inherit global PAX metadata into later + // Snowball members. + ExtractEntryKind::Skip + } +} + +fn is_header_only_special_entry(entry_type: tokio_tar::EntryType) -> bool { + entry_type.is_character_special() || entry_type.is_block_special() || entry_type.is_fifo() +} + +fn validate_extract_special_entry_size(entry_type: tokio_tar::EntryType, size: u64) -> S3Result<()> { + if is_header_only_special_entry(entry_type) && size != 0 { + return Err(object_s3_error( + S3ErrorCode::InvalidArgument, + "Snowball special archive member declares a non-zero body", + )); + } + Ok(()) +} + +fn is_legacy_null_directory(header: &tokio_tar::Header, path: &str) -> bool { + header.as_bytes()[TAR_TYPEFLAG_OFFSET] == b'\0' && path.as_bytes().ends_with(b"/") +} + +fn is_snowball_transport_header(key: &str) -> bool { + if key.eq_ignore_ascii_case(AMZ_SNOWBALL_EXTRACT) || key.eq_ignore_ascii_case(AMZ_SNOWBALL_EXTRACT_COMPAT) { + return true; + } + + if is_exact_snowball_meta_key(key, SNOWBALL_PREFIX_HEADER_KEYS) + || is_exact_snowball_meta_key(key, SNOWBALL_IGNORE_DIRS_HEADER_KEYS) + || is_exact_snowball_meta_key(key, SNOWBALL_IGNORE_ERRORS_HEADER_KEYS) + { + return true; + } + + let key = key.to_ascii_lowercase(); + if SNOWBALL_STORED_TRANSPORT_KEYS_LOWER.contains(&key.as_str()) { + return true; + } + + key.starts_with(AMZ_META_PREFIX_LOWER) + && (key.ends_with(SNOWBALL_PREFIX_SUFFIX_LOWER) + || key.ends_with(SNOWBALL_IGNORE_DIRS_SUFFIX_LOWER) + || key.ends_with(SNOWBALL_IGNORE_ERRORS_SUFFIX_LOWER)) +} + +fn snowball_member_headers(headers: &HeaderMap) -> HeaderMap { + let mut member_headers = headers.clone(); + let transport_headers: Vec<_> = member_headers + .keys() + .filter(|name| is_snowball_transport_header(name.as_str())) + .cloned() + .collect(); + for name in transport_headers { + member_headers.remove(name); + } + member_headers +} + +fn normalize_extract_version_id(value: &str) -> S3Result { + let value = value.trim(); + if value == "null" { + return Ok(ExtractNormalizedVersion { + storage_id: Uuid::nil().to_string(), + authorization_id: "null".to_string(), + requires_versioning: false, + }); + } + + let version_id = Uuid::parse_str(value).map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball PAX version ID"))?; + let version_id = version_id.to_string(); + Ok(ExtractNormalizedVersion { + storage_id: version_id.clone(), + authorization_id: version_id, + requires_versioning: true, + }) +} + +fn apply_extract_version_id(value: &str, opts: &mut ObjectOptions) -> S3Result { + let normalized = normalize_extract_version_id(value)?; + if normalized.requires_versioning && !opts.versioned { + return Err(object_s3_error( + S3ErrorCode::InvalidArgument, + "Snowball version ID requires bucket versioning to be enabled", + )); + } + opts.version_id = Some(normalized.storage_id); + Ok(normalized.authorization_id) +} + +fn extract_notification_version_id(version_id: Option, versioned: bool, version_suspended: bool) -> String { + match version_id { + Some(version_id) if !version_id.is_nil() => version_id.to_string(), + Some(_) if versioned || version_suspended => "null".to_string(), + _ => String::new(), + } +} + pub(super) fn is_put_object_extract_requested(headers: &HeaderMap) -> bool { header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT) || header_value_is_true(headers, AMZ_SNOWBALL_EXTRACT_COMPAT) } @@ -421,6 +1158,10 @@ fn map_extract_archive_error(err: std::io::Error) -> S3Error { archive_error } +fn map_extract_pax_text_error(err: impl std::fmt::Display) -> S3Error { + object_s3_error(S3ErrorCode::InvalidArgument, format!("Failed to decode archive PAX metadata: {}", err)) +} + #[derive(Debug)] enum ExtractEntryError { Fatal(S3Error), @@ -447,38 +1188,10 @@ impl ExtractEntryError { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ExtractEntryDisposition { - File, - Directory, - FormatSkip, -} - -fn classify_extract_entry_type(entry_type: tokio_tar::EntryType) -> ExtractEntryDisposition { - use tokio_tar::EntryType; - - match entry_type { - EntryType::Regular | EntryType::Char | EntryType::Block | EntryType::Fifo | EntryType::GNUSparse => { - ExtractEntryDisposition::File - } - EntryType::Directory => ExtractEntryDisposition::Directory, - EntryType::Link - | EntryType::Symlink - | EntryType::GNULongName - | EntryType::GNULongLink - | EntryType::Continuous - | EntryType::XGlobalHeader - | EntryType::XHeader - | EntryType::SolarisXHeader - | EntryType::Other(_) => ExtractEntryDisposition::FormatSkip, - _ => ExtractEntryDisposition::FormatSkip, - } -} - -fn extract_entry_quota_growth(disposition: ExtractEntryDisposition, entry_size: u64) -> u64 { - match disposition { - ExtractEntryDisposition::File => entry_size, - ExtractEntryDisposition::Directory | ExtractEntryDisposition::FormatSkip => 0, +fn extract_entry_quota_growth(kind: ExtractEntryKind, entry_size: u64) -> u64 { + match kind { + ExtractEntryKind::Object => entry_size, + ExtractEntryKind::Directory | ExtractEntryKind::Skip => 0, } } @@ -591,12 +1304,419 @@ fn record_extract_pax_metadata_record( Ok(()) } +fn is_extract_user_metadata_header(name: &http::HeaderName) -> bool { + ["x-amz-meta-", "x-minio-meta-", "x-rustfs-meta-"] + .iter() + .any(|prefix| name.as_str().starts_with(prefix)) +} + +fn extract_pax_header_bytes(name: &http::HeaderName, value: &HeaderValue) -> usize { + name.as_str().len().saturating_add(value.as_bytes().len()) +} + +fn validate_extract_pax_header_budget(headers: &HeaderMap) -> S3Result<()> { + if headers.len() > EXTRACT_MAX_EFFECTIVE_PAX_FIELDS { + return Err(object_s3_error( + S3ErrorCode::InvalidArgument, + "Snowball PAX metadata field count exceeds limit", + )); + } + + let mut header_bytes = 0usize; + let mut user_metadata_bytes = 0usize; + for (name, value) in headers { + let field_bytes = extract_pax_header_bytes(name, value); + header_bytes = header_bytes + .checked_add(field_bytes) + .ok_or_else(|| object_s3_error(S3ErrorCode::InvalidArgument, "Snowball PAX metadata size overflowed"))?; + if is_extract_user_metadata_header(name) { + user_metadata_bytes = user_metadata_bytes + .checked_add(field_bytes) + .ok_or_else(|| object_s3_error(S3ErrorCode::InvalidArgument, "Snowball PAX user metadata size overflowed"))?; + } + } + + if header_bytes > EXTRACT_MAX_EFFECTIVE_PAX_HEADER_BYTES { + return Err(object_s3_error( + S3ErrorCode::InvalidArgument, + "Snowball PAX metadata exceeds effective size limit", + )); + } + if user_metadata_bytes > EXTRACT_MAX_EFFECTIVE_PAX_USER_METADATA_BYTES { + return Err(object_s3_error( + S3ErrorCode::InvalidArgument, + "Snowball PAX user metadata exceeds effective size limit", + )); + } + Ok(()) +} + +fn try_insert_extract_header(headers: &mut HeaderMap, name: http::HeaderName, value: HeaderValue) -> S3Result<()> { + headers + .try_insert(name, value) + .map(|_| ()) + .map_err(|_| object_s3_error(S3ErrorCode::InvalidArgument, "Snowball PAX metadata field count exceeds header capacity")) +} + +fn replace_extract_header(headers: &mut HeaderMap, name: &'static str, value: &str) -> S3Result<()> { + let value = HeaderValue::from_str(value) + .map_err(|_| object_s3_error(S3ErrorCode::InvalidArgument, "Invalid canonical Snowball PAX metadata value"))?; + let name = http::HeaderName::from_bytes(name.as_bytes()) + .map_err(|_| object_s3_error(S3ErrorCode::InvalidArgument, "Invalid canonical Snowball PAX metadata header"))?; + try_insert_extract_header(headers, name, value) +} + +fn extract_pax_metadata_delta_bytes(baseline: &HashMap, metadata: &HashMap) -> S3Result { + metadata + .iter() + .filter(|(key, value)| baseline.get(*key) != Some(*value)) + .try_fold(0u64, |total, (key, value)| { + let field_bytes = key + .len() + .checked_add(value.len()) + .and_then(|size| u64::try_from(size).ok()) + .ok_or_else(|| object_s3_error(S3ErrorCode::InvalidArgument, "Snowball expanded PAX metadata size overflowed"))?; + total + .checked_add(field_bytes) + .ok_or_else(|| object_s3_error(S3ErrorCode::InvalidArgument, "Snowball expanded PAX metadata size overflowed")) + }) +} + +fn validate_extract_expanded_pax_metadata_total(total: u64) -> S3Result<()> { + if total > EXTRACT_MAX_EXPANDED_PAX_METADATA_BYTES { + return Err(object_s3_error( + S3ErrorCode::InvalidArgument, + "Snowball expanded PAX metadata exceeds archive limit", + )); + } + Ok(()) +} + +impl ExtractPaxOverrides { + fn overlay_record(&mut self, key: &str, value: &str) -> S3Result<()> { + if key == "minio.versionId" { + if value.is_empty() { + self.version_id = None; + } else { + self.version_id = Some(normalize_extract_version_id(value)?.authorization_id); + } + return Ok(()); + } + + let Some(meta_key) = key.strip_prefix("minio.metadata.") else { + return Ok(()); + }; + if meta_key.is_empty() { + return Ok(()); + } + + let name = http::HeaderName::from_bytes(meta_key.as_bytes()) + .map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball PAX metadata header"))?; + if is_snowball_transport_header(name.as_str()) { + return Ok(()); + } + if value.is_empty() { + if let Some(previous) = self.headers.remove(&name) { + let previous_bytes = extract_pax_header_bytes(&name, &previous); + self.header_bytes = self.header_bytes.saturating_sub(previous_bytes); + if is_extract_user_metadata_header(&name) { + self.user_metadata_bytes = self.user_metadata_bytes.saturating_sub(previous_bytes); + } + } + return Ok(()); + } + let header_value = HeaderValue::from_str(value) + .map_err(|_| object_s3_error(S3ErrorCode::InvalidArgument, "Invalid Snowball PAX metadata value"))?; + + let previous_bytes = self + .headers + .get(&name) + .map(|previous| extract_pax_header_bytes(&name, previous)) + .unwrap_or_default(); + let next_bytes = extract_pax_header_bytes(&name, &header_value); + let next_header_bytes = self + .header_bytes + .checked_sub(previous_bytes) + .and_then(|bytes| bytes.checked_add(next_bytes)) + .ok_or_else(|| object_s3_error(S3ErrorCode::InvalidArgument, "Snowball PAX metadata size overflowed"))?; + if next_header_bytes > EXTRACT_MAX_EFFECTIVE_PAX_HEADER_BYTES { + return Err(object_s3_error( + S3ErrorCode::InvalidArgument, + "Snowball PAX metadata exceeds effective size limit", + )); + } + + let next_user_metadata_bytes = if is_extract_user_metadata_header(&name) { + self.user_metadata_bytes + .checked_sub(previous_bytes) + .and_then(|bytes| bytes.checked_add(next_bytes)) + .ok_or_else(|| object_s3_error(S3ErrorCode::InvalidArgument, "Snowball PAX user metadata size overflowed"))? + } else { + self.user_metadata_bytes + }; + if next_user_metadata_bytes > EXTRACT_MAX_EFFECTIVE_PAX_USER_METADATA_BYTES { + return Err(object_s3_error( + S3ErrorCode::InvalidArgument, + "Snowball PAX user metadata exceeds effective size limit", + )); + } + if !self.headers.contains_key(&name) && self.headers.len() >= EXTRACT_MAX_EFFECTIVE_PAX_FIELDS { + return Err(object_s3_error( + S3ErrorCode::InvalidArgument, + "Snowball PAX metadata field count exceeds limit", + )); + } + + try_insert_extract_header(&mut self.headers, name, header_value)?; + self.header_bytes = next_header_bytes; + self.user_metadata_bytes = next_user_metadata_bytes; + Ok(()) + } +} + +async fn overlay_extract_pax_extensions( + entry: &mut tokio_tar::Entry>, + overrides: &mut ExtractPaxOverrides, +) -> S3Result<()> +where + R: AsyncRead + Send + Unpin + 'static, +{ + let Some(extensions) = entry.pax_extensions().await.map_err(map_extract_archive_error)? else { + return Ok(()); + }; + + for ext in extensions { + let ext = ext.map_err(map_extract_archive_error)?; + let key = ext.key().map_err(map_extract_pax_text_error)?; + let value = ext.value().map_err(map_extract_pax_text_error)?; + overrides.overlay_record(key, value)?; + } + Ok(()) +} + #[derive(Debug, Default)] struct ExtractEntryPaxAuthorization { headers: HeaderMap, + version_id: Option, object_lock_legal_hold_status: Option, object_lock_mode: Option, object_lock_retain_until_date: Option, + expanded_metadata_bytes: u64, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct ExtractMemberIamRequirements { + tagging: bool, + retention: bool, + legal_hold: bool, + replication: bool, +} + +fn extract_member_iam_requirements( + metadata: &HashMap, + object_lock_legal_hold_status: Option<&ObjectLockLegalHoldStatus>, + object_lock_mode: Option<&ObjectLockMode>, + object_lock_retain_until_date: Option<&Timestamp>, + explicit_version_id: Option<&str>, + replica: bool, +) -> ExtractMemberIamRequirements { + ExtractMemberIamRequirements { + tagging: metadata.contains_key(AMZ_OBJECT_TAGGING), + retention: object_lock_mode.is_some() || object_lock_retain_until_date.is_some(), + legal_hold: object_lock_legal_hold_status.is_some(), + replication: explicit_version_id.is_some() || replica, + } +} + +fn apply_extract_pax_overrides( + overrides: &ExtractPaxOverrides, + bucket: &str, + object_name: &str, + object_lock_config_state: &metadata_sys::ObjectLockConfigState, + metadata: &mut HashMap, + opts: &mut ObjectOptions, +) -> S3Result { + let baseline_metadata = metadata.clone(); + let mut canonical_headers = overrides.headers.clone(); + let normalized_version = overrides + .version_id + .as_deref() + .map(normalize_extract_version_id) + .transpose()?; + if let Some(version) = normalized_version.as_ref() { + opts.version_id = Some(version.storage_id.clone()); + } + + let storage_class = canonical_headers + .get(AMZ_STORAGE_CLASS) + .map(|value| { + value + .to_str() + .map(str::trim) + .map(str::to_owned) + .map_err(|_| object_s3_error_default(S3ErrorCode::InvalidStorageClass)) + }) + .transpose()?; + if let Some(storage_class) = storage_class.as_deref() { + if !is_valid_storage_class(storage_class) { + return Err(object_s3_error_default(S3ErrorCode::InvalidStorageClass)); + } + replace_extract_header(&mut canonical_headers, AMZ_STORAGE_CLASS, storage_class)?; + } + + let tagging = canonical_headers + .get("x-amz-tagging") + .map(|value| { + let value = value + .to_str() + .map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball object tagging value"))?; + crate::app::storage_api::object_usecase::s3_api::tagging::parse_copy_object_tags(value) + }) + .transpose()?; + if let Some(tagging) = tagging.as_deref() { + replace_extract_header(&mut canonical_headers, "x-amz-tagging", tagging)?; + } + + let object_lock_mode = canonical_headers + .get(AMZ_OBJECT_LOCK_MODE_LOWER) + .map(|value| { + let value = value + .to_str() + .map(str::trim) + .map(str::to_ascii_uppercase) + .map_err(|_| object_s3_error(S3ErrorCode::InvalidArgument, "Invalid Snowball Object Lock mode"))?; + match value.as_str() { + ObjectLockMode::GOVERNANCE => Ok(ObjectLockMode::from_static(ObjectLockMode::GOVERNANCE)), + ObjectLockMode::COMPLIANCE => Ok(ObjectLockMode::from_static(ObjectLockMode::COMPLIANCE)), + _ => Err(s3_error!(InvalidArgument, "Invalid Snowball Object Lock mode")), + } + }) + .transpose()?; + if let Some(mode) = object_lock_mode.as_ref() { + replace_extract_header(&mut canonical_headers, AMZ_OBJECT_LOCK_MODE_LOWER, mode.as_str())?; + } + + let object_lock_retain_until_date = canonical_headers + .get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER) + .map(|value| { + let value = value + .to_str() + .map(str::trim) + .map_err(|_| object_s3_error(S3ErrorCode::InvalidArgument, "Invalid Snowball Object Lock retain-until date"))?; + Timestamp::parse(TimestampFormat::DateTime, value) + .map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball Object Lock retain-until date")) + }) + .transpose()?; + if let Some(retain_until_date) = object_lock_retain_until_date.as_ref() { + let formatted = OffsetDateTime::from(retain_until_date.clone()) + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball Object Lock retain-until date"))?; + replace_extract_header(&mut canonical_headers, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, &formatted)?; + } + + let object_lock_legal_hold_status = canonical_headers + .get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER) + .map(|value| { + let value = + value.to_str().map(str::trim).map(str::to_ascii_uppercase).map_err(|_| { + object_s3_error(S3ErrorCode::InvalidArgument, "Invalid Snowball Object Lock legal-hold status") + })?; + match value.as_str() { + ObjectLockLegalHoldStatus::ON => Ok(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::ON)), + ObjectLockLegalHoldStatus::OFF => Ok(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF)), + _ => Err(s3_error!(InvalidArgument, "Invalid Snowball Object Lock legal-hold status")), + } + }) + .transpose()?; + if let Some(status) = object_lock_legal_hold_status.as_ref() { + replace_extract_header(&mut canonical_headers, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, status.as_str())?; + } + + let replica = canonical_headers + .get(AMZ_BUCKET_REPLICATION_STATUS) + .map(|value| { + let value = value + .to_str() + .map(str::trim) + .map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball replication status"))?; + if value.eq_ignore_ascii_case(ReplicationStatusType::Replica.as_str()) { + Ok(true) + } else { + Err(s3_error!(InvalidArgument, "Invalid Snowball replication status")) + } + }) + .transpose()? + .unwrap_or(false); + if replica { + replace_extract_header( + &mut canonical_headers, + AMZ_BUCKET_REPLICATION_STATUS, + ReplicationStatusType::Replica.as_str(), + )?; + } + + validate_extract_pax_header_budget(&canonical_headers)?; + + for (name, value) in &canonical_headers { + let value = value + .to_str() + .map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball PAX metadata value"))?; + preserve_unclassified_user_metadata(metadata, name.as_str(), value); + } + + let mut authorization_headers = HeaderMap::new(); + for name in [ + AMZ_STORAGE_CLASS, + "x-amz-tagging", + AMZ_OBJECT_LOCK_MODE_LOWER, + AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, + AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, + ] { + if let Some(value) = canonical_headers.get(name) { + try_insert_extract_header(&mut authorization_headers, http::HeaderName::from_static(name), value.clone())?; + } + } + + let mut metadata_headers = canonical_headers; + metadata_headers.remove("x-amz-tagging"); + metadata_headers.remove(AMZ_OBJECT_LOCK_MODE_LOWER); + metadata_headers.remove(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER); + metadata_headers.remove(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER); + metadata_headers.remove(AMZ_BUCKET_REPLICATION_STATUS); + + if let Some(tagging) = tagging { + metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tagging); + } + + extract_metadata_from_mime_with_object_name(&metadata_headers, metadata, false, Some(object_name)); + if replica { + metadata.retain(|key, _| !key.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS)); + metadata.insert( + AMZ_BUCKET_REPLICATION_STATUS.to_string(), + ReplicationStatusType::Replica.as_str().to_string(), + ); + opts.set_replica_status(ReplicationStatusType::Replica); + } + if let Some(object_lock_metadata) = build_put_like_object_lock_metadata( + bucket, + object_lock_config_state, + object_lock_legal_hold_status.clone(), + object_lock_mode.clone(), + object_lock_retain_until_date.clone(), + )? { + metadata.extend(object_lock_metadata); + } + let expanded_metadata_bytes = extract_pax_metadata_delta_bytes(&baseline_metadata, metadata)?; + + Ok(ExtractEntryPaxAuthorization { + headers: authorization_headers, + version_id: normalized_version.map(|version| version.authorization_id), + object_lock_legal_hold_status, + object_lock_mode, + object_lock_retain_until_date, + expanded_metadata_bytes, + }) } async fn count_extract_entry_pax_metadata( @@ -643,138 +1763,12 @@ async fn apply_extract_entry_pax_extensions( where R: AsyncRead + Send + Unpin + 'static, { - let Some(extensions) = entry - .pax_extensions() + let mut overrides = ExtractPaxOverrides::default(); + overlay_extract_pax_extensions(entry, &mut overrides) .await - .map_err(|err| ExtractEntryError::Fatal(map_extract_archive_error(err)))? - else { - return Ok(ExtractEntryPaxAuthorization::default()); - }; - - let mut pax_headers = HeaderMap::new(); - let mut pax_version_id = None; - for ext in extensions { - let ext = ext.map_err(|err| ExtractEntryError::Fatal(map_extract_archive_error(err)))?; - let key = ext.key().map_err(|err| { - ExtractEntryError::Fatal(object_s3_error( - S3ErrorCode::InvalidArgument, - format!("Failed to process archive PAX key: {}", err), - )) - })?; - let value = ext.value().map_err(|err| { - ExtractEntryError::Fatal(object_s3_error( - S3ErrorCode::InvalidArgument, - format!("Failed to process archive PAX value: {}", err), - )) - })?; - - if let Some(meta_key) = key.strip_prefix("minio.metadata.") { - if !meta_key.is_empty() { - let name = http::HeaderName::from_bytes(meta_key.as_bytes()).map_err(|_| { - ExtractEntryError::Recoverable(s3_error!(InvalidArgument, "Invalid Snowball PAX metadata header")) - })?; - let header_value = HeaderValue::from_str(value).map_err(|_| { - ExtractEntryError::Recoverable(s3_error!(InvalidArgument, "Invalid Snowball PAX metadata value")) - })?; - preserve_unclassified_user_metadata(metadata, name.as_str(), value); - pax_headers.insert(name, header_value); - } - continue; - } - - if key == "minio.versionId" && !value.is_empty() { - if Uuid::parse_str(value).is_err() { - return Err(ExtractEntryError::Recoverable(s3_error!( - InvalidArgument, - "Invalid Snowball PAX version ID" - ))); - } - pax_version_id = Some(value.to_string()); - } - } - - let has_replica_status = pax_headers.contains_key(AMZ_BUCKET_REPLICATION_STATUS); - if let Some(value) = pax_headers.get(AMZ_BUCKET_REPLICATION_STATUS) { - let status = value - .to_str() - .map_err(|_| ExtractEntryError::Recoverable(s3_error!(InvalidArgument, "Invalid Snowball replication status")))?; - if !status.eq_ignore_ascii_case(ReplicationStatusType::Replica.as_str()) { - return Err(ExtractEntryError::Recoverable(s3_error!( - InvalidArgument, - "Invalid Snowball replication status" - ))); - } - pax_headers.insert(AMZ_BUCKET_REPLICATION_STATUS, HeaderValue::from_static("REPLICA")); - } - - let authorization_headers = pax_headers.clone(); - - if let Some(value) = pax_headers.remove("x-amz-tagging") { - let value = value - .to_str() - .map_err(|_| ExtractEntryError::Recoverable(s3_error!(InvalidArgument, "Invalid Snowball object tagging value")))?; - metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), value.to_owned()); - } - - let object_lock_mode = pax_headers - .remove(AMZ_OBJECT_LOCK_MODE_LOWER) - .map(|value| { - value - .to_str() - .map(|value| ObjectLockMode::from(value.to_string())) - .map_err(|_| ExtractEntryError::Recoverable(s3_error!(InvalidArgument, "Invalid Snowball Object Lock mode"))) - }) - .transpose()?; - let object_lock_retain_until_date = pax_headers - .remove(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER) - .map(|value| { - let value = value.to_str().map_err(|_| { - ExtractEntryError::Recoverable(s3_error!(InvalidArgument, "Invalid Snowball Object Lock retain-until date")) - })?; - OffsetDateTime::parse(value, &Rfc3339).map(Timestamp::from).map_err(|_| { - ExtractEntryError::Recoverable(s3_error!(InvalidArgument, "Invalid Snowball Object Lock retain-until date")) - }) - }) - .transpose()?; - let object_lock_legal_hold_status = pax_headers - .remove(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER) - .map(|value| { - value - .to_str() - .map(|value| ObjectLockLegalHoldStatus::from(value.to_string())) - .map_err(|_| { - ExtractEntryError::Recoverable(s3_error!(InvalidArgument, "Invalid Snowball Object Lock legal-hold status")) - }) - }) - .transpose()?; - opts.version_id = pax_version_id; - - extract_metadata_from_mime_with_object_name(&pax_headers, metadata, false, Some(object_name)); - if has_replica_status { - metadata.retain(|key, _| !key.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS)); - metadata.insert( - AMZ_BUCKET_REPLICATION_STATUS.to_string(), - ReplicationStatusType::Replica.as_str().to_string(), - ); - } - if let Some(object_lock_metadata) = build_put_like_object_lock_metadata( - bucket, - object_lock_config_state, - object_lock_legal_hold_status.clone(), - object_lock_mode.clone(), - object_lock_retain_until_date.clone(), - ) - .map_err(ExtractEntryError::Recoverable)? - { - metadata.extend(object_lock_metadata); - } - - Ok(ExtractEntryPaxAuthorization { - headers: authorization_headers, - object_lock_legal_hold_status, - object_lock_mode, - object_lock_retain_until_date, - }) + .map_err(ExtractEntryError::Recoverable)?; + apply_extract_pax_overrides(&overrides, bucket, object_name, object_lock_config_state, metadata, opts) + .map_err(ExtractEntryError::Recoverable) } #[cfg(test)] @@ -904,8 +1898,12 @@ impl DefaultObjectUsecase { async fn execute_put_object_extract_inner(&self, req: S3Request) -> S3Result> { let helper = OperationHelper::new(&req, EventName::ObjectCreatedPut, S3Operation::PutObject).suppress_event(); let request_context = helper.request_context_or_from_request(&req); + let extract_options = resolve_put_object_extract_options(&req.headers)?; + let member_request_headers = snowball_member_headers(&req.headers); let auth_method = req.method.clone(); let auth_uri = req.uri.clone(); + // Authorization retains the complete signed request context. Snowball + // transport controls are filtered only at member metadata/storage boundaries. let auth_headers = req.headers.clone(); let auth_extensions = req.extensions.clone(); let auth_credentials = req.credentials.clone(); @@ -952,7 +1950,7 @@ impl DefaultObjectUsecase { .. } = input; - let event_version_id = version_id; + let outer_version_id = version_id; let (h_algo, h_key, h_md5) = extract_ssec_params_from_headers(&req.headers)?; let sse_customer_algorithm = sse_customer_algorithm.or(h_algo); let sse_customer_key = sse_customer_key.or(h_key); @@ -1016,6 +2014,27 @@ impl DefaultObjectUsecase { u64::try_from(size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent))?, ) .await?; + match get_concurrency_manager() + .admit_put_object(size) + .await + .map_err(|_| object_s3_error(S3ErrorCode::InternalError, "foreground write admission closed"))? + { + ForegroundWriteAdmission::Disabled => {} + ForegroundWriteAdmission::Admitted(permit) => { + counter!("rustfs.put_object.foreground_admission.total", "result" => "admitted").increment(1); + // The archive admission is a preflight. Each member later uses + // its own logical size against the same foreground PUT policy. + drop(permit); + } + ForegroundWriteAdmission::Rejected => { + counter!("rustfs.put_object.foreground_admission.total", "result" => "rejected").increment(1); + return Err(object_s3_error( + S3ErrorCode::SlowDown, + "foreground write concurrency limit reached, please reduce your request rate", + )); + } + } + let mut put_request_guard = ExtractPutRequestGuard::new(); // Apply adaptive buffer sizing based on file size for optimal streaming performance. // Uses workload profile configuration (enabled by default) to select appropriate buffer size. @@ -1070,7 +2089,6 @@ impl DefaultObjectUsecase { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - let extract_options = resolve_put_object_extract_options(&req.headers)?; let extract_quota_check = if let Some(metadata_sys) = self.bucket_metadata_sys() { let quota_checker = QuotaChecker::new(metadata_sys); let check_result = @@ -1082,78 +2100,140 @@ impl DefaultObjectUsecase { let extract_quota_enabled = extract_quota_check .as_ref() .is_some_and(|result| result.quota_limit.is_some()); - let version_id = match event_version_id { - Some(v) => v.to_string(), - None => String::new(), - }; - let notify = current_notify_interface_for_context(self.context.as_deref()); let req_params = rustfs_targets::extract_params_header(&req.headers); let host = get_request_host(&req.headers); let port = get_request_port(&req.headers); let user_agent = get_request_user_agent(&req.headers); - let mut wrote_any_entry = false; let mut extracted_entry_count = 0usize; let mut resource_total_size = 0u64; let mut legacy_quota_growth = 0u64; let mut total_pax_metadata_size = 0u64; let mut total_pax_metadata_records = 0usize; + let mut total_expanded_pax_metadata = 0u64; let object_lock_config_snapshot = store.object_lock_config_snapshot(&bucket).await.map_err(ApiError::from)?; let object_lock_config_state = object_lock_config_snapshot.state(); + let commit_context = Arc::new(ExtractCommitContext { + store, + cache_adapter: self.object_data_cache(), + notify, + bucket: bucket.clone(), + quota_enabled: extract_quota_enabled, + request_context, + req_params, + host, + port, + user_agent, + wrote_any_entry: AtomicBool::new(false), + }); + let durable_quota = extract_quota_check + .as_ref() + .is_some_and(|result| result.uses_durable_reservations); + // Without ignore-errors, the legacy contract stops before attempting a + // later member after the first storage failure. Parallel commits cannot + // preserve that boundary, so concurrency requires both ignore-errors + // and an explicit max-inflight value above the serial default. Quota + // accounting can fail after storage commit, so quota-enabled imports + // also remain serial. An opted-in micro-batch is always drained; a + // fatal outcome stops later batches but cannot roll back peers that + // already committed in the current batch. + let max_inflight = select_put_object_extract_max_inflight( + put_object_extract_max_inflight(), + extract_options.ignore_errors, + extract_quota_enabled, + ); + let mut batch = Vec::with_capacity(max_inflight); + let mut batch_state = ExtractBatchState::default(); + + macro_rules! extract_try { + ($expression:expr) => { + match $expression { + Ok(value) => value, + Err(error) => { + let error: S3Error = error.into(); + flush_extract_batch(&mut batch, &mut batch_state, &commit_context, extract_options.ignore_errors).await?; + return Err(error); + } + } + }; + } while let Some(entry) = entries.next().await { let mut f = match entry { Ok(f) => f, - Err(e) => { - error!(error = %e, "Archive entry read failed"); - return Err(s3_error!(InvalidArgument, "Failed to read archive entry: {:?}", e)); + Err(error) => { + error!(error = %error, "Archive entry read failed"); + flush_extract_batch(&mut batch, &mut batch_state, &commit_context, extract_options.ignore_errors).await?; + return Err(s3_error!(InvalidArgument, "Failed to read archive entry: {:?}", error)); } }; extracted_entry_count = extracted_entry_count.saturating_add(1); - validate_put_object_extract_entry_count(extracted_entry_count, extract_limits)?; + extract_try!(validate_put_object_extract_entry_count(extracted_entry_count, extract_limits,)); let entry_size = f.effective_size(); - validate_put_object_extract_entry_size("archive member", entry_size, extract_limits)?; - resource_total_size = resource_total_size - .checked_add(entry_size) - .ok_or_else(|| s3_error!(InvalidArgument, "Archive total unpacked size overflowed while processing entries"))?; - validate_put_object_extract_total_size(resource_total_size, extract_limits)?; - count_extract_entry_pax_metadata( - &mut f, - &mut total_pax_metadata_size, - &mut total_pax_metadata_records, - extract_limits, - ) - .await - .map_err(ExtractEntryError::into_s3_error)?; + extract_try!(validate_put_object_extract_entry_size("archive member", entry_size, extract_limits,)); + resource_total_size = extract_try!(resource_total_size.checked_add(entry_size).ok_or_else(|| { + s3_error!(InvalidArgument, "Archive total unpacked size overflowed while processing entries") + })); + extract_try!(validate_put_object_extract_total_size(resource_total_size, extract_limits,)); + extract_try!( + count_extract_entry_pax_metadata( + &mut f, + &mut total_pax_metadata_size, + &mut total_pax_metadata_records, + extract_limits, + ) + .await + .map_err(ExtractEntryError::into_s3_error) + ); - let entry_type = classify_extract_entry_type(f.header().entry_type()); - if entry_type == ExtractEntryDisposition::FormatSkip { - continue; + let archive_entry_type = f.header().entry_type(); + let entry_kind = classify_extract_entry_type(archive_entry_type); + match entry_kind { + ExtractEntryKind::Skip => continue, + ExtractEntryKind::Directory | ExtractEntryKind::Object => {} } - let is_dir = entry_type == ExtractEntryDisposition::Directory; - if is_dir && extract_options.ignore_dirs { - continue; - } - let fpath = { - let path_bytes = f.path_bytes().map_err(map_extract_archive_error)?; + extract_try!(validate_extract_special_entry_size(archive_entry_type, entry_size)); + + let (fpath, is_dir) = { + let path_bytes = extract_try!(f.path_bytes().map_err(map_extract_archive_error)); let path = match strict_extract_entry_path(path_bytes.as_ref()) { Ok(path) => path, - Err(err) => { - err.ignore_or_return(extract_options.ignore_errors)?; + Err(error) => { + extract_try!(error.ignore_or_return(extract_options.ignore_errors)); continue; } }; if is_empty_extract_entry_path(path) { continue; } - normalize_extract_entry_key(path, extract_options.prefix.as_deref(), is_dir)? + let is_dir = entry_kind == ExtractEntryKind::Directory || is_legacy_null_directory(f.header(), path); + if is_dir && extract_options.ignore_dirs { + continue; + } + let fpath = match normalize_extract_entry_key(path, extract_options.prefix.as_deref(), is_dir) { + Ok(fpath) => fpath, + Err(error) => { + extract_try!(ExtractEntryError::Fatal(error).ignore_or_return(extract_options.ignore_errors)); + continue; + } + }; + (fpath, is_dir) }; - if let Err(err) = validate_extract_member_key(&fpath, extract_limits) { - err.ignore_or_return(extract_options.ignore_errors)?; + if let Err(error) = validate_extract_member_key(&fpath, extract_limits) { + extract_try!(error.ignore_or_return(extract_options.ignore_errors)); continue; } - validate_table_catalog_object_mutation(&bucket, &fpath).await?; + if batch_state.keys.contains(&fpath) + || durable_quota + || max_inflight <= 1 + || (!is_dir && entry_size > EXTRACT_SMALL_MEMBER_MAX_BYTES as u64) + { + extract_try!( + flush_extract_batch(&mut batch, &mut batch_state, &commit_context, extract_options.ignore_errors,).await + ); + } + extract_try!(validate_table_catalog_object_mutation(&bucket, &fpath).await); let mut auth_req = S3Request { input: PutObjectInput::default(), @@ -1167,22 +2247,23 @@ impl DefaultObjectUsecase { trailing_headers: auth_trailing_headers.clone(), }; { - let req_info = req_info_mut(&mut auth_req)?; + let req_info = extract_try!(req_info_mut(&mut auth_req)); req_info.bucket = Some(bucket.clone()); req_info.object = Some(fpath.clone()); req_info.version_id = None; } - let mut size = - i64::try_from(entry_size).map_err(|_| s3_error!(InvalidArgument, "Archive entry size does not fit into i64"))?; + let mut size = extract_try!( + i64::try_from(entry_size).map_err(|_| s3_error!(InvalidArgument, "Archive entry size does not fit into i64")) + ); // mtime 0 or a negative GNU base-256 value means "unset". xl.meta // also cannot represent the Unix epoch as an object mod_time, so // those cases fall back to the upload time (rustfs#4842). - let archive_entry_mod_time = extract_archive_entry_mod_time(f.header())?; + let archive_entry_mod_time = extract_try!(extract_archive_entry_mod_time(f.header())); let mut metadata = HashMap::new(); let has_explicit_object_lock_retention = object_lock_mode.is_some() || object_lock_retain_until_date.is_some(); - apply_put_request_metadata( + extract_try!(apply_put_request_metadata( &mut metadata, - &req.headers, + &member_request_headers, &fpath, cache_control.clone(), content_disposition.clone(), @@ -1193,25 +2274,27 @@ impl DefaultObjectUsecase { website_redirect_location.clone(), tagging.clone(), storage_class.clone(), - )?; - apply_bucket_default_lock_retention( + )); + extract_try!(apply_bucket_default_lock_retention( &bucket, object_lock_config_state, &mut metadata, has_explicit_object_lock_retention, - )?; - let mut opts = put_opts_with_replication_authorization( - &bucket, - &fpath, - None, - &req.headers, - metadata.clone(), - replication_authorized, - ) - .await - .map_err(ApiError::from)?; + )); + let mut opts = extract_try!( + put_opts_with_replication_authorization( + &bucket, + &fpath, + outer_version_id.clone(), + &member_request_headers, + metadata.clone(), + replication_authorized, + ) + .await + .map_err(ApiError::from) + ); if let Some(quota_check) = extract_quota_check.as_ref() { - apply_quota_admission(&mut opts, quota_check)?; + extract_try!(apply_quota_admission(&mut opts, quota_check)); } opts.expected_bucket_incarnation_id = expected_bucket_incarnation_id; opts.object_lock_config_snapshot = Some(Arc::clone(&object_lock_config_snapshot)); @@ -1226,36 +2309,48 @@ impl DefaultObjectUsecase { .await { Ok(authorization) => authorization, - Err(err) => { - err.ignore_or_return(extract_options.ignore_errors)?; + Err(error) => { + extract_try!(error.ignore_or_return(extract_options.ignore_errors)); continue; } }; - if let Some(quota_check) = extract_quota_check.as_ref() { - let next_legacy_quota_growth = legacy_quota_growth - .checked_add(extract_entry_quota_growth(entry_type, entry_size)) + total_expanded_pax_metadata = extract_try!( + total_expanded_pax_metadata + .checked_add(pax_authorization.expanded_metadata_bytes) .ok_or_else(|| { - object_s3_error(S3ErrorCode::InvalidArgument, "Archive quota growth overflowed while processing entries") - })?; - ensure_legacy_archive_size_within_quota(quota_check, next_legacy_quota_growth)?; + object_s3_error(S3ErrorCode::InvalidArgument, "Snowball expanded PAX metadata size overflowed") + }) + ); + extract_try!(validate_extract_expanded_pax_metadata_total(total_expanded_pax_metadata,)); + if let Some(quota_check) = extract_quota_check.as_ref() { + let next_legacy_quota_growth = extract_try!( + legacy_quota_growth + .checked_add(extract_entry_quota_growth( + if is_dir { ExtractEntryKind::Directory } else { entry_kind }, + entry_size, + )) + .ok_or_else(|| { + object_s3_error( + S3ErrorCode::InvalidArgument, + "Archive quota growth overflowed while processing entries", + ) + }) + ); + extract_try!(ensure_legacy_archive_size_within_quota(quota_check, next_legacy_quota_growth,)); legacy_quota_growth = next_legacy_quota_growth; } for (name, value) in &pax_authorization.headers { - auth_req.headers.insert(name.clone(), value.clone()); - } - if let Some(version_id) = opts.version_id.as_ref() { - req_info_mut(&mut auth_req)?.version_id = Some(version_id.clone()); - } - authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectAction)).await?; - if pax_authorization.object_lock_mode.is_some() || pax_authorization.object_lock_retain_until_date.is_some() { - authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectRetentionAction)).await?; - } - if pax_authorization.object_lock_legal_hold_status.is_some() { - authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectLegalHoldAction)).await?; - } - if opts.version_id.is_some() || pax_authorization.headers.contains_key(AMZ_BUCKET_REPLICATION_STATUS) { - authorize_request(&mut auth_req, Action::S3Action(S3Action::ReplicateObjectAction)).await?; + extract_try!(auth_req.headers.try_insert(name.clone(), value.clone()).map_err(|_| { + object_s3_error(S3ErrorCode::InvalidArgument, "Snowball IAM condition header capacity exceeded") + })); } + let explicit_version_id = pax_authorization.version_id.as_deref().or(outer_version_id.as_deref()); + let authorization_version_id = extract_try!( + explicit_version_id + .map(|version_id| apply_extract_version_id(version_id, &mut opts)) + .transpose() + ); + extract_try!(req_info_mut(&mut auth_req)).version_id = authorization_version_id; let effective_object_lock_legal_hold_status = pax_authorization .object_lock_legal_hold_status .clone() @@ -1269,6 +2364,28 @@ impl DefaultObjectUsecase { } else { (object_lock_mode.clone(), object_lock_retain_until_date.clone()) }; + let iam_requirements = extract_member_iam_requirements( + &metadata, + effective_object_lock_legal_hold_status.as_ref(), + effective_object_lock_mode.as_ref(), + effective_object_lock_retain_until_date.as_ref(), + explicit_version_id, + opts.delete_marker_replication_status() == ReplicationStatusType::Replica, + ); + extract_try!(authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectAction)).await); + if iam_requirements.tagging { + extract_try!(authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectTaggingAction)).await); + } + if iam_requirements.retention { + extract_try!(authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectRetentionAction)).await); + } + if iam_requirements.legal_hold { + extract_try!(authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectLegalHoldAction)).await); + } + if iam_requirements.replication { + extract_try!(authorize_request(&mut auth_req, Action::S3Action(S3Action::ReplicateObjectAction)).await); + } + drop(auth_req); if archive_entry_mod_time.is_some() { opts.mod_time = archive_entry_mod_time; } @@ -1278,69 +2395,65 @@ impl DefaultObjectUsecase { if is_dir { size = 0; } - let actual_size = size; - let should_compress = !is_dir && is_disk_compressible(&HeaderMap::new(), &fpath) && size > MIN_DISK_COMPRESSIBLE_SIZE as i64; - let mut write_plan = WritePlan::new(); - let mut hrd = if is_dir { - HashReader::from_stream(std::io::Cursor::new(Vec::new()), size, actual_size, None, None, false) - .map_err(ApiError::from)? - } else if should_compress { + if should_compress { let algorithm = CompressionAlgorithm::default(); insert_str(&mut metadata, SUFFIX_COMPRESSION, compression_metadata_value(algorithm)); insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); - - let hrd = HashReader::from_stream(f, size, actual_size, None, None, false).map_err(ApiError::from)?; write_plan = write_plan.with_compression(algorithm); - hrd - } else { - HashReader::from_stream(f, size, actual_size, None, None, false).map_err(ApiError::from)? - }; - apply_put_request_object_lock_opts( + } + extract_try!(apply_put_request_object_lock_opts( &bucket, object_lock_config_state, effective_object_lock_legal_hold_status, effective_object_lock_mode, effective_object_lock_retain_until_date, &mut opts, - )?; - if let Some(material) = sse_encryption(EncryptionRequest { - bucket: &bucket, - key: &fpath, - server_side_encryption: effective_sse.clone(), - ssekms_key_id: effective_kms_key_id.clone(), - ssekms_context: extract_ssekms_context_from_headers(&req.headers)?, - sse_customer_algorithm: sse_customer_algorithm.clone(), - sse_customer_key: sse_customer_key.clone(), - sse_customer_key_md5: sse_customer_key_md5.clone(), - content_size: actual_size, - principal: extract_principal.as_ref(), - }) - .await? - { + )); + if let Some(material) = extract_try!( + sse_encryption(EncryptionRequest { + bucket: &bucket, + key: &fpath, + server_side_encryption: effective_sse.clone(), + ssekms_key_id: effective_kms_key_id.clone(), + ssekms_context: extract_try!(extract_ssekms_context_from_headers(&req.headers)), + sse_customer_algorithm: sse_customer_algorithm.clone(), + sse_customer_key: sse_customer_key.clone(), + sse_customer_key_md5: sse_customer_key_md5.clone(), + content_size: actual_size, + principal: extract_principal.as_ref(), + }) + .await + ) { effective_sse = Some(material.server_side_encryption.clone()); effective_kms_key_id = material.kms_key_id.clone(); - write_plan = write_plan.with_encryption(material.write_encryption(None)); - - let encryption_metadata = encryption_material_to_metadata(&material)?; + let encryption_metadata = extract_try!(encryption_material_to_metadata(&material)); metadata.extend(encryption_metadata.clone()); opts.user_defined.extend(encryption_metadata); } - hrd = write_plan.apply(hrd, actual_size).map_err(ApiError::from)?; - let (hrd, member_read_failed) = track_extract_member_read_errors(hrd).map_err(ApiError::from)?; opts.user_defined.extend(metadata); - // Each extracted member is an independent user write and joins - // bucket replication like a regular PUT (MinIO PutObjectExtract - // parity). One immutable decision drives both the pending metadata - // and the post-commit schedule below, same contract as the PUT path - // (https://github.com/rustfs/backlog/issues/1320); inbound replica - // writes are declined inside `must_replicate_object`. - let dsc = must_replicate_object( + // Reserve the global member lifecycle before constructing the + // potentially large replication decision. If this archive already + // owns a batch, drain it before waiting so requests cannot deadlock + // while each retains lifecycle capacity needed by its own batch. + let member_permit = extract_try!( + acquire_extract_member_lifecycle_permit( + &mut batch, + &mut batch_state, + &commit_context, + extract_options.ignore_errors, + ) + .await + ); + + // One immutable decision drives both the pending metadata and the + // post-commit schedule, matching the regular PUT contract. + let replication = must_replicate_object( &bucket, &fpath, &opts.user_defined, @@ -1349,95 +2462,132 @@ impl DefaultObjectUsecase { opts.clone(), ) .await; - if dsc.replicate_any() { + if replication.replicate_any() { insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string()); insert_str( &mut opts.user_defined, SUFFIX_REPLICATION_STATUS, - dsc.pending_status().unwrap_or_default(), + replication.pending_status().unwrap_or_default(), ); } - let mut reader = PutObjReader::new(hrd); - let cache_adapter = self.object_data_cache(); - let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &fpath).await; + // Never wait while retaining a fully constructed replication + // decision. The lifecycle gate bounds builders, and this byte gate + // accounts every serial or staged member until storage returns. + let manager = get_concurrency_manager(); - let (obj_info, backfilled_old_current_size) = match store - .put_object_with_old_current_size(&bucket, &fpath, &mut reader, &opts) - .await - { - Ok(result) => result, - Err(e) => { - if should_ignore_extract_member_write_error(extract_options.ignore_errors, &member_read_failed) { - warn!(error = %e, "Archive object write skipped due to ignore-errors"); - continue; - } - return Err(ApiError::from(e).into()); - } - }; - let extract_versioned = BucketVersioningSys::prefix_enabled(&bucket, &fpath).await; - let post_commit_error = match quota_accounting_object_size(&obj_info, extract_quota_enabled) { - Ok(committed_size) => { - match previous_current_size_from_backfill(backfilled_old_current_size) { - Some(previous_current_size) => { - if extract_versioned { - record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await; - } else { - record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await; - } - } - None => { - record_bucket_object_write_unknown_previous_memory(&bucket, committed_size, extract_versioned).await; - } - } - None - } - Err(err) => Some(err), - }; - let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &fpath).await; + let member_size = extract_try!( + usize::try_from(size) + .map_err(|_| object_s3_error(S3ErrorCode::InvalidArgument, "Snowball member size does not fit into usize")) + ); + let staging_weight = extract_try!(checked_extract_member_staging_weight(&fpath, member_size, &opts, &replication,)); + let action = batch_state.action(&fpath, member_size, staging_weight, max_inflight, durable_quota); - // Reuse the per-entry pre-commit decision (see `dsc` above) so the - // persisted pending marker and the schedule always agree. - if dsc.replicate_any() { - schedule_object_replication(obj_info.clone(), store.clone(), dsc).await; + if matches!(action, ExtractBatchAction::FlushThenStage | ExtractBatchAction::SerialBarrier) { + extract_try!( + flush_extract_batch(&mut batch, &mut batch_state, &commit_context, extract_options.ignore_errors,).await + ); } - if !wrote_any_entry { - rustfs_scanner::record_dirty_usage_bucket(&bucket); - wrote_any_entry = true; + let mut staging_permit = try_acquire_extract_staging_permit(manager, staging_weight); + if staging_permit.is_err() && !batch.is_empty() { + extract_try!( + flush_extract_batch(&mut batch, &mut batch_state, &commit_context, extract_options.ignore_errors,).await + ); + staging_permit = try_acquire_extract_staging_permit(manager, staging_weight); + } + let staging_permit = match staging_permit { + Ok(permit) => permit, + Err(error) if extract_options.ignore_errors => { + warn!(error = %error, "Archive object staging skipped due to ignore-errors"); + continue; + } + Err(error) => extract_try!(Err::(error)), + }; + + if action == ExtractBatchAction::SerialBarrier { + // Large and durable-quota members retain the legacy streaming + // path. Await the owner inline so the producer never advances + // the TAR while its Entry is alive. The complete commit and + // notification aggregation move into that owner before ECStore + // starts, so caller cancellation cannot interrupt a mutation + // after its quorum commit point. + let member_context = commit_context.clone(); + let owner_context = commit_context.clone(); + if is_dir { + drop(f); + extract_try!( + run_extract_outcomes_owner(owner_context, extract_options.ignore_errors, async move { + vec![ + commit_extract_member( + member_context, + ExtractPreparedMember { + archive_seq: extracted_entry_count, + key: fpath, + size, + actual_size, + body: std::io::Cursor::new(Vec::new()), + write_plan, + opts, + replication, + staging_permit, + member_permit, + }, + ) + .await, + ] + }) + .await + ); + } else { + extract_try!( + run_extract_outcomes_owner(owner_context, extract_options.ignore_errors, async move { + vec![ + commit_extract_member( + member_context, + ExtractPreparedMember { + archive_seq: extracted_entry_count, + key: fpath, + size, + actual_size, + body: f, + write_plan, + opts, + replication, + staging_permit, + member_permit, + }, + ) + .await, + ] + }) + .await + ); + } + continue; } - let _manager = get_concurrency_manager(); - let _fpath_clone = fpath.clone(); - let _bucket_clone = bucket.clone(); - let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)); - - let output = PutObjectOutput { - e_tag, - ..Default::default() + let body = if is_dir { + drop(f); + ExtractStagedBody::empty() + } else { + extract_try!(stage_extract_member_body(&mut f, member_size).await) }; - - let event_args = rustfs_notify::EventArgs { - event_name: put_event_name_for_post_object(false), - bucket_name: bucket.clone(), - object: convert_ecstore_object_info(obj_info.clone()), - req_params: req_params.clone(), - resp_elements: build_event_resp_elements(&S3Response::new(output.clone()), &request_context.request_id), - version_id: version_id.clone(), - host: host.clone(), - port, - user_agent: user_agent.clone(), - }; - - let notify = notify.clone(); - spawn_background_with_context(Some(request_context.clone()), async move { - notify.notify(event_args).await; + batch_state.record(&fpath, staging_weight); + batch.push(ExtractPreparedMember { + archive_seq: extracted_entry_count, + key: fpath, + size, + actual_size, + body, + write_plan, + opts, + replication, + staging_permit, + member_permit, }); - - if let Some(err) = post_commit_error { - return Err(err); - } } + extract_try!(flush_extract_batch(&mut batch, &mut batch_state, &commit_context, extract_options.ignore_errors,).await); let mut checksums = PutObjectChecksums { crc32: input.checksum_crc32, @@ -1488,6 +2638,7 @@ impl DefaultObjectUsecase { }; let result = Ok(S3Response::new(output)); let _ = helper.complete(&result); + put_request_guard.finish_ok(); result } } @@ -1500,6 +2651,524 @@ mod tests { use tokio::io::AsyncReadExt; use tokio_tar::{Builder, EntryType, Header}; + struct RecordingNotify { + events: tokio::sync::mpsc::UnboundedSender, + } + + #[async_trait::async_trait] + impl crate::runtime_sources::NotifyInterface for RecordingNotify { + async fn notify(&self, args: rustfs_notify::EventArgs) { + let _ = self.events.send(args.version_id); + } + + async fn add_event_specific_rules( + &self, + _bucket_name: &str, + _region: &str, + _event_rules: &[(Vec, String, String, Vec)], + ) -> Result<(), rustfs_notify::NotificationError> { + Ok(()) + } + + async fn clear_bucket_notification_rules(&self, _bucket_name: &str) -> Result<(), rustfs_notify::NotificationError> { + Ok(()) + } + } + + async fn recording_extract_commit_context( + notify: Arc, + ) -> Arc { + Arc::new(ExtractCommitContext { + store: crate::app::gating_test_env::shared_gating_ecstore().await, + cache_adapter: ObjectDataCacheAdapter::disabled_arc(), + notify, + bucket: "snowball-owner-test".to_string(), + quota_enabled: false, + request_context: request_context::RequestContext::from_headers(&HeaderMap::new()), + req_params: hashbrown::HashMap::new(), + host: String::new(), + port: 0, + user_agent: String::new(), + wrote_any_entry: AtomicBool::new(false), + }) + } + + #[test] + fn snowball_max_inflight_has_a_serial_compatibility_floor_and_bounded_ceiling() { + assert_eq!(EXTRACT_DEFAULT_MAX_INFLIGHT, 1); + assert_eq!(normalize_put_object_extract_max_inflight(0), 1); + assert_eq!(normalize_put_object_extract_max_inflight(1), 1); + assert_eq!(normalize_put_object_extract_max_inflight(usize::MAX), EXTRACT_BATCH_MAX_MEMBERS); + assert_eq!( + select_put_object_extract_max_inflight(EXTRACT_BATCH_MAX_MEMBERS, false, false), + 1, + "requests that stop on write errors must preserve serial member semantics" + ); + assert_eq!( + select_put_object_extract_max_inflight(EXTRACT_BATCH_MAX_MEMBERS, true, false), + EXTRACT_BATCH_MAX_MEMBERS, + "ignore-errors requests may use bounded parallel member commits" + ); + assert_eq!( + select_put_object_extract_max_inflight(EXTRACT_BATCH_MAX_MEMBERS, true, true), + 1, + "quota accounting can fail after storage commit and must remain serial" + ); + } + + #[test] + fn snowball_batch_state_flushes_on_duplicates_limits_and_serial_barriers() { + let mut state = ExtractBatchState::default(); + assert_eq!( + state.action("first", 4096, 8192, EXTRACT_BATCH_MAX_MEMBERS, false), + ExtractBatchAction::Stage + ); + state.record("first", 8192); + assert_eq!( + state.action("first", 4096, 8192, EXTRACT_BATCH_MAX_MEMBERS, false), + ExtractBatchAction::FlushThenStage + ); + assert_eq!( + state.action("large", EXTRACT_SMALL_MEMBER_MAX_BYTES + 1, 8192, EXTRACT_BATCH_MAX_MEMBERS, false,), + ExtractBatchAction::SerialBarrier + ); + assert_eq!( + state.action("quota", 4096, 8192, EXTRACT_BATCH_MAX_MEMBERS, true), + ExtractBatchAction::SerialBarrier + ); + assert_eq!(state.action("compat", 4096, 8192, 1, false), ExtractBatchAction::SerialBarrier); + assert_eq!( + state.action( + "exact-small-boundary", + EXTRACT_SMALL_MEMBER_MAX_BYTES, + 8192, + EXTRACT_BATCH_MAX_MEMBERS, + false, + ), + ExtractBatchAction::Stage + ); + assert_eq!( + state.action( + "oversized-context", + 4096, + EXTRACT_BATCH_MAX_STAGING_BYTES + 1, + EXTRACT_BATCH_MAX_MEMBERS, + false, + ), + ExtractBatchAction::SerialBarrier + ); + + state.clear(); + state.staging_bytes = EXTRACT_BATCH_MAX_STAGING_BYTES - 1024; + assert_eq!( + state.action("exact-memory", 4096, 1024, EXTRACT_BATCH_MAX_MEMBERS, false), + ExtractBatchAction::Stage + ); + assert_eq!( + state.action("memory", 4096, 2048, EXTRACT_BATCH_MAX_MEMBERS, false), + ExtractBatchAction::FlushThenStage + ); + + state.clear(); + for index in 0..EXTRACT_BATCH_MAX_MEMBERS { + state.record(&format!("key-{index}"), 1); + } + assert_eq!( + state.action("overflow", 1, 1, EXTRACT_BATCH_MAX_MEMBERS, false), + ExtractBatchAction::FlushThenStage + ); + } + + #[test] + fn snowball_staging_weight_includes_body_capacity_path_and_frozen_metadata() { + let mut opts = ObjectOptions::default(); + opts.user_defined.insert("x-amz-meta-one".to_string(), "value".to_string()); + opts.eval_metadata = Some(HashMap::from([("auth-view".to_string(), "retained".to_string())])); + let path = "prefix/object"; + let weight = checked_extract_member_staging_weight(path, 1, &opts, &ReplicateDecision::new()) + .expect("valid metadata must have a weight"); + assert_eq!( + weight, + EXTRACT_SMALL_MEMBER_MAX_BYTES + + EXTRACT_MEMBER_CONTEXT_OVERHEAD_BYTES + + (2 * path.len()) + + "x-amz-meta-one".len() + + "value".len() + + EXTRACT_METADATA_ENTRY_OVERHEAD_BYTES + + "auth-view".len() + + "retained".len() + + EXTRACT_METADATA_ENTRY_OVERHEAD_BYTES + ); + let empty_weight = checked_extract_member_staging_weight(path, 0, &ObjectOptions::default(), &ReplicateDecision::new()) + .expect("empty member must have a weight"); + assert_eq!(empty_weight, EXTRACT_MEMBER_CONTEXT_OVERHEAD_BYTES + (2 * path.len())); + } + + #[test] + fn snowball_staging_weight_accounts_replication_target_capacity() { + let path = "replicated/object"; + let opts = ObjectOptions::default(); + let baseline = checked_extract_member_staging_weight(path, 0, &opts, &ReplicateDecision::new()) + .expect("empty replication decision must have a weight"); + + let mut target_name = String::with_capacity(128); + target_name.push_str("target"); + let mut replication = ReplicateDecision::new(); + replication.targets_map.insert(target_name, Default::default()); + let map_allocation = + checked_extract_hash_map_allocation(&replication.targets_map).expect("target map allocation must fit"); + let retained_strings = { + let (name, target) = replication.targets_map.iter_mut().next().expect("target must exist"); + let mut arn = String::with_capacity(EXTRACT_BATCH_MAX_STAGING_BYTES); + arn.push('a'); + target.arn = arn; + let mut id = String::with_capacity(256); + id.push_str("id"); + target.id = id; + name.capacity() + target.arn.capacity() + target.id.capacity() + }; + let weight = checked_extract_member_staging_weight(path, 0, &opts, &replication) + .expect("replication decision must have a bounded weight"); + assert_eq!(weight, baseline + retained_strings + map_allocation); + assert_eq!( + ExtractBatchState::default().action("replicated/object", 0, weight, EXTRACT_BATCH_MAX_MEMBERS, false), + ExtractBatchAction::SerialBarrier, + "a retained target capacity above the micro-batch budget must remain serial" + ); + } + + #[test] + fn snowball_serial_members_consume_the_global_staging_budget() { + let manager = ConcurrencyManager::new(); + let serial_weight = EXTRACT_BATCH_MAX_STAGING_BYTES + 1; + assert_eq!( + ExtractBatchState::default().action("large-context", 0, serial_weight, EXTRACT_BATCH_MAX_MEMBERS, false), + ExtractBatchAction::SerialBarrier + ); + + let first = try_acquire_extract_staging_permit(&manager, serial_weight) + .expect("the first serial member must reserve its retained context"); + let error = try_acquire_extract_staging_permit(&manager, serial_weight) + .expect_err("a second serial member must not exceed the global staging budget"); + assert_eq!(error.code(), &S3ErrorCode::SlowDown); + + drop(first); + assert!( + try_acquire_extract_staging_permit(&manager, serial_weight).is_ok(), + "serial staging capacity must be reusable after storage releases it" + ); + } + + #[test] + fn snowball_staging_admission_rejects_without_retaining_waiters() { + let manager = ConcurrencyManager::new(); + let full_budget = manager + .try_acquire_snowball_staging_bytes( + u32::try_from(SNOWBALL_STAGING_BYTES_LIMIT).expect("the staging budget must fit into u32"), + ) + .expect("the exact global staging budget must be available"); + let error = + try_acquire_extract_staging_permit(&manager, 1).expect_err("a saturated staging budget must reject immediately"); + assert_eq!(error.code(), &S3ErrorCode::SlowDown); + drop(full_budget); + + let error = try_acquire_extract_staging_permit(&manager, SNOWBALL_STAGING_BYTES_LIMIT + 1) + .expect_err("a single retained context larger than the global budget must reject"); + assert_eq!(error.code(), &S3ErrorCode::SlowDown); + } + + #[tokio::test] + async fn snowball_staging_rejects_truncated_and_oversized_member_bodies() { + let mut truncated = std::io::Cursor::new(b"ab".to_vec()); + let error = stage_extract_member_body(&mut truncated, 3) + .await + .err() + .expect("a truncated TAR member must fail staging"); + assert_eq!(error.code(), &S3ErrorCode::IncompleteBody); + + let mut oversized = std::io::Cursor::new(b"abc".to_vec()); + let error = stage_extract_member_body(&mut oversized, 2) + .await + .err() + .expect("a TAR member longer than its declared size must fail staging"); + assert_eq!(error.code(), &S3ErrorCode::UnexpectedContent); + } + + #[tokio::test] + async fn snowball_staging_consumes_each_tar_entry_before_advancing() { + let mut builder = Builder::new(Vec::new()); + for (path, payload) in [("first", b"first-body".as_slice()), ("second", b"second-body".as_slice())] { + let mut header = Header::new_gnu(); + header.set_size(payload.len() as u64); + header.set_cksum(); + builder + .append_data(&mut header, path, std::io::Cursor::new(payload)) + .await + .expect("TAR member must append"); + } + let archive_bytes = builder.into_inner().await.expect("TAR fixture must finalize"); + let mut archive = Archive::new(std::io::Cursor::new(archive_bytes)); + let mut entries = archive.entries().expect("TAR entries must open"); + + let mut first = entries + .next() + .await + .expect("first entry must exist") + .expect("first entry must parse"); + let mut staged = stage_extract_member_body(&mut first, "first-body".len()) + .await + .expect("first body must stage"); + let mut staged_bytes = Vec::new(); + staged.read_to_end(&mut staged_bytes).await.expect("staged body must read"); + assert_eq!(staged_bytes, b"first-body"); + drop(first); + + let mut second = entries + .next() + .await + .expect("second entry must exist") + .expect("second entry must parse"); + let mut second_bytes = Vec::new(); + second.read_to_end(&mut second_bytes).await.expect("second body must read"); + assert_eq!(second_bytes, b"second-body"); + drop(second); + drop(entries); + assert!(archive.into_inner().is_ok(), "no TAR entry may escape the sequential producer"); + } + + #[test] + fn snowball_batch_error_selection_uses_archive_order() { + let outcomes = vec![ + ExtractCommitOutcome { + archive_seq: 2, + event: None, + error: Some(ExtractCommitError::Fatal(object_s3_error(S3ErrorCode::InvalidArgument, "later"))), + }, + ExtractCommitOutcome { + archive_seq: 1, + event: None, + error: Some(ExtractCommitError::Fatal(object_s3_error(S3ErrorCode::NoSuchKey, "earlier"))), + }, + ]; + let (_, error) = ordered_extract_outcomes(outcomes, false); + let error = error.expect("the earliest archive error must be returned"); + assert_eq!(error.code(), &S3ErrorCode::NoSuchKey); + } + + #[test] + fn snowball_batch_notifications_follow_archive_order() { + let outcomes = vec![ + ExtractCommitOutcome { + archive_seq: 2, + event: Some(EventArgsBuilder::default().version_id("second").build()), + error: None, + }, + ExtractCommitOutcome { + archive_seq: 1, + event: Some(EventArgsBuilder::default().version_id("first").build()), + error: None, + }, + ]; + let (events, error) = ordered_extract_outcomes(outcomes, false); + assert!(error.is_none(), "successful outcomes must be accepted"); + assert_eq!( + events.iter().map(|event| event.version_id.as_str()).collect::>(), + ["first", "second"] + ); + } + + #[tokio::test] + async fn snowball_notification_dispatch_detaches_slow_delivery() { + let started = Arc::new(tokio::sync::Notify::new()); + spawn_extract_notification_batch(None, vec![EventArgsBuilder::default().version_id("slow-target").build()], { + let started = started.clone(); + move |_event| { + let started = started.clone(); + async move { + started.notify_one(); + std::future::pending::<()>().await; + } + } + }); + + tokio::time::timeout(std::time::Duration::from_secs(1), started.notified()) + .await + .expect("detached notification delivery must start without blocking the caller"); + } + + #[tokio::test] + #[serial_test::serial] + async fn snowball_commit_owner_survives_request_cancellation() { + let entered = Arc::new(tokio::sync::Barrier::new(2)); + let release = Arc::new(tokio::sync::Barrier::new(2)); + let (events, mut received_events) = tokio::sync::mpsc::unbounded_channel(); + let context = recording_extract_commit_context(Arc::new(RecordingNotify { events })).await; + let manager = ConcurrencyManager::new(); + let owner_permit = manager + .acquire_snowball_member_commit() + .await + .expect("owner lifecycle admission must remain open"); + let mut other_permits = Vec::with_capacity(SNOWBALL_MEMBER_COMMIT_LIMIT - 1); + for _ in 1..SNOWBALL_MEMBER_COMMIT_LIMIT { + other_permits.push( + manager + .acquire_snowball_member_commit() + .await + .expect("remaining lifecycle admission must remain open"), + ); + } + + let request = spawn_traced_join({ + let entered = entered.clone(); + let release = release.clone(); + async move { + run_extract_outcomes_owner(context, false, async move { + let owner_permit = owner_permit; + entered.wait().await; + release.wait().await; + let outcome = ExtractCommitOutcome { + archive_seq: 1, + event: Some(EventArgsBuilder::default().version_id("cancelled-owner-event").build()), + error: None, + }; + drop(owner_permit); + vec![outcome] + }) + .await + } + }); + tokio::time::timeout(std::time::Duration::from_secs(1), entered.wait()) + .await + .expect("commit owner must start before the request waits for storage"); + request.abort(); + let join_error = request.await.expect_err("request task must be cancelled"); + assert!(join_error.is_cancelled()); + assert!( + manager.try_acquire_snowball_member_commit().is_none(), + "a cancelled request must not admit a 33rd member while its detached tail is running" + ); + release.wait().await; + let replacement = tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + if let Some(permit) = manager.try_acquire_snowball_member_commit() { + break permit; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("commit owner must finish after request cancellation"); + let event = tokio::time::timeout(std::time::Duration::from_secs(1), received_events.recv()) + .await + .expect("the detached owner must dispatch its committed event") + .expect("the notification recorder must remain open"); + assert_eq!(event, "cancelled-owner-event"); + assert!( + matches!( + received_events.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty | tokio::sync::mpsc::error::TryRecvError::Disconnected) + ), + "the detached owner must dispatch the committed event exactly once" + ); + drop((replacement, other_permits)); + } + + #[test] + fn snowball_ignore_errors_skips_storage_write_failure_and_keeps_later_success() { + let outcomes = vec![ + ExtractCommitOutcome { + archive_seq: 1, + event: None, + error: Some(ExtractCommitError::StorageWrite(object_s3_error( + S3ErrorCode::InternalError, + "injected write failure", + ))), + }, + ExtractCommitOutcome { + archive_seq: 2, + event: Some(EventArgsBuilder::default().version_id("later-success").build()), + error: None, + }, + ]; + + let (events, error) = ordered_extract_outcomes(outcomes, true); + assert!(error.is_none(), "ignore-errors must skip a storage-only write failure"); + assert_eq!(events.len(), 1); + assert_eq!(events[0].version_id, "later-success"); + + let slowdown = ExtractCommitError::StorageWrite(object_s3_error(S3ErrorCode::SlowDown, "injected admission rejection")); + assert!( + slowdown.into_unignored(true).is_none(), + "ignore-errors must treat member admission rejection as a skipped write" + ); + let slowdown = ExtractCommitError::StorageWrite(object_s3_error(S3ErrorCode::SlowDown, "injected admission rejection")); + assert_eq!( + slowdown + .into_unignored(false) + .expect("non-ignore requests must return admission rejection") + .code(), + &S3ErrorCode::SlowDown + ); + } + + #[test] + fn snowball_ignore_errors_never_skips_reader_or_other_fatal_failures() { + let outcomes = vec![ExtractCommitOutcome { + archive_seq: 1, + event: None, + error: Some(ExtractCommitError::Fatal(object_s3_error( + S3ErrorCode::IncompleteBody, + "injected reader failure", + ))), + }]; + + let (_, error) = ordered_extract_outcomes(outcomes, true); + let error = error.expect("ignore-errors must not hide reader, codec, length, resource, or post-commit failures"); + assert_eq!(error.code(), &S3ErrorCode::IncompleteBody); + } + + #[tokio::test] + async fn snowball_batch_runner_polls_commits_concurrently_and_drains_every_outcome() { + let completed = Arc::new(AtomicUsize::new(0)); + let active = Arc::new(AtomicUsize::new(0)); + let max_active = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(tokio::sync::Barrier::new(3)); + let outcomes = tokio::time::timeout( + std::time::Duration::from_secs(1), + run_extract_commits(1..=3, { + let active = active.clone(); + let max_active = max_active.clone(); + let barrier = barrier.clone(); + let completed = completed.clone(); + move |archive_seq| { + let active = active.clone(); + let max_active = max_active.clone(); + let barrier = barrier.clone(); + let completed = completed.clone(); + async move { + let current = active.fetch_add(1, Ordering::AcqRel) + 1; + max_active.fetch_max(current, Ordering::AcqRel); + barrier.wait().await; + active.fetch_sub(1, Ordering::AcqRel); + completed.fetch_add(1, Ordering::Relaxed); + ExtractCommitOutcome { + archive_seq, + event: None, + error: (archive_seq == 2) + .then(|| ExtractCommitError::Fatal(object_s3_error(S3ErrorCode::InvalidArgument, "injected"))), + } + } + } + }), + ) + .await + .expect("the production batch runner must poll all commits concurrently"); + assert_eq!(outcomes.len(), 3); + assert_eq!(completed.load(Ordering::Relaxed), 3); + assert_eq!(max_active.load(Ordering::Acquire), 3); + assert_eq!(outcomes.iter().filter(|outcome| outcome.error.is_some()).count(), 1); + } + #[test] fn archive_format_uses_only_the_ambiguous_zlib_extension_as_a_fallback() { assert_eq!( @@ -1828,7 +3497,7 @@ mod tests { ( authorization.object_lock_mode.is_some() || authorization.object_lock_retain_until_date.is_some(), authorization.object_lock_legal_hold_status.is_some(), - opts.version_id.is_some() || authorization.headers.contains_key(AMZ_BUCKET_REPLICATION_STATUS), + opts.version_id.is_some() || opts.delete_marker_replication_status() == ReplicationStatusType::Replica, ), expected, "{case} must request only its own additional authorization" @@ -1838,15 +3507,21 @@ mod tests { assert!(authorization.headers.contains_key(AMZ_OBJECT_LOCK_MODE_LOWER)); assert!(authorization.headers.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER)); } - "legal-hold" => assert!(authorization.headers.contains_key(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER)), + "legal-hold" => { + assert_eq!( + authorization + .headers + .get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER) + .and_then(|value| value.to_str().ok()), + Some("ON") + ); + assert!(authorization.object_lock_legal_hold_status.is_some()); + } "version-id" => assert_eq!(opts.version_id.as_deref(), Some(Uuid::nil().to_string().as_str())), - "replication-status" => assert_eq!( - authorization - .headers - .get(AMZ_BUCKET_REPLICATION_STATUS) - .and_then(|value| value.to_str().ok()), - Some("REPLICA") - ), + "replication-status" => { + assert!(!authorization.headers.contains_key(AMZ_BUCKET_REPLICATION_STATUS)); + assert_eq!(opts.delete_marker_replication_status(), ReplicationStatusType::Replica); + } _ => unreachable!(), } } @@ -1866,6 +3541,7 @@ mod tests { pax_record("minio.metadata.x-amz-replication-status", b"INVALID"), ), ("invalid-version-id", pax_record("minio.versionId", b"not-a-uuid")), + ("non-exact-null-version-id", pax_record("minio.versionId", b"NULL")), ]; let state = metadata_sys::ObjectLockConfigState::Configured { config: ObjectLockConfiguration { @@ -1966,7 +3642,43 @@ mod tests { assert!(authorization.object_lock_retain_until_date.is_some()); assert!(authorization.object_lock_legal_hold_status.is_some()); assert!(opts.version_id.is_some()); - assert!(authorization.headers.contains_key(AMZ_BUCKET_REPLICATION_STATUS)); + assert!(!authorization.headers.contains_key(AMZ_BUCKET_REPLICATION_STATUS)); + assert_eq!(opts.delete_marker_replication_status(), ReplicationStatusType::Replica); + } + + #[test] + fn snowball_pax_retention_auth_view_normalizes_offset_to_same_instant() { + let mut overrides = ExtractPaxOverrides::default(); + overrides + .overlay_record("minio.metadata.x-amz-object-lock-mode", "GOVERNANCE") + .expect("retention mode should parse"); + overrides + .overlay_record("minio.metadata.x-amz-object-lock-retain-until-date", "2099-01-01T00:00:00-02:00") + .expect("offset retention date should parse"); + let state = metadata_sys::ObjectLockConfigState::Configured { + config: ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: None, + }, + updated_at: OffsetDateTime::now_utc(), + }; + let mut metadata = HashMap::new(); + let authorization = + apply_extract_pax_overrides(&overrides, "bucket", "object", &state, &mut metadata, &mut ObjectOptions::default()) + .expect("valid offset retention should apply"); + + let auth_value = authorization + .headers + .get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER) + .and_then(|value| value.to_str().ok()) + .expect("IAM view should contain canonical retention date"); + assert_eq!(auth_value, "2099-01-01T02:00:00Z"); + let stored_value = metadata + .get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER) + .expect("retention date should be persisted"); + let auth_instant = OffsetDateTime::parse(auth_value, &Rfc3339).expect("canonical auth time should parse"); + let stored_instant = OffsetDateTime::parse(stored_value, &Rfc3339).expect("stored retention time should parse"); + assert_eq!(auth_instant, stored_instant); } #[tokio::test] @@ -2036,10 +3748,326 @@ mod tests { .unwrap_err() .into_s3_error(); - assert_eq!(err.code(), &S3ErrorCode::MalformedXML); + assert_eq!(err.code(), &S3ErrorCode::InvalidArgument); assert!(!metadata.contains_key(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER)); } + #[test] + fn snowball_entry_type_allowlist_matches_minio_object_semantics() { + for entry_type in [ + EntryType::Regular, + EntryType::new(b'\0'), + EntryType::Char, + EntryType::Block, + EntryType::Fifo, + EntryType::GNUSparse, + ] { + assert_eq!(classify_extract_entry_type(entry_type), ExtractEntryKind::Object); + } + assert_eq!(classify_extract_entry_type(EntryType::Directory), ExtractEntryKind::Directory); + for entry_type in [ + EntryType::Link, + EntryType::Symlink, + EntryType::Continuous, + EntryType::XGlobalHeader, + EntryType::XHeader, + EntryType::SolarisXHeader, + EntryType::Other(b'9'), + ] { + assert_eq!(classify_extract_entry_type(entry_type), ExtractEntryKind::Skip); + } + } + + #[test] + fn snowball_legacy_null_regular_directory_uses_effective_path_suffix() { + let mut header = Header::new_old(); + header.as_mut_bytes()[TAR_TYPEFLAG_OFFSET] = b'\0'; + + assert!(is_legacy_null_directory(&header, "directory/")); + assert!(!is_legacy_null_directory(&header, "object")); + + header.set_entry_type(EntryType::Regular); + assert!(!is_legacy_null_directory(&header, "directory/")); + } + + #[test] + fn snowball_special_members_require_zero_declared_size() { + for entry_type in [EntryType::Char, EntryType::Block, EntryType::Fifo] { + validate_extract_special_entry_size(entry_type, 0).expect("zero-sized special member should be accepted"); + assert!(validate_extract_special_entry_size(entry_type, 1).is_err()); + } + validate_extract_special_entry_size(EntryType::GNUSparse, 1).expect("GNU sparse members retain payload semantics"); + } + + #[test] + fn snowball_header_views_preserve_auth_context_and_filter_member_storage() { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SNOWBALL_EXTRACT, HeaderValue::from_static("true")); + headers.insert(AMZ_SNOWBALL_EXTRACT_COMPAT, HeaderValue::from_static("true")); + headers.insert(AMZ_MINIO_SNOWBALL_PREFIX, HeaderValue::from_static("prefix")); + headers.insert("x-amz-meta-acme-snowball-ignore-dirs", HeaderValue::from_static("true")); + headers.insert("snowball-auto-extract", HeaderValue::from_static("true")); + headers.insert("rustfs-snowball-ignore-errors", HeaderValue::from_static("true")); + headers.insert("x-amz-meta-owner", HeaderValue::from_static("alice")); + headers.insert("cache-control", HeaderValue::from_static("max-age=60")); + + let auth_headers = headers.clone(); + let member_headers = snowball_member_headers(&headers); + + assert_eq!(auth_headers, headers, "IAM conditions must see every signed request header"); + assert!(auth_headers.contains_key(AMZ_SNOWBALL_EXTRACT)); + assert!(auth_headers.contains_key(AMZ_MINIO_SNOWBALL_PREFIX)); + assert!(!member_headers.contains_key(AMZ_SNOWBALL_EXTRACT)); + assert!(!member_headers.contains_key(AMZ_SNOWBALL_EXTRACT_COMPAT)); + assert!(!member_headers.contains_key(AMZ_MINIO_SNOWBALL_PREFIX)); + assert!(!member_headers.contains_key("x-amz-meta-acme-snowball-ignore-dirs")); + assert!(!member_headers.contains_key("snowball-auto-extract")); + assert!(!member_headers.contains_key("rustfs-snowball-ignore-errors")); + assert_eq!(member_headers.get("x-amz-meta-owner"), Some(&HeaderValue::from_static("alice"))); + assert_eq!(member_headers.get("cache-control"), Some(&HeaderValue::from_static("max-age=60"))); + } + + #[test] + fn snowball_pax_tagging_reuses_put_tag_parser_and_validator() { + let mut valid = ExtractPaxOverrides::default(); + valid + .overlay_record("minio.metadata.x-amz-tagging", "project=rustfs&label=snowball%20import") + .expect("encoded tags should fit in a PAX header"); + let mut metadata = HashMap::new(); + apply_extract_pax_overrides( + &valid, + "bucket", + "tagged.txt", + &metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + &mut metadata, + &mut ObjectOptions::default(), + ) + .expect("valid object tags should be canonicalized"); + assert_eq!( + metadata.get(AMZ_OBJECT_TAGGING).map(String::as_str), + Some("project=rustfs&label=snowball+import") + ); + + let too_many = (0..11) + .map(|index| format!("k{index}=v{index}")) + .collect::>() + .join("&"); + for (case, tagging) in [ + ("duplicate", "project=rustfs&project=cli".to_string()), + ("bad-percent-encoding", "project=rustfs%ZZ".to_string()), + ("too-many", too_many), + ] { + let mut invalid = ExtractPaxOverrides::default(); + invalid + .overlay_record("minio.metadata.x-amz-tagging", &tagging) + .expect("tag validation should happen at member application"); + let err = apply_extract_pax_overrides( + &invalid, + "bucket", + "tagged.txt", + &metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + &mut HashMap::new(), + &mut ObjectOptions::default(), + ) + .expect_err("invalid PAX object tags must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::InvalidTag, "{case}"); + } + } + + #[test] + fn snowball_pax_auth_view_only_contains_applied_condition_fields() { + let mut overrides = ExtractPaxOverrides::default(); + for (name, value) in [ + ("user-agent", "trusted"), + ("authorization", "AWS4-HMAC-SHA256 injected"), + ("x-amz-server-side-encryption", "AES256"), + (AMZ_STORAGE_CLASS, "STANDARD"), + ("x-amz-tagging", "project=rustfs"), + ] { + overrides + .overlay_record(&format!("minio.metadata.{name}"), value) + .expect("test PAX metadata should parse"); + } + + let authorization = apply_extract_pax_overrides( + &overrides, + "bucket", + "object", + &metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + &mut HashMap::new(), + &mut ObjectOptions::default(), + ) + .expect("allowed PAX metadata should apply"); + + assert_eq!(authorization.headers.len(), 2); + assert_eq!(authorization.headers.get(AMZ_STORAGE_CLASS), Some(&HeaderValue::from_static("STANDARD"))); + assert_eq!( + authorization.headers.get("x-amz-tagging"), + Some(&HeaderValue::from_static("project=rustfs")) + ); + for prohibited in ["user-agent", "authorization", "x-amz-server-side-encryption"] { + assert!(!authorization.headers.contains_key(prohibited)); + } + } + + #[test] + fn snowball_pax_rejects_invalid_storage_class() { + let mut overrides = ExtractPaxOverrides::default(); + overrides + .overlay_record("minio.metadata.x-amz-storage-class", "INVALID") + .expect("storage class validation should happen after PAX parsing"); + + let err = apply_extract_pax_overrides( + &overrides, + "bucket", + "object", + &metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + &mut HashMap::new(), + &mut ObjectOptions::default(), + ) + .expect_err("invalid PAX storage class must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass); + } + + #[test] + fn snowball_pax_enforces_effective_metadata_budgets_without_panicking() { + let header_name = "x-test"; + let exact_header_value = "v".repeat(EXTRACT_MAX_EFFECTIVE_PAX_HEADER_BYTES - header_name.len()); + let mut exact_headers = ExtractPaxOverrides::default(); + exact_headers + .overlay_record(&format!("minio.metadata.{header_name}"), &exact_header_value) + .expect("exact effective header budget should be accepted"); + assert_eq!(exact_headers.header_bytes, EXTRACT_MAX_EFFECTIVE_PAX_HEADER_BYTES); + assert!( + exact_headers + .overlay_record( + &format!("minio.metadata.{header_name}"), + &"v".repeat(EXTRACT_MAX_EFFECTIVE_PAX_HEADER_BYTES - header_name.len() + 1), + ) + .is_err() + ); + + let user_name = "x-amz-meta-owner"; + let exact_user_value = "u".repeat(EXTRACT_MAX_EFFECTIVE_PAX_USER_METADATA_BYTES - user_name.len()); + let mut exact_user_metadata = ExtractPaxOverrides::default(); + exact_user_metadata + .overlay_record(&format!("minio.metadata.{user_name}"), &exact_user_value) + .expect("exact user metadata budget should be accepted"); + assert_eq!(exact_user_metadata.user_metadata_bytes, EXTRACT_MAX_EFFECTIVE_PAX_USER_METADATA_BYTES); + assert!( + exact_user_metadata + .overlay_record( + &format!("minio.metadata.{user_name}"), + &"u".repeat(EXTRACT_MAX_EFFECTIVE_PAX_USER_METADATA_BYTES - user_name.len() + 1), + ) + .is_err() + ); + + let mut many_fields = ExtractPaxOverrides::default(); + let mut first_error = None; + for index in 0..EXTRACT_MAX_EFFECTIVE_PAX_FIELDS + 1 { + if let Err(err) = many_fields.overlay_record(&format!("minio.metadata.x-field-{index}"), "v") { + first_error = Some(err); + break; + } + } + assert!(first_error.is_some(), "bounded PAX state must reject before HeaderMap capacity"); + assert!(many_fields.headers.len() < EXTRACT_MAX_EFFECTIVE_PAX_FIELDS); + } + + #[test] + fn snowball_expanded_pax_metadata_total_accepts_exact_limit() { + validate_extract_expanded_pax_metadata_total(EXTRACT_MAX_EXPANDED_PAX_METADATA_BYTES) + .expect("exact expanded metadata limit should be accepted"); + assert!(validate_extract_expanded_pax_metadata_total(EXTRACT_MAX_EXPANDED_PAX_METADATA_BYTES + 1).is_err()); + } + + #[test] + fn snowball_pax_metadata_precedence_is_outer_then_local() { + let mut local = ExtractPaxOverrides::default(); + local + .overlay_record("minio.metadata.x-amz-meta-snowball-auto-extract", "true") + .expect("transport metadata should be ignored"); + local + .overlay_record("minio.metadata.X-Amz-Meta-Owner", "local") + .expect("local owner metadata should parse"); + local + .overlay_record("minio.metadata.x-amz-tagging", "classification=public") + .expect("local tags should parse"); + let mut local_metadata = HashMap::from([("owner".to_string(), "outer".to_string())]); + let authorization = apply_extract_pax_overrides( + &local, + "bucket", + "local.txt", + &metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + &mut local_metadata, + &mut ObjectOptions::default(), + ) + .expect("local PAX metadata should apply"); + assert_eq!(local_metadata.get("owner").map(String::as_str), Some("local")); + assert_eq!(local_metadata.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some("classification=public")); + assert!(!local_metadata.contains_key("snowball-auto-extract")); + assert!(!authorization.headers.contains_key(AMZ_SNOWBALL_EXTRACT)); + } + + #[test] + fn snowball_version_id_accepts_exact_null_and_requires_versioning_for_uuids() { + let mut unversioned = ObjectOptions::default(); + assert_eq!( + apply_extract_version_id("null", &mut unversioned).expect("exact null should be accepted"), + "null" + ); + assert_eq!(unversioned.version_id.as_deref(), Some(Uuid::nil().to_string().as_str())); + assert!(apply_extract_version_id("NULL", &mut ObjectOptions::default()).is_err()); + + let version_id = Uuid::new_v4().to_string(); + assert!(apply_extract_version_id(&version_id, &mut ObjectOptions::default()).is_err()); + let mut versioned = ObjectOptions { + versioned: true, + ..Default::default() + }; + assert_eq!( + apply_extract_version_id(&version_id, &mut versioned).expect("UUID should be accepted for a versioned key"), + version_id + ); + } + + #[test] + fn snowball_notification_version_id_follows_member_versioning_state() { + let nil = Some(Uuid::nil()); + assert!(extract_notification_version_id(nil, false, false).is_empty()); + assert_eq!(extract_notification_version_id(nil, true, false), "null"); + assert_eq!(extract_notification_version_id(nil, false, true), "null"); + assert!(extract_notification_version_id(None, true, false).is_empty()); + + let version_id = Uuid::new_v4(); + assert_eq!(extract_notification_version_id(Some(version_id), true, false), version_id.to_string()); + } + + #[test] + fn snowball_iam_requirements_follow_final_member_state() { + let metadata = HashMap::from([(AMZ_OBJECT_TAGGING.to_string(), "project=snowball".to_string())]); + let legal_hold = ObjectLockLegalHoldStatus::from("ON".to_string()); + let mode = ObjectLockMode::from("GOVERNANCE".to_string()); + let retain_until = Timestamp::from(OffsetDateTime::now_utc()); + + assert_eq!( + extract_member_iam_requirements(&metadata, Some(&legal_hold), Some(&mode), Some(&retain_until), Some("null"), true,), + ExtractMemberIamRequirements { + tagging: true, + retention: true, + legal_hold: true, + replication: true, + } + ); + + let bucket_default_retention = HashMap::from([ + (AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string()), + (AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2099-01-01T00:00:00Z".to_string()), + ]); + assert!(!extract_member_iam_requirements(&bucket_default_retention, None, None, None, None, false,).retention); + } + #[test] fn is_put_object_extract_requested_accepts_meta_header() { let mut headers = HeaderMap::new(); @@ -2278,8 +4306,8 @@ mod tests { #[test] fn classify_extract_entry_type_skips_links_extensions_and_continuous_entries() { - assert_eq!(classify_extract_entry_type(EntryType::Regular), ExtractEntryDisposition::File); - assert_eq!(classify_extract_entry_type(EntryType::Directory), ExtractEntryDisposition::Directory); + assert_eq!(classify_extract_entry_type(EntryType::Regular), ExtractEntryKind::Object); + assert_eq!(classify_extract_entry_type(EntryType::Directory), ExtractEntryKind::Directory); for entry_type in [ EntryType::Link, EntryType::Symlink, @@ -2289,7 +4317,7 @@ mod tests { ] { assert_eq!( classify_extract_entry_type(entry_type), - ExtractEntryDisposition::FormatSkip, + ExtractEntryKind::Skip, "{entry_type:?} must not be materialized as an object" ); } @@ -2297,9 +4325,9 @@ mod tests { #[test] fn extract_entry_quota_growth_counts_only_materialized_files() { - assert_eq!(extract_entry_quota_growth(ExtractEntryDisposition::File, 9), 9); - assert_eq!(extract_entry_quota_growth(ExtractEntryDisposition::Directory, 9), 0); - assert_eq!(extract_entry_quota_growth(ExtractEntryDisposition::FormatSkip, 9), 0); + assert_eq!(extract_entry_quota_growth(ExtractEntryKind::Object, 9), 9); + assert_eq!(extract_entry_quota_growth(ExtractEntryKind::Directory, 9), 0); + assert_eq!(extract_entry_quota_growth(ExtractEntryKind::Skip, 9), 0); } #[test] @@ -2486,11 +4514,18 @@ mod tests { assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); assert!(failed.load(Ordering::Acquire)); - assert!(!should_ignore_extract_member_write_error(true, &failed)); + let classified = classify_extract_member_write_error(object_s3_error_default(S3ErrorCode::IncompleteBody), &failed) + .into_unignored(true) + .expect("a reader-backed store error must remain fatal"); + assert_eq!(classified.code(), &S3ErrorCode::IncompleteBody); let storage_only_failure = AtomicBool::new(false); - assert!(should_ignore_extract_member_write_error(true, &storage_only_failure)); - assert!(!should_ignore_extract_member_write_error(false, &storage_only_failure)); + assert!( + classify_extract_member_write_error(object_s3_error_default(S3ErrorCode::InternalError), &storage_only_failure) + .into_unignored(true) + .is_none(), + "a storage-only write failure may be ignored" + ); } #[tokio::test] diff --git a/rustfs/src/app/object/mod.rs b/rustfs/src/app/object/mod.rs index 8f75eee9e..6dbb5d24a 100644 --- a/rustfs/src/app/object/mod.rs +++ b/rustfs/src/app/object/mod.rs @@ -182,6 +182,10 @@ fn object_s3_error(code: S3ErrorCode, message: impl Into S3Error { + S3Error::new(code) +} + mod copy; mod delete; mod extract; diff --git a/rustfs/src/app/object/put.rs b/rustfs/src/app/object/put.rs index abb65524a..673498e9b 100644 --- a/rustfs/src/app/object/put.rs +++ b/rustfs/src/app/object/put.rs @@ -308,14 +308,14 @@ pub(crate) fn guard_put_object_body_read_timeout( }) } -struct PooledBufferReader { +pub(super) struct PooledBufferReader { buffer: PooledBuffer, len: usize, pos: usize, } impl PooledBufferReader { - fn new(buffer: PooledBuffer, len: usize) -> Self { + pub(super) fn new(buffer: PooledBuffer, len: usize) -> Self { Self { buffer, len, pos: 0 } } } @@ -631,7 +631,7 @@ fn select_put_path_with_concurrency( /// where the allocation cost is negligible (≤4KiB memcpy). const POOL_BYPASS_MAX_SIZE: usize = 4 * 1024; -async fn read_small_put_body_into(body: &mut R, buf: &mut B, size: usize) -> S3Result<()> +pub(super) async fn read_small_put_body_into(body: &mut R, buf: &mut B, size: usize) -> S3Result<()> where R: AsyncRead + Unpin, B: bytes::BufMut, @@ -958,11 +958,9 @@ impl DefaultObjectUsecase { return Err(s3_error!(InvalidStorageClass)); } // An authorized inbound replication PUT must store the replica verbatim. - // A snowball-extracted member object keeps `x-amz-meta-snowball-auto-extract` - // in its user metadata, and the replication client replays stored metadata - // as headers — re-dispatching that PUT into the extract path would try to - // untar the member's own bytes (failing replication for any non-archive - // member) instead of writing the replica. + // Legacy snowball-extracted members may still carry the auto-extract + // metadata, which replication replays as a header. Do not interpret that + // historical user metadata as a request to untar the member again. let inbound_replication_put = replication_request_authorized(&req) && get_header(&req.headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true"); if max_content_length.is_some() && is_put_object_extract_requested(&req.headers) { diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 2dd54126d..8eb19d5d4 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -978,9 +978,12 @@ pub(crate) mod bucket { } pub(crate) mod concurrency { + #[cfg(test)] + pub(crate) use crate::storage::storage_api::concurrency_consumer::SNOWBALL_MEMBER_COMMIT_LIMIT; pub(crate) use crate::storage::storage_api::concurrency_consumer::{ ConcurrencyManager, DiskReadAdmission, ForegroundWriteAdmission, GetObjectGuard, IoQueueStatus, IoStrategy, - PutObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager, get_put_concurrency_aware_buffer_size, + PutObjectGuard, SNOWBALL_STAGING_BYTES_LIMIT, get_concurrency_aware_buffer_size, get_concurrency_manager, + get_put_concurrency_aware_buffer_size, }; } @@ -1097,7 +1100,9 @@ pub(crate) mod s3_api { } pub(crate) mod tagging { - pub(crate) use crate::storage::storage_api::s3_api_consumer::tagging::resolve_copy_object_tags; + pub(crate) use crate::storage::storage_api::s3_api_consumer::tagging::{ + parse_copy_object_tags, resolve_copy_object_tags, + }; } } diff --git a/rustfs/src/auth.rs b/rustfs/src/auth.rs index 7b7c3ff54..6ac7c5122 100644 --- a/rustfs/src/auth.rs +++ b/rustfs/src/auth.rs @@ -815,7 +815,15 @@ pub fn get_condition_values_with_query_and_client_info( /// `key`, either because the server already derived that key from verified state or /// because it is a well-known identity/context key that only the server may populate. fn is_reserved_condition_key(key: &str, server_derived: &HashMap>) -> bool { - server_derived.contains_key(key) || is_server_derived_condition_key(key) + server_derived.contains_key(key) + || [ + AMZ_OBJECT_LOCK_MODE_LOWER, + AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, + AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, + ] + .iter() + .any(|header| key.eq_ignore_ascii_case(header.trim_start_matches("x-amz-"))) + || is_server_derived_condition_key(key) } /// Get request authentication type @@ -1670,17 +1678,37 @@ mod tests { let cred = create_test_credentials(); let mut headers = HeaderMap::new(); headers.insert(AMZ_OBJECT_LOCK_MODE_LOWER, HeaderValue::from_static("GOVERNANCE")); + headers.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, HeaderValue::from_static("OFF")); headers.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, HeaderValue::from_static("2024-12-31T23:59:59Z")); + headers.insert("object-lock-mode", HeaderValue::from_static("COMPLIANCE")); + headers.insert("object-lock-legal-hold", HeaderValue::from_static("ON")); + headers.insert("object-lock-retain-until-date", HeaderValue::from_static("2099-12-31T23:59:59Z")); let conditions = get_condition_values(&headers, &cred, None, None, None); assert_eq!(conditions.get("object-lock-mode"), Some(&vec!["GOVERNANCE".to_string()])); + assert_eq!(conditions.get("object-lock-legal-hold"), Some(&vec!["OFF".to_string()])); assert_eq!( conditions.get("object-lock-retain-until-date"), Some(&vec!["2024-12-31T23:59:59Z".to_string()]) ); } + #[test] + fn object_lock_condition_aliases_cannot_spoof_canonical_headers() { + let cred = create_test_credentials(); + let mut headers = HeaderMap::new(); + headers.insert("object-lock-mode", HeaderValue::from_static("COMPLIANCE")); + headers.insert("object-lock-legal-hold", HeaderValue::from_static("ON")); + headers.insert("object-lock-retain-until-date", HeaderValue::from_static("2099-12-31T23:59:59Z")); + + let conditions = get_condition_values(&headers, &cred, None, None, None); + + assert_eq!(conditions.get("object-lock-mode"), None); + assert_eq!(conditions.get("object-lock-legal-hold"), None); + assert_eq!(conditions.get("object-lock-retain-until-date"), None); + } + #[test] fn test_get_condition_values_with_grant_headers() { let cred = create_test_credentials(); diff --git a/rustfs/src/storage/concurrency/manager.rs b/rustfs/src/storage/concurrency/manager.rs index 4c5d20ef0..779f9b5c0 100644 --- a/rustfs/src/storage/concurrency/manager.rs +++ b/rustfs/src/storage/concurrency/manager.rs @@ -31,10 +31,12 @@ use rustfs_io_metrics::bandwidth::{BandwidthMonitor, BandwidthSnapshot}; use rustfs_io_metrics::{MetricsCollector, PerformanceMetrics}; use std::sync::{Arc, LazyLock, Mutex}; use std::time::Duration; -use tokio::sync::Semaphore; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tracing::debug; const DERIVED_LARGE_PUT_ADMISSION_LIMIT_MAX: usize = 32; +pub(crate) const SNOWBALL_MEMBER_COMMIT_LIMIT: usize = 32; +pub(crate) const SNOWBALL_STAGING_BYTES_LIMIT: usize = 4 * MI_B; /// Global concurrency manager instance pub(crate) static CONCURRENCY_MANAGER: LazyLock = LazyLock::new(ConcurrencyManager::new); @@ -69,6 +71,12 @@ pub struct ConcurrencyManager { metrics_collector: Arc, /// Foreground write admission policy, resolved once at startup. foreground_write_admission_policy: ForegroundWriteAdmissionPolicy, + /// Snowball members are internal PUTs, so they use a separate global gate + /// from preparation through the independently owned post-commit tail. + snowball_member_commit_semaphore: Arc, + /// Bounds the owned member bodies and metadata retained between TAR parsing + /// and storage commit across all extract requests. + snowball_staging_bytes_semaphore: Arc, } impl std::fmt::Debug for ConcurrencyManager { @@ -417,6 +425,8 @@ impl ConcurrencyManager { bandwidth_monitor, metrics_collector, foreground_write_admission_policy, + snowball_member_commit_semaphore: Arc::new(Semaphore::new(SNOWBALL_MEMBER_COMMIT_LIMIT)), + snowball_staging_bytes_semaphore: Arc::new(Semaphore::new(SNOWBALL_STAGING_BYTES_LIMIT)), } } @@ -556,6 +566,40 @@ impl ConcurrencyManager { .await } + /// Admit a Snowball member through the foreground PUT policy using the + /// member's logical size. The outer archive has a separate preflight, while + /// every member shares the ordinary PUT gate and its wait/rejection policy. + pub(crate) async fn admit_snowball_foreground_write( + &self, + member_size: i64, + ) -> Result { + self.foreground_write_admission_policy + .admit(ForegroundWriteAdmissionKind::PutObject, member_size) + .await + } + + /// Acquire one global Snowball member lifecycle slot. + pub(crate) async fn acquire_snowball_member_commit(&self) -> Result { + self.snowball_member_commit_semaphore.clone().acquire_owned().await + } + + /// Try to acquire one global Snowball member lifecycle slot. + pub(crate) fn try_acquire_snowball_member_commit(&self) -> Option { + self.snowball_member_commit_semaphore.clone().try_acquire_owned().ok() + } + + /// Try to reserve prepared-member bytes without waiting. + /// + /// A producer holding a non-empty micro-batch must use this method and + /// flush before waiting, otherwise several archives can each retain part of + /// the global budget while waiting forever for the remainder. + pub(crate) fn try_acquire_snowball_staging_bytes(&self, bytes: u32) -> Option { + self.snowball_staging_bytes_semaphore + .clone() + .try_acquire_many_owned(bytes) + .ok() + } + /// Admit a multipart UploadPart request under the configured write gate. /// /// Multipart workloads can saturate memory and internode write streams with @@ -1050,13 +1094,138 @@ impl Default for ConcurrencyManager { mod integration_tests { use super::super::io_schedule::{IoLoadLevel, IoPriority}; use super::super::request_guard::GetObjectGuard; - use super::{ConcurrencyManager, ForegroundWriteAdmission, derive_large_put_admission_limit}; + use super::{ + ConcurrencyManager, ForegroundWriteAdmission, SNOWBALL_MEMBER_COMMIT_LIMIT, SNOWBALL_STAGING_BYTES_LIMIT, + derive_large_put_admission_limit, + }; use crate::storage::storage_api::concurrency_consumer::PutObjectGuard; use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshotProvider, WorkloadClass}; use rustfs_io_core::io_profile::{AccessPattern, StorageMedia}; use serial_test::serial; use std::time::Duration; + #[test] + fn test_snowball_gates_are_global_bounded_and_reusable() { + let manager = ConcurrencyManager::new(); + let clone = manager.clone(); + + let commit_permits = manager + .snowball_member_commit_semaphore + .clone() + .try_acquire_many_owned(u32::try_from(SNOWBALL_MEMBER_COMMIT_LIMIT).expect("Snowball commit limit must fit into u32")) + .expect("the exact Snowball commit limit must be available"); + assert!( + clone.snowball_member_commit_semaphore.clone().try_acquire_owned().is_err(), + "a cloned manager must share the global commit gate" + ); + drop(commit_permits); + assert!(clone.snowball_member_commit_semaphore.clone().try_acquire_owned().is_ok()); + + let staging_bytes = u32::try_from(SNOWBALL_STAGING_BYTES_LIMIT).expect("Snowball staging limit must fit into u32"); + let staging_permit = manager + .try_acquire_snowball_staging_bytes(staging_bytes) + .expect("the exact Snowball staging budget must be available"); + assert!( + clone.try_acquire_snowball_staging_bytes(1).is_none(), + "a cloned manager must share the global staging budget" + ); + drop(staging_permit); + assert!(clone.try_acquire_snowball_staging_bytes(1).is_some()); + } + + #[tokio::test] + async fn test_snowball_members_share_the_strict_foreground_put_gate() { + let manager = ConcurrencyManager::with_put_admission_for_test(true, 2, Duration::ZERO); + let outer = match manager + .admit_put_object(1) + .await + .expect("strict outer admission must remain open") + { + ForegroundWriteAdmission::Admitted(permit) => permit, + outcome => panic!("strict outer admission must return a permit: {outcome:?}"), + }; + let member = match manager + .admit_snowball_foreground_write(1) + .await + .expect("strict member admission must remain open") + { + ForegroundWriteAdmission::Admitted(permit) => permit, + outcome => panic!("strict member admission must return a permit: {outcome:?}"), + }; + assert!( + matches!( + manager + .admit_snowball_foreground_write(1) + .await + .expect("strict member admission must remain open"), + ForegroundWriteAdmission::Rejected + ), + "a saturated Snowball member admission must preserve the zero-wait rejection policy" + ); + assert!( + matches!( + manager.admit_put_object(1).await.expect("strict gate must remain usable"), + ForegroundWriteAdmission::Rejected + ), + "outer PUTs and Snowball members must exhaust the same strict gate" + ); + + drop(outer); + let replacement = match manager + .admit_snowball_foreground_write(1) + .await + .expect("released strict capacity must be reusable") + { + ForegroundWriteAdmission::Admitted(permit) => permit, + outcome => panic!("strict replacement admission must return a permit: {outcome:?}"), + }; + drop((member, replacement)); + } + + #[tokio::test] + async fn test_snowball_members_use_their_size_for_the_large_foreground_put_gate() { + let min_size = 16 * 1024 * 1024; + let manager = ConcurrencyManager::with_large_put_admission_for_test(true, 1, min_size, Duration::ZERO); + + assert!(matches!( + manager + .admit_put_object((min_size - 1) as i64) + .await + .expect("small outer archive admission must remain open"), + ForegroundWriteAdmission::Disabled + )); + let large_member = match manager + .admit_snowball_foreground_write(min_size as i64) + .await + .expect("large Snowball member admission must remain open") + { + ForegroundWriteAdmission::Admitted(permit) => permit, + outcome => panic!("large Snowball member must consume the large PUT gate: {outcome:?}"), + }; + assert!(matches!( + manager + .admit_snowball_foreground_write(min_size as i64) + .await + .expect("saturated Snowball member admission must remain open"), + ForegroundWriteAdmission::Rejected + )); + assert!(matches!( + manager + .admit_put_object(min_size as i64) + .await + .expect("ordinary large PUT admission must remain open"), + ForegroundWriteAdmission::Rejected + )); + assert!(matches!( + manager + .admit_snowball_foreground_write((min_size - 1) as i64) + .await + .expect("small Snowball member admission must remain open"), + ForegroundWriteAdmission::Disabled + )); + drop(large_member); + } + #[tokio::test] #[serial] async fn test_concurrency_manager_priority_queue_integration() { diff --git a/rustfs/src/storage/concurrency/mod.rs b/rustfs/src/storage/concurrency/mod.rs index 7996649ab..03f52abb9 100644 --- a/rustfs/src/storage/concurrency/mod.rs +++ b/rustfs/src/storage/concurrency/mod.rs @@ -51,6 +51,9 @@ pub use io_schedule::{ pub use request_guard::{GetObjectGuard, PutObjectGuard}; // Concurrency manager +#[cfg(test)] +pub(crate) use manager::SNOWBALL_MEMBER_COMMIT_LIMIT; +pub(crate) use manager::SNOWBALL_STAGING_BYTES_LIMIT; pub use manager::{ConcurrencyManager, DiskReadAdmission, ForegroundWriteAdmission}; // ============================================ diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 0288ac712..f82adf0ce 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -125,9 +125,12 @@ pub(crate) mod access_consumer { } pub(crate) mod concurrency_consumer { + #[cfg(test)] + pub(crate) use super::super::concurrency::SNOWBALL_MEMBER_COMMIT_LIMIT; pub(crate) use super::super::concurrency::{ ConcurrencyManager, DiskReadAdmission, ForegroundWriteAdmission, GetObjectGuard, IoQueueStatus, IoStrategy, - PutObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager, get_put_concurrency_aware_buffer_size, + PutObjectGuard, SNOWBALL_STAGING_BYTES_LIMIT, get_concurrency_aware_buffer_size, get_concurrency_manager, + get_put_concurrency_aware_buffer_size, }; } @@ -350,7 +353,7 @@ pub(crate) mod s3_api_consumer { } pub(crate) mod tagging { - pub(crate) use super::super::super::s3_api::tagging::resolve_copy_object_tags; + pub(crate) use super::super::super::s3_api::tagging::{parse_copy_object_tags, resolve_copy_object_tags}; } } diff --git a/rustfs/src/table_catalog/tests.rs b/rustfs/src/table_catalog/tests.rs index 40f3d7c88..6ed547c03 100644 --- a/rustfs/src/table_catalog/tests.rs +++ b/rustfs/src/table_catalog/tests.rs @@ -16445,7 +16445,7 @@ fn object_mutation_entrypoints_call_reserved_prefix_guard() { "if let Err(err) = validate_table_catalog_object_mutation(&bucket, &obj_id.key).await", "validate_table_catalog_object_mutation(&bucket, &object).await?;", "validate_object_key(&key, \"PUT\")?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;", - "validate_table_catalog_object_mutation(&bucket, &fpath).await?;", + "extract_try!(validate_table_catalog_object_mutation(&bucket, &fpath).await);", ] { assert!(source.contains(expected), "missing object mutation guard: {expected}"); }