fix(ecstore): settle checkpoint capacity retries

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=9dccb0cd537cf79ae70c1c20e8281d36d03f2f09f81142a5341e26e3dc18709d
+sha256-darwin=ef914ec0b8daa9c2c5e52f501d339914662f42d6f6ed9d33877d56b97adf16f9
 sha256-linux=a8a816d7bb0e7cb5632b1863b33794bcb9fc7e765f150aa5e1bf16518e28dfb4

Cargo.lock

@@ -10449,7 +10449,6 @@ dependencies = [
  "base64-simd",
  "bytes",
  "crc-fast",
- "criterion",
  "faster-hex",
  "futures",
  "hex-simd",
@@ -10461,7 +10460,6 @@ dependencies = [
  "md-5 0.11.0",
  "minlz",
  "pin-project-lite",
- "proptest",
  "rand 0.10.2",
  "reqwest",
  "rustfs-config",

crates/config/src/constants/object.rs

@@ -197,6 +197,27 @@ pub const DEFAULT_POOL_META_V3_FLEET_CONFIRMED: bool = false;
 const _: () = assert!(!DEFAULT_POOL_META_V3_WRITE);
 const _: () = assert!(!DEFAULT_POOL_META_V3_FLEET_CONFIRMED);
 
+/// Maximum unpacked size accepted for one Snowball archive member.
+///
+/// The value is expressed in bytes. Invalid values use the default, while
+/// valid values are clamped to [`MAX_SNOWBALL_ENTRY_BYTES`].
+pub const ENV_SNOWBALL_MAX_ENTRY_BYTES: &str = "RUSTFS_SNOWBALL_MAX_ENTRY_BYTES";
+pub const DEFAULT_SNOWBALL_MAX_ENTRY_BYTES: u64 = 1024 * 1024 * 1024;
+pub const MAX_SNOWBALL_ENTRY_BYTES: u64 = 1024 * DEFAULT_SNOWBALL_MAX_ENTRY_BYTES;
+
+/// Maximum cumulative unpacked object bytes accepted from one Snowball
+/// archive request.
+///
+/// This does not include tar headers or bounded PAX metadata. The value is
+/// expressed in bytes and is clamped to
+/// [`MAX_SNOWBALL_UNPACKED_BYTES`].
+pub const ENV_SNOWBALL_MAX_UNPACKED_BYTES: &str = "RUSTFS_SNOWBALL_MAX_UNPACKED_BYTES";
+pub const DEFAULT_SNOWBALL_MAX_UNPACKED_BYTES: u64 = 10 * 1024 * 1024 * 1024;
+pub const MAX_SNOWBALL_UNPACKED_BYTES: u64 = 10 * 1024 * DEFAULT_SNOWBALL_MAX_ENTRY_BYTES;
+
+const _: () = assert!(DEFAULT_SNOWBALL_MAX_ENTRY_BYTES <= MAX_SNOWBALL_ENTRY_BYTES);
+const _: () = assert!(DEFAULT_SNOWBALL_MAX_UNPACKED_BYTES <= MAX_SNOWBALL_UNPACKED_BYTES);
+
 // =============================================================================
 // Concurrent Request Fix - Timeout and Backpressure Configuration
 // =============================================================================
@@ -820,4 +841,10 @@ mod remote_version_state_tests {
         assert_eq!(super::ENV_POOL_META_V3_WRITE, "RUSTFS_POOL_META_V3_WRITE");
         assert_eq!(super::ENV_POOL_META_V3_FLEET_CONFIRMED, "RUSTFS_POOL_META_V3_FLEET_CONFIRMED");
     }
+
+    #[test]
+    fn snowball_limit_environment_names_are_stable() {
+        assert_eq!(super::ENV_SNOWBALL_MAX_ENTRY_BYTES, "RUSTFS_SNOWBALL_MAX_ENTRY_BYTES");
+        assert_eq!(super::ENV_SNOWBALL_MAX_UNPACKED_BYTES, "RUSTFS_SNOWBALL_MAX_UNPACKED_BYTES");
+    }
 }

crates/e2e_test/src/fake_s3_target/README.md

@@ -1,15 +1,11 @@
 # Programmable fake S3 target
 
-This module is the shared failure-injection boundary for replication end-to-end tests and the programmable external source for on-demand-migration (ODM) tests. It runs an in-process, path-style S3 endpoint backed by `s3s`; no production crate depends on it.
+This module is the shared failure-injection boundary for replication end-to-end tests. It runs an in-process, path-style S3 endpoint backed by `s3s`; no production crate depends on it.
 
 `FakeS3Target::start()` creates the listener. Add target buckets with `create_bucket`, point a RustFS remote target at `address()`, use `FAKE_ACCESS_KEY` / `FAKE_SECRET_KEY`, then enqueue per-operation faults with `inject`. Faults for one operation are consumed in FIFO order and do not consume faults queued for another operation. A fault is consumed only after `s3s` verifies the full request signature, so anonymous, other-access-key, and bad-signature traffic cannot disturb a script.
 
-Supported data operations are HeadBucket, GetBucketVersioning, ListObjectsV2, PUT/GET/HEAD/DELETE Object, Get/Put/Delete ObjectTagging (tags live per version; Put replaces the whole set, Delete clears it), and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets created with `create_bucket` are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
+Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, Get/Put/Delete ObjectTagging (tags live per version; Put replaces the whole set, Delete clears it), and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
 
-`create_bucket_with_mode(name, BucketMode::Unversioned)` models a plain migration source: PUT overwrites in place, DELETE removes the key without a delete marker, GetBucketVersioning reports no status, and no `x-amz-version-id` is returned by PUT, GET, HEAD, tagging, or multipart completion. The only `versionId` such a bucket accepts is `null`; any other value is rejected with `InvalidArgument`. The mode is fixed at creation.
+Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions. Each record also journals a `ProxyHeaderSnapshot` — the read-proxy anti-loop marker (`x-{rustfs,minio}-source-proxy-request`), the replication-check exemption header, and the client SSE-C header family (algorithm and key-MD5 values; for the key itself only its presence) — so proxy tests can pin the exact wire contract.
 
-ListObjectsV2 lists current versions only (a key whose newest version is a delete marker is hidden) in byte order and supports `prefix`, `delimiter`, `max-keys` (clamped to 1000), `start-after`, and `continuation-token`; common prefixes count toward `max-keys`, `IsTruncated` / `NextContinuationToken` / `KeyCount` follow S3, and continuation tokens are opaque. `encoding-type` and `fetch-owner` are accepted but ignored, and ListObjects (v1) is not implemented. GET and HEAD honor `Range` in the `bytes=first-last`, `bytes=first-`, and `bytes=-suffix` forms with a 206 status, exact `Content-Range`, and `Accept-Ranges: bytes`; unsatisfiable ranges answer 416 `InvalidRange` with `Content-Range: bytes */<length>`. PUT and CreateMultipartUpload accept `Content-Type`, `Content-Encoding`, `Content-Disposition`, `Content-Language`, `Cache-Control`, `Expires`, and `x-amz-meta-*` (names stored lowercased), and HEAD/GET replay them verbatim together with `Last-Modified` and the ETag (hex MD5 for single PUTs, `<md5-of-part-md5s>-<parts>` for multipart objects). `put_seed_object` stores an object directly, bypassing the wire, the fault script, and the journal, so a source can be seeded without polluting the assertions a scenario later makes.
-
-Fault actions cover HTTP 401/403/503 responses (`Status`), any 4xx/5xx status paired with the matching S3 error code (`ResponseStatus`), pre-dispatch delay, holding a fully computed successful response before its first byte (`Stall`), connection abort when a logical request-body threshold is reached, GetObject bodies cut off after N bytes while `Content-Length` announces the full size (`TruncateBodyAt`), streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions and `count_requests(operation, key)` counts entries for one exact key. Each record journals the `Range` and `User-Agent` request headers, the ListObjectsV2 `prefix` and `continuation-token` query values, and a `ProxyHeaderSnapshot` — the read-proxy anti-loop marker (`x-{rustfs,minio}-source-proxy-request`), the replication-check exemption header, and the client SSE-C header family (algorithm and key-MD5 values; for the key itself only its presence) — so proxy tests can pin the exact wire contract.
-
-The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type and each standard object header at 1 KiB. By default a PUT or uploaded part is capped at 64 MiB and a completed multipart object and all stored object/part data are capped at 128 MiB; `FakeS3Target::start_with_options(FakeS3TargetOptions { max_object_bytes })` raises the object cap up to 256 MiB, and the total budget then becomes twice the object cap (never below 128 MiB). Body drain, body-permit waits, delay, stall, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound.
+The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type at 1 KiB. A PUT or uploaded part is capped at 64 MiB; a completed multipart object and all stored object/part data are capped at 128 MiB. Body drain, body-permit waits, delay, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound.

crates/e2e_test/src/lib.rs

@@ -23,17 +23,10 @@ pub mod common;
 #[cfg(test)]
 pub mod chaos;
 
-// Programmable S3 target for replication failure-path tests (backlog#1147 repl-8)
-// and on-demand-migration source scenarios (backlog#2151).
+// Programmable S3 target for replication failure-path tests (backlog#1147 repl-8).
 #[cfg(test)]
 pub mod fake_s3_target;
 
-// On-demand migration (backlog#2147): shared two-server environment, admin
-// wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
-// added by later ODM tasks.
-#[cfg(test)]
-pub mod on_demand_migration;
-
 // Socket-level network fault-injection proxy for black-box cluster tests
 // (backlog#1325 network fault-injection block): latency / blackhole / one-way
 // partition on the wire between nodes. Serves #1312/#1319 (lock-plane one-way

crates/e2e_test/src/on_demand_migration/common.rs

@@ -1,452 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Shared environment for on-demand migration (ODM) end-to-end tests.
-//!
-//! [`OdmTestEnv`] pairs one RustFS server under test with one in-process
-//! programmable S3 source ([`FakeS3Target`]). Admin calls target the route
-//! convention fixed by the tracking plan
-//! (`/rustfs/admin/v3/on-demand-migration/{bucket}`, JSON bodies); the
-//! server side lands with ODM-07, so until then the wrappers compile but are
-//! not exercised by the harness self-test.
-
-use crate::common::{RustFSTestEnvironment, signed_request};
-use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FakeS3TargetOptions, SeedMetadata};
-use aws_config::retry::RetryConfig;
-use aws_sdk_s3::Client;
-use aws_sdk_s3::config::{Credentials, Region};
-use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
-use bytes::Bytes;
-use serde::Serialize;
-use std::fmt;
-
-pub type BoxError = Box<dyn std::error::Error + Send + Sync>;
-
-/// Module switch the server reads at startup (`false` before GA). The harness
-/// turns it on so scenario tests exercise the feature without repeating it.
-pub const ODM_MODULE_SWITCH_ENV: &str = "RUSTFS_ON_DEMAND_MIGRATION_ENABLED";
-/// Admin route prefix; the bucket name is appended as one path segment.
-pub const ODM_ADMIN_ROUTE: &str = "/rustfs/admin/v3/on-demand-migration";
-/// Region the fake source is addressed with (it accepts any SigV4 region).
-pub const FAKE_SOURCE_REGION: &str = "us-east-1";
-
-/// Wire form of the bucket-level ODM configuration (ODM-01 model). Every
-/// field is public so a scenario can tweak one knob and serialize the rest
-/// with the documented defaults.
-#[derive(Debug, Clone, Serialize)]
-pub struct OdmSourceSpec {
-    pub version: u32,
-    pub enabled: bool,
-    pub source: OdmSource,
-    pub filter: OdmFilter,
-    pub policy: OdmPolicy,
-}
-
-#[derive(Debug, Clone, Serialize)]
-pub struct OdmSource {
-    pub provider: String,
-    pub endpoint: String,
-    pub region: String,
-    pub bucket: String,
-    pub path_style: String,
-    pub credentials: Option<OdmCredentials>,
-    pub tls: OdmTls,
-}
-
-#[derive(Clone, Serialize)]
-pub struct OdmCredentials {
-    pub access_key: String,
-    pub secret_key: String,
-    pub session_token: Option<String>,
-}
-
-impl fmt::Debug for OdmCredentials {
-    /// Test logs are captured into CI artifacts; keep the secret out of them.
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("OdmCredentials")
-            .field("access_key", &self.access_key)
-            .field("secret_key", &"REDACTED")
-            .field("session_token", &self.session_token.as_ref().map(|_| "REDACTED"))
-            .finish()
-    }
-}
-
-#[derive(Debug, Clone, Default, Serialize)]
-pub struct OdmTls {
-    pub skip_verify: bool,
-    pub ca_cert_pem: Option<String>,
-}
-
-#[derive(Debug, Clone, Default, Serialize)]
-pub struct OdmFilter {
-    pub prefix: Option<String>,
-    pub source_prefix: Option<String>,
-}
-
-#[derive(Debug, Clone, Serialize)]
-pub struct OdmPolicy {
-    pub head: String,
-    pub range_get: String,
-    pub source_error: String,
-    pub respect_local_delete_marker: bool,
-    pub preserve_etag: bool,
-    pub copy_tags: bool,
-    pub emit_events: bool,
-    pub negative_cache_ttl_secs: u64,
-    pub inline_max_bytes: u64,
-    pub multipart_part_size_bytes: u64,
-    pub max_concurrent_pulls: u32,
-    pub pull_queue_capacity: u32,
-    pub source_timeout: OdmSourceTimeout,
-    pub bandwidth_limit_bytes_per_sec: Option<u64>,
-}
-
-#[derive(Debug, Clone, Serialize)]
-pub struct OdmSourceTimeout {
-    pub connect_ms: u64,
-    pub first_byte_ms: u64,
-    pub idle_ms: u64,
-}
-
-impl Default for OdmPolicy {
-    /// The ODM-01 defaults verbatim.
-    fn default() -> Self {
-        Self {
-            head: "proxy".to_string(),
-            range_get: "serve_and_backfill".to_string(),
-            source_error: "propagate".to_string(),
-            respect_local_delete_marker: true,
-            preserve_etag: true,
-            copy_tags: false,
-            emit_events: true,
-            negative_cache_ttl_secs: 30,
-            inline_max_bytes: 16 * 1024 * 1024,
-            multipart_part_size_bytes: 64 * 1024 * 1024,
-            max_concurrent_pulls: 8,
-            pull_queue_capacity: 1024,
-            source_timeout: OdmSourceTimeout {
-                connect_ms: 5_000,
-                first_byte_ms: 15_000,
-                idle_ms: 30_000,
-            },
-            bandwidth_limit_bytes_per_sec: None,
-        }
-    }
-}
-
-impl OdmSourceSpec {
-    /// Enabled configuration pointing at a bucket on the fake source with the
-    /// fixture credentials, path-style addressing, and default policy.
-    pub fn for_fake_source(source: &FakeS3Target, source_bucket: impl Into<String>) -> Self {
-        Self::new(
-            "s3",
-            source.endpoint(),
-            FAKE_SOURCE_REGION,
-            source_bucket,
-            FAKE_ACCESS_KEY,
-            FAKE_SECRET_KEY,
-        )
-    }
-
-    /// Enabled configuration pointing at a bucket on a second RustFS server
-    /// (see [`start_source_rustfs`]).
-    pub fn for_rustfs_source(source: &RustFSTestEnvironment, source_bucket: impl Into<String>) -> Self {
-        Self::new(
-            "rustfs",
-            &source.url,
-            FAKE_SOURCE_REGION,
-            source_bucket,
-            &source.access_key,
-            &source.secret_key,
-        )
-    }
-
-    fn new(
-        provider: &str,
-        endpoint: &str,
-        region: &str,
-        source_bucket: impl Into<String>,
-        access_key: &str,
-        secret_key: &str,
-    ) -> Self {
-        Self {
-            version: 1,
-            enabled: true,
-            source: OdmSource {
-                provider: provider.to_string(),
-                endpoint: endpoint.to_string(),
-                region: region.to_string(),
-                bucket: source_bucket.into(),
-                path_style: "path".to_string(),
-                credentials: Some(OdmCredentials {
-                    access_key: access_key.to_string(),
-                    secret_key: secret_key.to_string(),
-                    session_token: None,
-                }),
-                tls: OdmTls::default(),
-            },
-            filter: OdmFilter::default(),
-            policy: OdmPolicy::default(),
-        }
-    }
-
-    pub fn to_json(&self) -> serde_json::Value {
-        serde_json::to_value(self).expect("ODM source spec serializes")
-    }
-}
-
-/// Backfill job control (ODM-12 route shape).
-#[derive(Debug, Clone)]
-pub enum BackfillOp {
-    Start(BackfillRequest),
-    Cancel,
-    Status,
-}
-
-#[derive(Debug, Clone, Default, Serialize)]
-pub struct BackfillRequest {
-    pub prefix: Option<String>,
-    pub skip_existing: Option<String>,
-    pub dry_run: bool,
-}
-
-/// Status plus raw body of an admin call, so a scenario can assert on the
-/// HTTP status first and only then parse the JSON.
-#[derive(Debug, Clone)]
-pub struct AdminResponse {
-    pub status: u16,
-    pub body: String,
-}
-
-impl AdminResponse {
-    pub fn json(&self) -> Result<serde_json::Value, BoxError> {
-        Ok(serde_json::from_str(&self.body)?)
-    }
-}
-
-/// One object to seed into the source.
-#[derive(Clone)]
-pub struct SeedObject {
-    pub key: String,
-    pub body: Bytes,
-    pub metadata: SeedMetadata,
-}
-
-impl SeedObject {
-    pub fn new(key: impl Into<String>, body: impl Into<Bytes>) -> Self {
-        Self {
-            key: key.into(),
-            body: body.into(),
-            metadata: SeedMetadata::new(),
-        }
-    }
-
-    pub fn with_metadata(mut self, metadata: SeedMetadata) -> Self {
-        self.metadata = metadata;
-        self
-    }
-}
-
-/// RustFS under test plus its fake S3 source.
-pub struct OdmTestEnv {
-    pub rustfs: RustFSTestEnvironment,
-    pub source: FakeS3Target,
-    /// S3 client for the RustFS under test.
-    pub client: Client,
-}
-
-impl OdmTestEnv {
-    /// Start a fake source with default limits and a RustFS server with the
-    /// ODM module switch enabled.
-    pub async fn start() -> Result<Self, BoxError> {
-        Self::start_with_options(FakeS3TargetOptions::default()).await
-    }
-
-    pub async fn start_with_options(options: FakeS3TargetOptions) -> Result<Self, BoxError> {
-        let source = FakeS3Target::start_with_options(options).await?;
-        let mut rustfs = RustFSTestEnvironment::new().await?;
-        rustfs
-            .start_rustfs_server_with_env(vec![], &[(ODM_MODULE_SWITCH_ENV, "true")])
-            .await?;
-        let client = rustfs.create_s3_client();
-        Ok(Self { rustfs, source, client })
-    }
-
-    /// S3 client addressing the fake source directly, for assertions on the
-    /// source's own state. Retries are off so a scripted fault is consumed by
-    /// exactly the request the test issued.
-    pub fn source_client(&self) -> Client {
-        fake_source_client(&self.source)
-    }
-
-    /// Enabled ODM configuration for `source_bucket` on the fake source.
-    pub fn fake_source_spec(&self, source_bucket: impl Into<String>) -> OdmSourceSpec {
-        OdmSourceSpec::for_fake_source(&self.source, source_bucket)
-    }
-
-    /// `PUT /rustfs/admin/v3/on-demand-migration/{bucket}` with the JSON spec.
-    pub async fn configure_source(&self, bucket: &str, spec: &OdmSourceSpec) -> Result<AdminResponse, BoxError> {
-        self.admin(http::Method::PUT, &format!("/{bucket}"), Some(spec.to_json()))
-            .await
-    }
-
-    /// Same as [`Self::configure_source`] with `dry-run=true`: validate and
-    /// probe without persisting.
-    pub async fn validate_source(&self, bucket: &str, spec: &OdmSourceSpec) -> Result<AdminResponse, BoxError> {
-        self.admin(http::Method::PUT, &format!("/{bucket}?dry-run=true"), Some(spec.to_json()))
-            .await
-    }
-
-    /// `GET .../{bucket}`: redacted configuration, 404 when unconfigured.
-    pub async fn get_config(&self, bucket: &str) -> Result<AdminResponse, BoxError> {
-        self.admin(http::Method::GET, &format!("/{bucket}"), None).await
-    }
-
-    /// `DELETE .../{bucket}`: remove the configuration (idempotent).
-    pub async fn disable(&self, bucket: &str) -> Result<AdminResponse, BoxError> {
-        self.admin(http::Method::DELETE, &format!("/{bucket}"), None).await
-    }
-
-    /// `GET .../{bucket}/status`: runtime snapshot.
-    pub async fn status(&self, bucket: &str) -> Result<AdminResponse, BoxError> {
-        self.admin(http::Method::GET, &format!("/{bucket}/status"), None).await
-    }
-
-    /// Backfill control: `POST .../{bucket}/backfill?op=start|cancel` or
-    /// `GET .../{bucket}/backfill` for the checkpoint.
-    pub async fn backfill(&self, bucket: &str, op: BackfillOp) -> Result<AdminResponse, BoxError> {
-        match op {
-            BackfillOp::Start(request) => {
-                self.admin(
-                    http::Method::POST,
-                    &format!("/{bucket}/backfill?op=start"),
-                    Some(serde_json::to_value(request)?),
-                )
-                .await
-            }
-            BackfillOp::Cancel => {
-                self.admin(http::Method::POST, &format!("/{bucket}/backfill?op=cancel"), None)
-                    .await
-            }
-            BackfillOp::Status => self.admin(http::Method::GET, &format!("/{bucket}/backfill"), None).await,
-        }
-    }
-
-    async fn admin(
-        &self,
-        method: http::Method,
-        path_and_query: &str,
-        body: Option<serde_json::Value>,
-    ) -> Result<AdminResponse, BoxError> {
-        let url = format!("{}{ODM_ADMIN_ROUTE}{path_and_query}", self.rustfs.url);
-        let body = body.map(|value| serde_json::to_vec(&value)).transpose()?;
-        let content_type = body.is_some().then_some("application/json");
-        let response = signed_request(method, &url, &self.rustfs.access_key, &self.rustfs.secret_key, body, content_type).await?;
-        Ok(AdminResponse {
-            status: response.status().as_u16(),
-            body: response.text().await?,
-        })
-    }
-
-    /// Store objects directly in the fake source (no wire traffic, no journal
-    /// entries). Returns the ETags in input order.
-    pub fn seed_source(&self, source_bucket: &str, objects: &[SeedObject]) -> Vec<String> {
-        objects
-            .iter()
-            .map(|object| {
-                self.source
-                    .put_seed_object(source_bucket, object.key.clone(), object.body.clone(), &object.metadata)
-            })
-            .collect()
-    }
-
-    /// Whether `key` is listed by the RustFS under test. Listing is served from
-    /// local state only, so this does not trigger a migration the way GET or
-    /// HEAD would.
-    pub async fn local_key_listed(&self, bucket: &str, key: &str) -> Result<bool, BoxError> {
-        let listed = self
-            .client
-            .list_objects_v2()
-            .bucket(bucket)
-            .prefix(key)
-            .max_keys(1)
-            .send()
-            .await?;
-        Ok(listed.contents().iter().any(|object| object.key() == Some(key)))
-    }
-
-    /// Panics unless `key` is stored locally with exactly `expected` bytes.
-    /// Presence is checked through listing first so a missing object fails
-    /// here instead of being pulled from the source by the GET.
-    pub async fn assert_local_present(&self, bucket: &str, key: &str, expected: &[u8]) {
-        assert!(
-            self.local_key_listed(bucket, key)
-                .await
-                .unwrap_or_else(|error| panic!("listing {bucket}/{key} failed: {error}")),
-            "{bucket}/{key} must be present locally"
-        );
-        let body = self
-            .client
-            .get_object()
-            .bucket(bucket)
-            .key(key)
-            .send()
-            .await
-            .unwrap_or_else(|error| panic!("GET {bucket}/{key} failed: {error}"))
-            .body
-            .collect()
-            .await
-            .unwrap_or_else(|error| panic!("reading {bucket}/{key} failed: {error}"))
-            .into_bytes();
-        assert_eq!(body.as_ref(), expected, "{bucket}/{key} local content mismatch");
-    }
-
-    /// Panics if `key` is listed locally.
-    pub async fn assert_local_absent(&self, bucket: &str, key: &str) {
-        assert!(
-            !self
-                .local_key_listed(bucket, key)
-                .await
-                .unwrap_or_else(|error| panic!("listing {bucket}/{key} failed: {error}")),
-            "{bucket}/{key} must be absent locally"
-        );
-    }
-}
-
-/// S3 client for the fake source with retries disabled (see
-/// [`OdmTestEnv::source_client`]).
-pub fn fake_source_client(source: &FakeS3Target) -> Client {
-    let credentials = Credentials::new(FAKE_ACCESS_KEY, FAKE_SECRET_KEY, None, None, "odm-fake-source");
-    Client::from_conf(
-        aws_sdk_s3::Config::builder()
-            .credentials_provider(credentials)
-            .region(Region::new(FAKE_SOURCE_REGION))
-            .endpoint_url(source.endpoint())
-            .force_path_style(true)
-            .behavior_version_latest()
-            .retry_config(RetryConfig::standard().with_max_attempts(1))
-            .http_client(SmithyHttpClientBuilder::new().build_http())
-            .build(),
-    )
-}
-
-/// Start a second, fully independent RustFS process (own port, data
-/// directory, and default credentials) to act as a real S3 source. It is
-/// spawned the same way `reliant::tiering` starts its cold tier; the process
-/// is stopped and its directory removed when the returned environment drops.
-pub async fn start_source_rustfs() -> Result<RustFSTestEnvironment, BoxError> {
-    let mut source = RustFSTestEnvironment::new().await?;
-    source.start_rustfs_server_without_cleanup(vec![]).await?;
-    Ok(source)
-}

crates/e2e_test/src/on_demand_migration/harness_self_test.rs

@@ -1,606 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Self-test of the ODM harness (rustfs/backlog#2151): the fake source's
-//! migration-facing surface (ListObjectsV2 paging, `Range`, unversioned
-//! buckets, metadata replay, fault actions) and the two-server environment.
-//! No ODM behavior is exercised here.
-
-use super::common::{OdmTestEnv, SeedObject, fake_source_client, start_source_rustfs};
-use crate::fake_s3_target::{BucketMode, FakeS3Target, FakeS3TargetOptions, FaultAction, Operation, SeedMetadata};
-use aws_sdk_s3::Client;
-use aws_sdk_s3::error::ProvideErrorMetadata;
-use aws_sdk_s3::primitives::{ByteStream, DateTime};
-use bytes::Bytes;
-use std::collections::BTreeSet;
-use std::time::{Duration, Instant};
-
-type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
-
-const SOURCE_BUCKET: &str = "odm-source";
-
-/// Position-dependent payload so a misaligned range read is caught.
-fn payload(len: usize) -> Bytes {
-    (0..len).map(|index| (index % 251) as u8).collect::<Vec<u8>>().into()
-}
-
-async fn fake_source() -> Result<(FakeS3Target, Client), Box<dyn std::error::Error + Send + Sync>> {
-    let source = FakeS3Target::start().await?;
-    source.create_bucket(SOURCE_BUCKET);
-    let client = fake_source_client(&source);
-    Ok((source, client))
-}
-
-/// Full ListObjectsV2 traversal. Returns `(keys, common prefixes, pages)` and
-/// checks the page shape on the way: every page except the last is full and
-/// truncated, the last carries no continuation token.
-async fn list_all(
-    client: &Client,
-    prefix: Option<&str>,
-    delimiter: Option<&str>,
-    start_after: Option<&str>,
-    max_keys: i32,
-) -> Result<(Vec<String>, Vec<String>, usize), Box<dyn std::error::Error + Send + Sync>> {
-    let mut keys = Vec::new();
-    let mut prefixes = Vec::new();
-    let mut pages = 0usize;
-    let mut token: Option<String> = None;
-    loop {
-        let page = client
-            .list_objects_v2()
-            .bucket(SOURCE_BUCKET)
-            .set_prefix(prefix.map(str::to_string))
-            .set_delimiter(delimiter.map(str::to_string))
-            .set_start_after(start_after.map(str::to_string))
-            .max_keys(max_keys)
-            .set_continuation_token(token.clone())
-            .send()
-            .await?;
-        pages += 1;
-        let page_keys: Vec<String> = page
-            .contents()
-            .iter()
-            .filter_map(|object| object.key().map(str::to_string))
-            .collect();
-        let page_prefixes: Vec<String> = page
-            .common_prefixes()
-            .iter()
-            .filter_map(|common| common.prefix().map(str::to_string))
-            .collect();
-        let entries = page_keys.len() + page_prefixes.len();
-        assert_eq!(page.key_count(), Some(entries as i32), "KeyCount must count keys and prefixes");
-        assert_eq!(page.continuation_token(), token.as_deref(), "the request token must be echoed");
-        keys.extend(page_keys);
-        prefixes.extend(page_prefixes);
-        if page.is_truncated() == Some(true) {
-            assert_eq!(entries as i32, max_keys, "every truncated page must be full");
-            token = Some(
-                page.next_continuation_token()
-                    .expect("truncated page must carry a continuation token")
-                    .to_string(),
-            );
-        } else {
-            assert!(page.next_continuation_token().is_none(), "final page must not carry a token");
-            return Ok((keys, prefixes, pages));
-        }
-    }
-}
-
-#[tokio::test]
-async fn fake_source_list_objects_v2_paginates_with_delimiter() -> TestResult {
-    let (source, client) = fake_source().await?;
-    let mut expected_keys = BTreeSet::new();
-    for directory in 0..30 {
-        for file in 0..30 {
-            expected_keys.insert(format!("d{directory:02}/k{file:03}"));
-        }
-    }
-    for index in 0..100 {
-        expected_keys.insert(format!("top-{index:03}"));
-    }
-    assert_eq!(expected_keys.len(), 1000);
-    for key in &expected_keys {
-        source.put_seed_object(SOURCE_BUCKET, key.clone(), Bytes::from(key.clone()), &SeedMetadata::new());
-    }
-    // A key whose current version is a delete marker must stay hidden.
-    client
-        .put_object()
-        .bucket(SOURCE_BUCKET)
-        .key("hidden/marker")
-        .body(ByteStream::from_static(b"gone"))
-        .send()
-        .await?;
-    client
-        .delete_object()
-        .bucket(SOURCE_BUCKET)
-        .key("hidden/marker")
-        .send()
-        .await?;
-    let expected_sorted: Vec<String> = expected_keys.iter().cloned().collect();
-    let expected_prefixes: Vec<String> = (0..30).map(|directory| format!("d{directory:02}/")).collect();
-    let expected_top: Vec<String> = (0..100).map(|index| format!("top-{index:03}")).collect();
-
-    // Flat traversal in byte order, 1000 keys in pages of 7.
-    let (keys, prefixes, pages) = list_all(&client, None, None, None, 7).await?;
-    assert_eq!(keys, expected_sorted);
-    assert!(prefixes.is_empty());
-    assert_eq!(pages, 143);
-
-    // Delimiter folding: 30 common prefixes then 100 top-level keys, pages of 7.
-    let (keys, prefixes, pages) = list_all(&client, None, Some("/"), None, 7).await?;
-    assert_eq!(prefixes, expected_prefixes);
-    assert_eq!(keys, expected_top);
-    assert_eq!(pages, 19);
-
-    // Empty prefix equals no prefix.
-    let (keys, _, _) = list_all(&client, Some(""), None, None, 1000).await?;
-    assert_eq!(keys, expected_sorted);
-
-    // No match: empty, not truncated, no token.
-    let (keys, prefixes, pages) = list_all(&client, Some("zzz/"), Some("/"), None, 7).await?;
-    assert!(keys.is_empty() && prefixes.is_empty());
-    assert_eq!(pages, 1);
-    let (keys, _, _) = list_all(&client, Some("hidden/"), None, None, 7).await?;
-    assert!(keys.is_empty(), "a current delete marker must hide its key");
-
-    // Exact page boundary: 30 keys under one directory, max-keys=30 -> one
-    // untruncated page.
-    let (keys, prefixes, pages) = list_all(&client, Some("d05/"), Some("/"), None, 30).await?;
-    assert_eq!(keys.len(), 30);
-    assert!(prefixes.is_empty());
-    assert_eq!(pages, 1);
-
-    // start-after skips keys at or before the marker.
-    let (keys, _, _) = list_all(&client, None, None, Some("top-097"), 1000).await?;
-    assert_eq!(keys, ["top-098", "top-099"]);
-
-    // max-keys is clamped to 1000; exactly 1000 keys fit in one page.
-    let (keys, _, pages) = list_all(&client, None, None, None, 5000).await?;
-    assert_eq!(keys.len(), 1000);
-    assert_eq!(pages, 1);
-
-    let listings: Vec<_> = source
-        .requests()
-        .into_iter()
-        .filter(|record| record.operation == Operation::ListObjectsV2)
-        .collect();
-    assert!(listings.len() >= 143 + 19);
-    assert!(listings.iter().any(|record| record.prefix.as_deref() == Some("d05/")));
-    assert!(
-        listings.iter().any(|record| record.continuation_token.is_some()),
-        "resumed pages must journal their continuation token"
-    );
-    assert!(listings.iter().all(|record| record.user_agent.is_some()));
-    source.shutdown().await;
-    Ok(())
-}
-
-#[tokio::test]
-async fn fake_source_range_get_variants_and_416() -> TestResult {
-    let (source, client) = fake_source().await?;
-    let body = payload(1000);
-    source.put_seed_object(SOURCE_BUCKET, "ranged", body.clone(), &SeedMetadata::new());
-
-    for (range, expected_range, expected_slice) in [
-        ("bytes=10-19", "bytes 10-19/1000", &body[10..20]),
-        ("bytes=990-", "bytes 990-999/1000", &body[990..]),
-        ("bytes=-5", "bytes 995-999/1000", &body[995..]),
-        ("bytes=0-5000", "bytes 0-999/1000", &body[..]),
-    ] {
-        let output = client
-            .get_object()
-            .bucket(SOURCE_BUCKET)
-            .key("ranged")
-            .range(range)
-            .send()
-            .await?;
-        assert_eq!(output.content_range(), Some(expected_range), "{range}");
-        assert_eq!(output.accept_ranges(), Some("bytes"), "{range}");
-        assert_eq!(output.content_length(), Some(expected_slice.len() as i64), "{range}");
-        let collected = output.body.collect().await?.into_bytes();
-        assert_eq!(collected.as_ref(), expected_slice, "{range}");
-    }
-    let head = client
-        .head_object()
-        .bucket(SOURCE_BUCKET)
-        .key("ranged")
-        .range("bytes=10-19")
-        .send()
-        .await?;
-    assert_eq!(head.content_range(), Some("bytes 10-19/1000"));
-    assert_eq!(head.content_length(), Some(10));
-
-    for range in ["bytes=1000-", "bytes=-0"] {
-        let error = client
-            .get_object()
-            .bucket(SOURCE_BUCKET)
-            .key("ranged")
-            .range(range)
-            .send()
-            .await
-            .expect_err("unsatisfiable range must fail");
-        let response = error.raw_response().expect("416 must retain the raw response");
-        assert_eq!(response.status().as_u16(), 416, "{range}");
-        assert_eq!(response.headers().get("content-range"), Some("bytes */1000"), "{range}");
-        assert_eq!(error.code(), Some("InvalidRange"), "{range}");
-    }
-
-    let ranged = source
-        .requests()
-        .into_iter()
-        .find(|record| record.operation == Operation::GetObject && record.range.as_deref() == Some("bytes=10-19"))
-        .expect("the Range header must be journaled verbatim");
-    assert_eq!(ranged.key.as_deref(), Some("ranged"));
-    assert!(source.count_requests(Operation::GetObject, "ranged") >= 6);
-    source.shutdown().await;
-    Ok(())
-}
-
-#[tokio::test]
-async fn fake_source_unversioned_bucket_overwrites_and_deletes() -> TestResult {
-    let (source, client) = fake_source().await?;
-    source.create_bucket_with_mode("plain-source", BucketMode::Unversioned);
-    let versioning = client.get_bucket_versioning().bucket("plain-source").send().await?;
-    assert!(versioning.status().is_none(), "unversioned bucket must report no versioning status");
-
-    let first = client
-        .put_object()
-        .bucket("plain-source")
-        .key("doc")
-        .body(ByteStream::from_static(b"first"))
-        .send()
-        .await?;
-    assert!(first.version_id().is_none());
-    let second = client
-        .put_object()
-        .bucket("plain-source")
-        .key("doc")
-        .body(ByteStream::from_static(b"second"))
-        .send()
-        .await?;
-    assert!(second.version_id().is_none());
-    let get = client.get_object().bucket("plain-source").key("doc").send().await?;
-    assert!(get.version_id().is_none(), "GET must not return x-amz-version-id");
-    assert_eq!(get.body.collect().await?.into_bytes().as_ref(), b"second");
-    let head = client.head_object().bucket("plain-source").key("doc").send().await?;
-    assert!(head.version_id().is_none(), "HEAD must not return x-amz-version-id");
-    assert_eq!(source.stored_versions("plain-source", "doc").len(), 1, "overwrite must replace in place");
-
-    let deleted = client.delete_object().bucket("plain-source").key("doc").send().await?;
-    assert!(deleted.delete_marker().is_none() && deleted.version_id().is_none());
-    let missing = client
-        .get_object()
-        .bucket("plain-source")
-        .key("doc")
-        .send()
-        .await
-        .expect_err("deleted object must be gone");
-    assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
-    assert_eq!(missing.code(), Some("NoSuchKey"));
-    let missing_head = client
-        .head_object()
-        .bucket("plain-source")
-        .key("doc")
-        .send()
-        .await
-        .expect_err("deleted object must fail HEAD");
-    assert_eq!(missing_head.raw_response().map(|response| response.status().as_u16()), Some(404));
-    assert!(source.stored_versions("plain-source", "doc").is_empty(), "DELETE must not leave a marker");
-
-    // The versioned bucket on the same target keeps its version ids.
-    let versioned = client
-        .put_object()
-        .bucket(SOURCE_BUCKET)
-        .key("doc")
-        .body(ByteStream::from_static(b"versioned"))
-        .send()
-        .await?;
-    assert!(versioned.version_id().is_some());
-    source.shutdown().await;
-    Ok(())
-}
-
-#[tokio::test]
-async fn fake_source_replays_standard_and_user_metadata() -> TestResult {
-    let (source, client) = fake_source().await?;
-    let body = payload(4096);
-    let expected_etag = format!("\"{}\"", {
-        use md5::Digest as _;
-        hex_simd::encode_to_string(md5::Md5::digest(&body), hex_simd::AsciiCase::Lower)
-    });
-    // 2026-01-01T00:00:00Z rendered as an HTTP date by the SDK.
-    let expires = DateTime::from_secs(1_767_225_600);
-    client
-        .put_object()
-        .bucket(SOURCE_BUCKET)
-        .key("meta")
-        .body(ByteStream::from(body.clone()))
-        .content_type("application/x-odm")
-        .content_encoding("gzip")
-        .content_disposition("attachment; filename=\"meta.bin\"")
-        .content_language("en-US")
-        .cache_control("max-age=60")
-        .expires(expires)
-        .metadata("Foo-Bar", "mixed case name")
-        .metadata("UPPER", "upper name")
-        .metadata("already-lower", "lower name")
-        .send()
-        .await?;
-
-    let head = client.head_object().bucket(SOURCE_BUCKET).key("meta").send().await?;
-    let get = client.get_object().bucket(SOURCE_BUCKET).key("meta").send().await?;
-    for (label, content_type, content_encoding, content_disposition, content_language, cache_control, expires_string, e_tag) in [
-        (
-            "HEAD",
-            head.content_type(),
-            head.content_encoding(),
-            head.content_disposition(),
-            head.content_language(),
-            head.cache_control(),
-            head.expires_string(),
-            head.e_tag(),
-        ),
-        (
-            "GET",
-            get.content_type(),
-            get.content_encoding(),
-            get.content_disposition(),
-            get.content_language(),
-            get.cache_control(),
-            get.expires_string(),
-            get.e_tag(),
-        ),
-    ] {
-        assert_eq!(content_type, Some("application/x-odm"), "{label}");
-        assert_eq!(content_encoding, Some("gzip"), "{label}");
-        assert_eq!(content_disposition, Some("attachment; filename=\"meta.bin\""), "{label}");
-        assert_eq!(content_language, Some("en-US"), "{label}");
-        assert_eq!(cache_control, Some("max-age=60"), "{label}");
-        assert_eq!(expires_string, Some("Thu, 01 Jan 2026 00:00:00 GMT"), "{label}");
-        assert_eq!(e_tag, Some(expected_etag.as_str()), "{label}");
-    }
-    for metadata in [head.metadata(), get.metadata()] {
-        let metadata = metadata.expect("user metadata must be replayed");
-        assert_eq!(metadata.get("foo-bar").map(String::as_str), Some("mixed case name"));
-        assert_eq!(metadata.get("upper").map(String::as_str), Some("upper name"));
-        assert_eq!(metadata.get("already-lower").map(String::as_str), Some("lower name"));
-        assert!(!metadata.contains_key("Foo-Bar") && !metadata.contains_key("UPPER"));
-    }
-    assert!(head.last_modified().is_some());
-    assert_eq!(head.last_modified(), get.last_modified());
-    assert_eq!(head.content_length(), Some(4096));
-    assert_eq!(get.body.collect().await?.into_bytes(), body);
-
-    // Seeded objects replay the same way.
-    let seeded_etag = source.put_seed_object(
-        SOURCE_BUCKET,
-        "seeded",
-        Bytes::from_static(b"seeded"),
-        &SeedMetadata::new()
-            .content_type("text/plain")
-            .content_encoding("identity")
-            .cache_control("no-store")
-            .user_metadata("Origin", "seed"),
-    );
-    let seeded = client.head_object().bucket(SOURCE_BUCKET).key("seeded").send().await?;
-    assert_eq!(seeded.e_tag(), Some(format!("\"{seeded_etag}\"").as_str()));
-    assert_eq!(seeded.content_type(), Some("text/plain"));
-    assert_eq!(seeded.content_encoding(), Some("identity"));
-    assert_eq!(seeded.cache_control(), Some("no-store"));
-    assert_eq!(
-        seeded
-            .metadata()
-            .and_then(|metadata| metadata.get("origin"))
-            .map(String::as_str),
-        Some("seed")
-    );
-    source.shutdown().await;
-    Ok(())
-}
-
-#[tokio::test]
-async fn fake_source_fault_actions_truncate_stall_and_status() -> TestResult {
-    let (source, client) = fake_source().await?;
-    let body = payload(4096);
-    source.put_seed_object(SOURCE_BUCKET, "faulty", body.clone(), &SeedMetadata::new());
-
-    // TruncateBodyAt: headers promise 4096 bytes, the body ends after 100.
-    source.inject_for_key(Operation::GetObject, "faulty", FaultAction::TruncateBodyAt(100), 1);
-    let truncated = client.get_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
-    assert_eq!(truncated.content_length(), Some(4096));
-    let short_read = truncated
-        .body
-        .collect()
-        .await
-        .expect_err("a truncated body must fail to collect");
-    let short_read = short_read.to_string();
-    assert!(!short_read.is_empty());
-
-    // ResponseStatus: arbitrary status with the matching S3 error code.
-    for (code, expected_code) in [
-        (429u16, "SlowDown"),
-        (404, "NoSuchKey"),
-        (500, "InternalError"),
-        (503, "ServiceUnavailable"),
-    ] {
-        source.inject(Operation::GetObject, FaultAction::ResponseStatus(code), 1);
-        let error = client
-            .get_object()
-            .bucket(SOURCE_BUCKET)
-            .key("faulty")
-            .send()
-            .await
-            .expect_err("scripted status must fail");
-        assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(code));
-        assert_eq!(error.code(), Some(expected_code));
-    }
-
-    // Stall: the fully computed response is held before its first byte.
-    source.inject(Operation::HeadObject, FaultAction::Stall(Duration::from_millis(400)), 1);
-    let started = Instant::now();
-    let stalled = client.head_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
-    assert!(started.elapsed() >= Duration::from_millis(350), "stall must delay the first byte");
-    assert_eq!(stalled.content_length(), Some(4096));
-    let post_stall_started = Instant::now();
-    client.head_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
-    assert!(post_stall_started.elapsed() < Duration::from_millis(350), "stall is consumed once");
-
-    // The object is intact once the script is drained.
-    let intact = client.get_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
-    assert_eq!(intact.body.collect().await?.into_bytes(), body);
-
-    assert_eq!(source.count_requests(Operation::GetObject, "faulty"), 6);
-    assert_eq!(source.count_requests(Operation::HeadObject, "faulty"), 2);
-    assert_eq!(source.count_requests(Operation::GetObject, "other"), 0);
-    let records = source.requests();
-    assert!(
-        records.iter().all(|record| record
-            .user_agent
-            .as_deref()
-            .is_some_and(|agent| agent.contains("aws-sdk-rust"))),
-        "the SDK user agent must be journaled"
-    );
-    assert!(
-        records
-            .iter()
-            .any(|record| record.fault == Some(FaultAction::TruncateBodyAt(100)))
-    );
-    assert!(
-        records
-            .iter()
-            .any(|record| record.fault == Some(FaultAction::Stall(Duration::from_millis(400))))
-    );
-    source.shutdown().await;
-    Ok(())
-}
-
-#[tokio::test]
-async fn fake_source_raised_object_cap_accepts_large_put() -> TestResult {
-    let source = FakeS3Target::start_with_options(FakeS3TargetOptions {
-        max_object_bytes: 96 * 1024 * 1024,
-    })
-    .await?;
-    source.create_bucket(SOURCE_BUCKET);
-    let client = fake_source_client(&source);
-    let len = 64 * 1024 * 1024 + 1;
-    client
-        .put_object()
-        .bucket(SOURCE_BUCKET)
-        .key("large")
-        .body(ByteStream::from(vec![7u8; len]))
-        .send()
-        .await?;
-    let head = client.head_object().bucket(SOURCE_BUCKET).key("large").send().await?;
-    assert_eq!(head.content_length(), Some(len as i64));
-    let tail = client
-        .get_object()
-        .bucket(SOURCE_BUCKET)
-        .key("large")
-        .range("bytes=-1")
-        .send()
-        .await?;
-    assert_eq!(tail.content_range(), Some(format!("bytes {}-{}/{len}", len - 1, len - 1).as_str()));
-    source.shutdown().await;
-    Ok(())
-}
-
-#[tokio::test]
-async fn odm_env_starts_rustfs_and_fake_source() -> TestResult {
-    let env = OdmTestEnv::start().await?;
-    env.source.create_bucket(SOURCE_BUCKET);
-    let local_bucket = "odm-local";
-    env.rustfs.create_test_bucket(local_bucket).await?;
-
-    let etags = env.seed_source(
-        SOURCE_BUCKET,
-        &[
-            SeedObject::new("seed/a", Bytes::from_static(b"alpha")),
-            SeedObject::new("seed/b", Bytes::from_static(b"beta"))
-                .with_metadata(SeedMetadata::new().content_type("text/plain").user_metadata("Kind", "seed")),
-        ],
-    );
-    assert_eq!(etags.len(), 2);
-    assert!(env.source.requests().is_empty(), "seeding must not touch the journal");
-    let source_client = env.source_client();
-    let seeded = source_client.head_object().bucket(SOURCE_BUCKET).key("seed/b").send().await?;
-    assert_eq!(seeded.content_type(), Some("text/plain"));
-    assert_eq!(seeded.e_tag(), Some(format!("\"{}\"", etags[1]).as_str()));
-    assert_eq!(env.source.count_requests(Operation::HeadObject, "seed/b"), 1);
-
-    env.assert_local_absent(local_bucket, "seed/a").await;
-    env.client
-        .put_object()
-        .bucket(local_bucket)
-        .key("seed/a")
-        .body(ByteStream::from_static(b"alpha"))
-        .send()
-        .await?;
-    env.assert_local_present(local_bucket, "seed/a", b"alpha").await;
-    env.assert_local_absent(local_bucket, "seed/b").await;
-
-    let spec = env.fake_source_spec(SOURCE_BUCKET).to_json();
-    assert_eq!(spec["version"], 1);
-    assert_eq!(spec["enabled"], true);
-    assert_eq!(spec["source"]["provider"], "s3");
-    assert_eq!(spec["source"]["endpoint"], env.source.endpoint());
-    assert_eq!(spec["source"]["bucket"], SOURCE_BUCKET);
-    assert_eq!(spec["source"]["credentials"]["secret_key"], "fake-secret");
-    assert_eq!(spec["policy"]["source_timeout"]["first_byte_ms"], 15_000);
-    assert!(spec["policy"]["bandwidth_limit_bytes_per_sec"].is_null());
-    let debug = format!("{:?}", env.fake_source_spec(SOURCE_BUCKET));
-    assert!(!debug.contains("fake-secret"), "Debug output must redact the secret");
-    Ok(())
-}
-
-#[tokio::test]
-async fn start_source_rustfs_round_trips_put_get() -> TestResult {
-    let env = OdmTestEnv::start().await?;
-    let source = start_source_rustfs().await?;
-    assert_ne!(source.url, env.rustfs.url, "the source must be a separate instance");
-
-    source.create_test_bucket(SOURCE_BUCKET).await?;
-    let source_client = source.create_s3_client();
-    let body = payload(70_000);
-    let put = source_client
-        .put_object()
-        .bucket(SOURCE_BUCKET)
-        .key("real/object")
-        .body(ByteStream::from(body.clone()))
-        .content_type("application/octet-stream")
-        .send()
-        .await?;
-    assert!(put.e_tag().is_some());
-    let get = source_client
-        .get_object()
-        .bucket(SOURCE_BUCKET)
-        .key("real/object")
-        .send()
-        .await?;
-    assert_eq!(get.content_type(), Some("application/octet-stream"));
-    assert_eq!(get.body.collect().await?.into_bytes(), body);
-
-    let visible_to_primary = env
-        .client
-        .list_buckets()
-        .send()
-        .await?
-        .buckets()
-        .iter()
-        .any(|bucket| bucket.name() == Some(SOURCE_BUCKET));
-    assert!(!visible_to_primary, "the two servers must not share state");
-    let spec = super::common::OdmSourceSpec::for_rustfs_source(&source, SOURCE_BUCKET).to_json();
-    assert_eq!(spec["source"]["provider"], "rustfs");
-    assert_eq!(spec["source"]["endpoint"], source.url);
-    Ok(())
-}

crates/e2e_test/src/on_demand_migration/mod.rs

@@ -1,24 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! On-demand migration (ODM) end-to-end suite (rustfs/backlog#2147).
-//!
-//! `common` is the shared environment: one RustFS under test, one programmable
-//! fake S3 source, admin-API wrappers, seeding and local-state assertions.
-//! `harness_self_test` proves the harness itself; ODM behavior scenarios are
-//! separate modules wired by later tasks.
-
-pub mod common;
-
-mod harness_self_test;

crates/ecstore/src/api/mod.rs

@@ -128,6 +128,7 @@ pub mod bucket {
     }
 
     pub mod metadata {
+        pub use crate::bucket::metadata::BUCKET_DURABILITY_CONFIG;
         pub use crate::bucket::metadata::{
             BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG,
             BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_QUOTA_CONFIG_FILE,
@@ -136,7 +137,6 @@ pub mod bucket {
             BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BUCKET_WEBSITE_CONFIG, BucketMetadata, OBJECT_LOCK_CONFIG,
             load_bucket_metadata, table_catalog_path_hash,
         };
-        pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
     }
 
     pub mod durability {
@@ -145,21 +145,6 @@ pub mod bucket {
         };
     }
 
-    pub mod on_demand_migration {
-        pub use crate::bucket::on_demand_migration::{
-            ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION,
-            OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy,
-            SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
-        };
-        pub mod source_client {
-            pub use crate::bucket::on_demand_migration::source_client::{
-                SourceClient, SourceClientSpec, SourceError, SourceGet, SourceHead, SourceObject, SourcePage, SourceProbe,
-                SourceProvider, SourceSse, SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value,
-                resolve_path_style,
-            };
-        }
-    }
-
     pub mod metadata_sys {
         #[cfg(feature = "test-util")]
         pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
@@ -169,11 +154,11 @@ pub mod bucket {
             delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
             get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
             get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
-            get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, get_public_access_block_config,
-            get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config,
-            get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata,
-            remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock,
-            update_config_with, update_if_incarnation, update_quota_if_incarnation, update_under_transaction_lock,
+            get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config,
+            get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config,
+            get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
+            set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
+            update_quota_if_incarnation, update_under_transaction_lock,
         };
     }
 
@@ -214,13 +199,6 @@ pub mod bucket {
         }
     }
 
-    pub mod remote_s3_client {
-        pub use crate::bucket::remote_s3_client::{
-            PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, build_remote_s3_client,
-            validate_remote_endpoint,
-        };
-    }
-
     pub mod replication {
         pub use crate::bucket::replication::replication_pool::{
             DurableMrfBacklogSummary, DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBacklogObservabilitySummary,

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -15,13 +15,17 @@
 use crate::bucket::metadata::BucketMetadata;
 use crate::bucket::metadata_sys::get_bucket_targets_config;
 use crate::bucket::metadata_sys::get_replication_config;
-use crate::bucket::remote_s3_client::{PathStyle, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client};
 use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
 use crate::bucket::target::ARN;
 use crate::bucket::target::BucketTargetType;
 use crate::bucket::target::{self, BucketTarget, BucketTargets, Credentials};
 use crate::bucket::versioning_sys::BucketVersioningSys;
 use crate::runtime::sources as runtime_sources;
+use aws_credential_types::Credentials as SdkCredentials;
+use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future};
+use aws_sdk_s3::config::Region as SdkRegion;
+use aws_sdk_s3::config::RequestChecksumCalculation;
+use aws_sdk_s3::config::SharedHttpClient;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::error::SdkError;
 use aws_sdk_s3::operation::complete_multipart_upload::CompleteMultipartUploadOutput;
@@ -33,17 +37,28 @@ use aws_sdk_s3::operation::head_object::HeadObjectError;
 use aws_sdk_s3::operation::put_object_tagging::{PutObjectTaggingError, PutObjectTaggingOutput};
 use aws_sdk_s3::operation::upload_part::UploadPartOutput;
 use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::BucketVersioningStatus;
 use aws_sdk_s3::types::Tagging as SdkTagging;
 use aws_sdk_s3::types::{
     ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
     ServerSideEncryption,
 };
-use aws_sdk_s3::{Client as S3Client, operation::head_object::HeadObjectOutput};
-use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
+use aws_sdk_s3::{Client as S3Client, Config as S3Config, operation::head_object::HeadObjectOutput};
+use aws_sdk_s3::{config::SharedCredentialsProvider, types::BucketVersioningStatus};
+use aws_smithy_http_client::{Builder as SmithyHttpClientBuilder, tls as smithy_tls};
+use aws_smithy_runtime_api::box_error::BoxError;
+use aws_smithy_runtime_api::client::http::{
+    HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
+};
+use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
+use aws_smithy_runtime_api::client::result::ConnectorError;
+use aws_smithy_types::body::SdkBody;
 use futures::{StreamExt, stream};
-use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
+use http::{HeaderMap, HeaderName, HeaderValue, StatusCode, Uri};
+use hyper_util::client::legacy::Client as HyperClient;
+use hyper_util::rt::{TokioExecutor, TokioTimer};
 use reqwest::Client as HttpClient;
+use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RUSTFS_CA_CERT, RUSTFS_TLS_CERT};
+use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url};
 use rustfs_utils::http::{
     AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
     AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING_LOWER, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header,
@@ -55,10 +70,12 @@ use rustfs_utils::http::{
     SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID,
     insert_header,
 };
+use rustls_pki_types::pem::PemObject;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::error::Error;
 use std::fmt;
+use std::path::Path;
 use std::str::FromStr as _;
 use std::sync::Arc;
 use std::sync::OnceLock;
@@ -67,6 +84,7 @@ use std::time::{Duration, Instant, SystemTime};
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};
 use tokio::sync::Mutex;
 use tokio::sync::RwLock;
+use tower::Service;
 use tracing::error;
 use tracing::warn;
 use url::Url;
@@ -74,50 +92,72 @@ use uuid::Uuid;
 
 const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
 const REDACTED_CREDENTIAL: &str = "<redacted>";
+const EXPIRED_REMOTE_TARGET_CREDENTIALS: &str = "remote target credentials have expired";
 
-fn remote_credentials(credentials: &Credentials, account_id: &str) -> RemoteCredentials {
-    RemoteCredentials {
-        access_key: credentials.access_key.clone(),
-        secret_key: credentials.secret_key.clone(),
-        session_token: credentials.effective_session_token().map(str::to_string),
-        expiration: credentials.effective_expiration().map(SystemTime::from),
-        account_id: account_id.to_string(),
-    }
+#[derive(Clone)]
+struct RemoteTargetCredentialsProvider {
+    credentials: SdkCredentials,
 }
 
-fn target_path_style(path: &str) -> PathStyle {
-    match path.trim().to_ascii_lowercase().as_str() {
-        // Explicit DNS/virtual-hosted-style requested by user.
-        "dns" | "off" | "false" => PathStyle::VirtualHost,
-        // Explicit path-style or legacy boolean-like values.
-        "path" | "on" | "true" => PathStyle::Path,
-        // `auto` and empty are defaulted to path-style for custom S3-compatible endpoints.
-        "auto" | "" => PathStyle::Auto,
-        // Unknown values: prefer compatibility with S3-compatible services.
-        _ => PathStyle::Path,
-    }
-}
-
-impl From<&BucketTarget> for RemoteS3EndpointSpec {
-    fn from(target: &BucketTarget) -> Self {
-        RemoteS3EndpointSpec {
-            endpoint: target.endpoint.clone(),
-            secure: target.secure,
-            region: target.region.clone(),
-            path_style: target_path_style(&target.path),
-            credentials: target
-                .credentials
-                .as_ref()
-                .map(|credentials| remote_credentials(credentials, &target.reset_id)),
-            skip_tls_verify: target.skip_tls_verify,
-            ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
-            connect_timeout: None,
-            read_timeout: None,
-            user_agent_suffix: "",
+impl RemoteTargetCredentialsProvider {
+    fn resolve_at(&self, now: SystemTime) -> aws_credential_types::provider::Result {
+        if self.credentials.expiry().is_some_and(|expiration| expiration <= now) {
+            return Err(CredentialsError::provider_error(std::io::Error::other(EXPIRED_REMOTE_TARGET_CREDENTIALS)));
         }
+        Ok(self.credentials.clone())
     }
 }
 
+impl fmt::Debug for RemoteTargetCredentialsProvider {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("RemoteTargetCredentialsProvider")
+            .field("temporary", &self.credentials.session_token().is_some())
+            .field("expiration", &self.credentials.expiry())
+            .finish()
+    }
+}
+
+impl ProvideCredentials for RemoteTargetCredentialsProvider {
+    fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a>
+    where
+        Self: 'a,
+    {
+        future::ProvideCredentials::ready(self.resolve_at(SystemTime::now()))
+    }
+
+    fn fallback_on_interrupt(&self) -> Option<SdkCredentials> {
+        self.resolve_at(SystemTime::now()).ok()
+    }
+}
+
+fn remote_target_sdk_credentials(
+    credentials: &Credentials,
+    account_id: &str,
+    now: SystemTime,
+) -> Result<SdkCredentials, &'static str> {
+    let session_token = credentials.effective_session_token();
+    let expiration = credentials.effective_expiration().map(SystemTime::from);
+    if expiration.is_some() && session_token.is_none() {
+        return Err("remote target credential expiration requires a session token");
+    }
+    if expiration.is_some_and(|expiration| expiration <= now) {
+        return Err(EXPIRED_REMOTE_TARGET_CREDENTIALS);
+    }
+
+    let mut builder = SdkCredentials::builder()
+        .access_key_id(credentials.access_key.clone())
+        .secret_access_key(credentials.secret_key.clone())
+        .account_id(account_id.to_string())
+        .provider_name("bucket_target_sys");
+    if let Some(session_token) = session_token {
+        builder = builder.session_token(session_token.to_string());
+    }
+    if let Some(expiration) = expiration {
+        builder = builder.expiry(expiration);
+    }
+    Ok(builder.build())
+}
+
 pub type HeadObjectSdkError = Box<SdkError<HeadObjectError>>;
 pub type GetObjectSdkError = Box<SdkError<GetObjectError>>;
 pub type GetObjectTaggingSdkError = Box<SdkError<GetObjectTaggingError>>;
@@ -1018,17 +1058,57 @@ impl BucketTargetSys {
             });
         };
 
-        let spec = RemoteS3EndpointSpec::from(target);
-        let client = build_remote_s3_client(&spec)
-            .await
-            .map_err(|err| BucketTargetError::RemoteTargetConnectionErr {
+        let creds = remote_target_sdk_credentials(credentials, &target.reset_id, SystemTime::now()).map_err(|error| {
+            BucketTargetError::RemoteTargetConnectionErr {
                 bucket: target.target_bucket.clone(),
                 access_key: credentials.access_key.clone(),
-                error: err.to_string(),
-            })?;
+                error: error.to_string(),
+            }
+        })?;
+
+        let endpoint = if target.secure {
+            format!("https://{}", target.endpoint)
+        } else {
+            format!("http://{}", target.endpoint)
+        };
+        let parsed_endpoint = Url::parse(&endpoint).map_err(|err| BucketTargetError::RemoteTargetConnectionErr {
+            bucket: target.target_bucket.clone(),
+            access_key: credentials.access_key.clone(),
+            error: format!("invalid target endpoint: {err}"),
+        })?;
+        validate_replication_target_endpoint(&parsed_endpoint).map_err(|err| BucketTargetError::RemoteTargetConnectionErr {
+            bucket: target.target_bucket.clone(),
+            access_key: credentials.access_key.clone(),
+            error: format!("target endpoint is not allowed: {err}"),
+        })?;
+
+        let mut config_builder = S3Config::builder()
+            .endpoint_url(endpoint.clone())
+            .credentials_provider(SharedCredentialsProvider::new(RemoteTargetCredentialsProvider { credentials: creds }))
+            .region(SdkRegion::new(target.region.clone()))
+            .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
+            .request_checksum_calculation(replication_request_checksum_calculation());
+
+        if should_force_path_style(target) {
+            config_builder = config_builder.force_path_style(true);
+        }
+
+        if let Some(http_client) =
+            build_aws_s3_http_client_for_target(target)
+                .await
+                .map_err(|err| BucketTargetError::RemoteTargetConnectionErr {
+                    bucket: target.target_bucket.clone(),
+                    access_key: credentials.access_key.clone(),
+                    error: err.to_string(),
+                })?
+        {
+            config_builder = config_builder.http_client(http_client);
+        }
+
+        let config = config_builder.build();
 
         Ok(TargetClient {
-            endpoint: spec.endpoint_url(),
+            endpoint,
             credentials: target.credentials.clone(),
             bucket: target.target_bucket.clone(),
             storage_class: target.storage_class.clone(),
@@ -1038,7 +1118,7 @@ impl BucketTargetSys {
             secure: target.secure,
             health_check_duration: target.health_check_duration,
             replicate_sync: target.replication_sync,
-            client: Arc::new(client),
+            client: Arc::new(S3Client::from_conf(config)),
         })
     }
 
@@ -1201,6 +1281,327 @@ impl BucketTargetSys {
     }
 }
 
+#[derive(Debug)]
+struct AcceptAnyServerCertVerifier;
+
+impl rustls::client::danger::ServerCertVerifier for AcceptAnyServerCertVerifier {
+    fn verify_server_cert(
+        &self,
+        _end_entity: &rustls_pki_types::CertificateDer<'_>,
+        _intermediates: &[rustls_pki_types::CertificateDer<'_>],
+        _server_name: &rustls_pki_types::ServerName<'_>,
+        _ocsp_response: &[u8],
+        _now: rustls_pki_types::UnixTime,
+    ) -> Result<rustls::client::danger::ServerCertVerified, rustls::Error> {
+        Ok(rustls::client::danger::ServerCertVerified::assertion())
+    }
+
+    fn verify_tls12_signature(
+        &self,
+        _message: &[u8],
+        _cert: &rustls_pki_types::CertificateDer<'_>,
+        _dss: &rustls::DigitallySignedStruct,
+    ) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
+        Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
+    }
+
+    fn verify_tls13_signature(
+        &self,
+        _message: &[u8],
+        _cert: &rustls_pki_types::CertificateDer<'_>,
+        _dss: &rustls::DigitallySignedStruct,
+    ) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
+        Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
+    }
+
+    fn supported_verify_schemes(&self) -> Vec<rustls::SignatureScheme> {
+        rustls::crypto::aws_lc_rs::default_provider()
+            .signature_verification_algorithms
+            .supported_schemes()
+    }
+}
+
+#[derive(Clone)]
+struct TargetHyperHttpConnector<C> {
+    client: HyperClient<C, SdkBody>,
+}
+
+impl<C> fmt::Debug for TargetHyperHttpConnector<C> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("TargetHyperHttpConnector")
+            .field("client", &"** hyper client **")
+            .finish()
+    }
+}
+
+impl<C> SmithyHttpConnector for TargetHyperHttpConnector<C>
+where
+    C: Clone + Send + Sync + 'static,
+    C: Service<Uri>,
+    C::Response:
+        hyper::rt::Read + hyper::rt::Write + hyper_util::client::legacy::connect::Connection + Send + Sync + Unpin + 'static,
+    C::Future: Unpin + Send + 'static,
+    C::Error: Into<BoxError>,
+{
+    fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
+        let request = match request.try_into_http1x() {
+            Ok(request) => request,
+            Err(err) => return HttpConnectorFuture::ready(Err(ConnectorError::user(err.into()))),
+        };
+
+        let mut client = self.client.clone();
+        let fut = client.call(request);
+        HttpConnectorFuture::new(async move {
+            let response = fut
+                .await
+                .map_err(|err| ConnectorError::io(err.into()))?
+                .map(SdkBody::from_body_1_x);
+            HttpResponse::try_from(response).map_err(|err| ConnectorError::other(err.into(), None))
+        })
+    }
+}
+
+fn ensure_rustls_crypto_provider() {
+    if rustls::crypto::CryptoProvider::get_default().is_none() {
+        let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+    }
+}
+
+fn has_custom_ca_pem(target: &BucketTarget) -> bool {
+    !target.ca_cert_pem.trim().is_empty()
+}
+
+/// Env opt-in that re-enables loopback replication targets. Loopback (`127.0.0.1`,
+/// `::1`, `localhost`) is a classic SSRF vector and stays rejected by default, but
+/// single-host multi-instance dev setups and the e2e harness legitimately replicate
+/// over loopback. Never set this in production.
+const ALLOW_LOOPBACK_REPLICATION_TARGET_ENV: &str = "RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET";
+
+fn loopback_replication_targets_allowed() -> bool {
+    std::env::var(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV)
+        .map(|v| v.eq_ignore_ascii_case("true") || v == "1")
+        .unwrap_or(false)
+}
+
+const REPLICATION_STREAMING_CHECKSUMS_ENV: &str = "RUSTFS_REPLICATION_STREAMING_CHECKSUMS";
+
+/// Streaming trailer checksums make the SDK frame request bodies as
+/// `aws-chunked`; a target that does not decode that framing stores the frames
+/// verbatim, silently corrupting every replica while the transfer itself
+/// succeeds (#6853). Plain signed payloads are the compatible default; the env
+/// knob restores trailer checksums for fleets whose targets are all known to
+/// decode them.
+fn replication_request_checksum_calculation() -> RequestChecksumCalculation {
+    if std::env::var(REPLICATION_STREAMING_CHECKSUMS_ENV)
+        .map(|v| v.eq_ignore_ascii_case("true") || v == "1")
+        .unwrap_or(false)
+    {
+        RequestChecksumCalculation::WhenSupported
+    } else {
+        RequestChecksumCalculation::WhenRequired
+    }
+}
+
+fn validate_replication_target_endpoint(url: &Url) -> Result<(), OutboundUrlError> {
+    validate_replication_target_endpoint_inner(url, loopback_replication_targets_allowed())
+}
+
+fn validate_replication_target_endpoint_inner(url: &Url, allow_loopback: bool) -> Result<(), OutboundUrlError> {
+    match validate_outbound_url(url) {
+        Ok(()) => Ok(()),
+        // Replication targets are trusted infrastructure the operator configures, and
+        // legitimately live on private networks, so private addresses are always allowed.
+        Err(OutboundUrlError::ForbiddenHost {
+            reason: "private address",
+            ..
+        }) => Ok(()),
+        // Loopback is far higher SSRF risk, so it is allowed only under the explicit,
+        // off-by-default opt-in above (single-host multi-instance / the e2e harness).
+        Err(OutboundUrlError::ForbiddenHost {
+            reason: "loopback address" | "loopback host",
+            ..
+        }) if allow_loopback => Ok(()),
+        Err(err) => Err(err),
+    }
+}
+
+fn build_insecure_aws_s3_http_client() -> SharedHttpClient {
+    ensure_rustls_crypto_provider();
+
+    let tls_config = rustls::ClientConfig::builder()
+        .dangerous()
+        .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCertVerifier))
+        .with_no_client_auth();
+
+    let https = hyper_rustls::HttpsConnectorBuilder::new()
+        .with_tls_config(tls_config)
+        .https_or_http()
+        .enable_http1()
+        .enable_http2()
+        .build();
+    let mut client_builder = HyperClient::builder(TokioExecutor::new());
+    client_builder.pool_timer(TokioTimer::new());
+    let client = client_builder.build(https);
+    let connector = SharedHttpConnector::new(TargetHyperHttpConnector { client });
+
+    http_client_fn(move |_settings, _components| connector.clone())
+}
+
+fn validate_ca_pem_bundle(ca_cert_pem: &[u8]) -> Result<(), String> {
+    let certs = rustls_pki_types::CertificateDer::pem_slice_iter(ca_cert_pem)
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|err| format!("invalid PEM encoding: {err}"))?;
+
+    if certs.is_empty() {
+        return Err("no certificates found".to_string());
+    }
+
+    // Smithy's rustls adapter defers parsing custom certificates and assumes
+    // they are valid when the HTTPS connector is built. Validate every DER
+    // certificate first so malformed configuration is reported rather than
+    // reaching an `expect` in the dependency.
+    let mut validation_store = rustls::RootCertStore::empty();
+    for cert in certs {
+        validation_store
+            .add(cert)
+            .map_err(|err| format!("invalid X.509 certificate: {err}"))?;
+    }
+
+    Ok(())
+}
+
+fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), BucketTargetError> {
+    validate_ca_pem_bundle(ca_cert_pem.as_bytes())
+        .map_err(|err| BucketTargetError::Io(std::io::Error::other(format!("invalid target CA PEM: {err}"))))
+}
+
+fn compose_replication_trust_store(certificate_bundles: impl IntoIterator<Item = Vec<u8>>) -> (smithy_tls::TrustStore, usize) {
+    // `TrustStore::default()` keeps the platform-native roots enabled. Target
+    // and RUSTFS_TLS_PATH certificates extend that baseline instead of
+    // replacing it with a target-specific trust island.
+    let mut trust_store = smithy_tls::TrustStore::default();
+    let mut custom_bundle_count = 0;
+    for pem in certificate_bundles {
+        trust_store.add_pem_certificate(pem);
+        custom_bundle_count += 1;
+    }
+
+    (trust_store, custom_bundle_count)
+}
+
+fn build_aws_s3_http_client_with_trust_store(trust_store: smithy_tls::TrustStore) -> Result<SharedHttpClient, BucketTargetError> {
+    let tls_context = smithy_tls::TlsContext::builder()
+        .with_trust_store(trust_store)
+        .build()
+        .map_err(|err| BucketTargetError::Io(std::io::Error::other(format!("invalid target CA PEM: {err}"))))?;
+
+    Ok(SmithyHttpClientBuilder::new()
+        .tls_provider(smithy_tls::Provider::rustls(smithy_tls::rustls_provider::CryptoMode::AwsLc))
+        .tls_context(tls_context)
+        .build_https())
+}
+
+async fn load_tls_path_ca_bundles(tls_dir: &Path, trust_leaf_cert_as_ca: bool) -> Vec<Vec<u8>> {
+    let mut certificate_bundles = Vec::new();
+
+    let ca_path = tls_dir.join(RUSTFS_CA_CERT);
+    match tokio::fs::read(&ca_path).await {
+        Ok(pem) => match validate_ca_pem_bundle(&pem) {
+            Ok(()) => certificate_bundles.push(pem),
+            Err(err) => warn!("ignoring invalid custom CA bundle {:?} for replication client: {}", ca_path, err),
+        },
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+        Err(e) => warn!("failed to read custom CA bundle {:?} for replication client: {}", ca_path, e),
+    }
+
+    if trust_leaf_cert_as_ca {
+        let leaf_cert_path = tls_dir.join(RUSTFS_TLS_CERT);
+        match tokio::fs::read(&leaf_cert_path).await {
+            Ok(pem) => match validate_ca_pem_bundle(&pem) {
+                Ok(()) => certificate_bundles.push(pem),
+                Err(err) => warn!(
+                    "ignoring invalid leaf certificate {:?} for replication client trust store: {}",
+                    leaf_cert_path, err
+                ),
+            },
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+            Err(e) => warn!("failed to read leaf cert {:?} for replication client trust store: {}", leaf_cert_path, e),
+        }
+    }
+
+    certificate_bundles
+}
+
+async fn load_configured_tls_ca_bundles() -> Vec<Vec<u8>> {
+    let tls_path = rustfs_utils::get_env_str(rustfs_config::ENV_RUSTFS_TLS_PATH, rustfs_config::DEFAULT_RUSTFS_TLS_PATH);
+    if tls_path.is_empty() {
+        return Vec::new();
+    }
+
+    load_tls_path_ca_bundles(
+        Path::new(&tls_path),
+        rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA),
+    )
+    .await
+}
+
+async fn build_aws_s3_http_client_from_target_ca_pem(ca_cert_pem: &str) -> Result<SharedHttpClient, BucketTargetError> {
+    validate_target_ca_pem(ca_cert_pem)?;
+
+    let mut certificate_bundles = load_configured_tls_ca_bundles().await;
+    certificate_bundles.push(ca_cert_pem.as_bytes().to_vec());
+    let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
+
+    build_aws_s3_http_client_with_trust_store(trust_store)
+}
+
+async fn build_aws_s3_http_client_for_target(target: &BucketTarget) -> Result<Option<SharedHttpClient>, BucketTargetError> {
+    if !target.secure {
+        return Ok(None);
+    }
+
+    if target.skip_tls_verify {
+        return Ok(Some(build_insecure_aws_s3_http_client()));
+    }
+
+    if has_custom_ca_pem(target) {
+        return build_aws_s3_http_client_from_target_ca_pem(&target.ca_cert_pem)
+            .await
+            .map(Some);
+    }
+
+    Ok(build_aws_s3_http_client_from_tls_path().await)
+}
+
+async fn build_aws_s3_http_client_from_tls_path() -> Option<aws_sdk_s3::config::SharedHttpClient> {
+    let certificate_bundles = load_configured_tls_ca_bundles().await;
+    if certificate_bundles.is_empty() {
+        return None;
+    }
+
+    let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
+    match build_aws_s3_http_client_with_trust_store(trust_store) {
+        Ok(client) => Some(client),
+        Err(e) => {
+            warn!("failed to build AWS SDK TLS context for replication client: {}", e);
+            None
+        }
+    }
+}
+
+fn should_force_path_style(target: &BucketTarget) -> bool {
+    match target.path.trim().to_ascii_lowercase().as_str() {
+        // Explicit DNS/virtual-hosted-style requested by user.
+        "dns" | "off" | "false" => false,
+        // Explicit path-style or legacy boolean-like values.
+        "path" | "on" | "true" => true,
+        // `auto` and empty are defaulted to path-style for custom S3-compatible endpoints.
+        "auto" | "" => true,
+        // Unknown values: prefer compatibility with S3-compatible services.
+        _ => true,
+    }
+}
+
 // generate ARN that is unique to this target type
 fn generate_arn(t: &BucketTarget, depl_id: &str) -> String {
     let uuid = if depl_id.is_empty() {
@@ -2306,24 +2707,7 @@ impl Error for BucketTargetError {}
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::bucket::remote_s3_client::{
-        EXPIRED_REMOTE_TARGET_CREDENTIALS, RemoteTargetCredentialsProvider, build_aws_s3_http_client_for_spec,
-        build_aws_s3_http_client_from_target_ca_pem, build_aws_s3_http_client_with_trust_store,
-        build_insecure_aws_s3_http_client, compose_replication_trust_store, ensure_rustls_crypto_provider,
-        load_tls_path_ca_bundles, remote_sdk_credentials, replication_request_checksum_calculation,
-        validate_remote_endpoint_inner, validate_target_ca_pem,
-    };
-    use aws_credential_types::Credentials as SdkCredentials;
-    use aws_sdk_s3::Config as S3Config;
-    use aws_sdk_s3::config::{Region as SdkRegion, RequestChecksumCalculation, SharedCredentialsProvider, SharedHttpClient};
-    use aws_smithy_runtime_api::client::http::{
-        HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
-    };
-    use aws_smithy_runtime_api::client::orchestrator::HttpResponse;
-    use aws_smithy_types::body::SdkBody;
     use rcgen::generate_simple_self_signed;
-    use rustfs_config::{RUSTFS_CA_CERT, RUSTFS_TLS_CERT};
-    use rustfs_utils::egress::OutboundUrlError;
 
     // The startup panic fix for hosts without a CA bundle (issue #6734) rests
     // on two properties: the health-check client constructor never panics, and
@@ -2550,8 +2934,8 @@ mod tests {
             expiration: Some(jiff::Timestamp::try_from(expiration).expect("test expiration should convert")),
         };
 
-        let sdk_credentials = remote_sdk_credentials(&remote_credentials(&credentials, "account"), now)
-            .expect("unexpired temporary credentials should build");
+        let sdk_credentials =
+            remote_target_sdk_credentials(&credentials, "account", now).expect("unexpired temporary credentials should build");
 
         assert_eq!(sdk_credentials.session_token(), Some("temporary-session-token"));
         assert_eq!(sdk_credentials.expiry(), Some(expiration));
@@ -2567,7 +2951,7 @@ mod tests {
             expiration: Some("0001-01-01T00:00:00Z".parse().expect("Go zero time should parse")),
         };
 
-        let sdk_credentials = remote_sdk_credentials(&remote_credentials(&credentials, ""), SystemTime::now())
+        let sdk_credentials = remote_target_sdk_credentials(&credentials, "", SystemTime::now())
             .expect("Go zero expiration should remain compatible with static credentials");
 
         assert!(sdk_credentials.session_token().is_none());
@@ -2585,14 +2969,14 @@ mod tests {
         };
 
         assert_eq!(
-            remote_sdk_credentials(&remote_credentials(&credentials, ""), SystemTime::UNIX_EPOCH + Duration::from_secs(1_000))
+            remote_target_sdk_credentials(&credentials, "", SystemTime::UNIX_EPOCH + Duration::from_secs(1_000))
                 .expect_err("expiration without a session token must fail"),
             "remote target credential expiration requires a session token"
         );
 
         credentials.session_token = Some("temporary-session-token".to_string());
         assert_eq!(
-            remote_sdk_credentials(&remote_credentials(&credentials, ""), expiration)
+            remote_target_sdk_credentials(&credentials, "", expiration)
                 .expect_err("credentials expire at the exact expiration boundary"),
             EXPIRED_REMOTE_TARGET_CREDENTIALS
         );
@@ -2652,7 +3036,7 @@ mod tests {
             session_token: Some("temporary-session-token".to_string()),
             expiration: Some("2099-01-01T00:00:00Z".parse().expect("future expiration should parse")),
         };
-        let sdk_credentials = remote_sdk_credentials(&remote_credentials(&credentials, ""), SystemTime::now())
+        let sdk_credentials = remote_target_sdk_credentials(&credentials, "", SystemTime::now())
             .expect("unexpired temporary credentials should build");
         let client = S3Client::from_conf(
             S3Config::builder()
@@ -2827,46 +3211,6 @@ mod tests {
         assert!(!replication_target_versioning_enabled(None));
     }
 
-    #[test]
-    fn remote_endpoint_spec_from_target_keeps_legacy_path_style_and_trust_semantics() {
-        for (path, expected) in [
-            ("dns", PathStyle::VirtualHost),
-            ("OFF", PathStyle::VirtualHost),
-            ("false", PathStyle::VirtualHost),
-            ("path", PathStyle::Path),
-            ("on", PathStyle::Path),
-            ("true", PathStyle::Path),
-            (" auto ", PathStyle::Auto),
-            ("", PathStyle::Auto),
-            ("something-else", PathStyle::Path),
-        ] {
-            assert_eq!(target_path_style(path), expected, "path={path:?}");
-        }
-
-        let spec = RemoteS3EndpointSpec::from(&BucketTarget {
-            endpoint: "192.168.1.10:9000".to_string(),
-            secure: true,
-            region: "us-east-1".to_string(),
-            ca_cert_pem: "   ".to_string(),
-            reset_id: "reset-1".to_string(),
-            credentials: Some(Credentials {
-                access_key: "access".to_string(),
-                secret_key: "secret".to_string(),
-                session_token: Some("  ".to_string()),
-                expiration: Some("0001-01-01T00:00:00Z".parse().expect("Go zero time should parse")),
-            }),
-            ..Default::default()
-        });
-        assert_eq!(spec.endpoint_url(), "https://192.168.1.10:9000");
-        assert!(spec.ca_cert_pem.is_none(), "whitespace-only CA PEM means unset");
-        assert!(spec.connect_timeout.is_none() && spec.read_timeout.is_none());
-        assert_eq!(spec.user_agent_suffix, "");
-        let credentials = spec.credentials.expect("credentials carry over");
-        assert_eq!(credentials.account_id, "reset-1");
-        assert!(credentials.session_token.is_none(), "blank session token is absent");
-        assert!(credentials.expiration.is_none(), "Go zero expiration is absent");
-    }
-
     fn parse_url(raw: &str) -> Url {
         Url::parse(raw).expect("test URL should parse")
     }
@@ -2876,16 +3220,16 @@ mod tests {
         // Public hosts and private-network targets are allowed regardless of the
         // loopback opt-in — replication commonly runs across trusted private infra.
         for allow_loopback in [false, true] {
-            assert!(validate_remote_endpoint_inner(&parse_url("https://s3.example.com"), allow_loopback).is_ok());
-            assert!(validate_remote_endpoint_inner(&parse_url("http://10.0.0.5:9000"), allow_loopback).is_ok());
-            assert!(validate_remote_endpoint_inner(&parse_url("http://192.168.1.20"), allow_loopback).is_ok());
+            assert!(validate_replication_target_endpoint_inner(&parse_url("https://s3.example.com"), allow_loopback).is_ok());
+            assert!(validate_replication_target_endpoint_inner(&parse_url("http://10.0.0.5:9000"), allow_loopback).is_ok());
+            assert!(validate_replication_target_endpoint_inner(&parse_url("http://192.168.1.20"), allow_loopback).is_ok());
         }
     }
 
     #[test]
     fn replication_endpoint_rejects_loopback_without_opt_in() {
         // Default (production) behaviour: loopback IP and localhost host both rejected.
-        let err = validate_remote_endpoint_inner(&parse_url("http://127.0.0.1:9000"), false)
+        let err = validate_replication_target_endpoint_inner(&parse_url("http://127.0.0.1:9000"), false)
             .expect_err("loopback IP must be rejected by default");
         assert!(matches!(
             err,
@@ -2894,7 +3238,7 @@ mod tests {
                 ..
             }
         ));
-        let err = validate_remote_endpoint_inner(&parse_url("http://localhost:9000"), false)
+        let err = validate_replication_target_endpoint_inner(&parse_url("http://localhost:9000"), false)
             .expect_err("localhost must be rejected by default");
         assert!(matches!(
             err,
@@ -2909,15 +3253,15 @@ mod tests {
     fn replication_endpoint_allows_loopback_with_opt_in() {
         // e2e harness / single-host multi-instance: opt-in re-enables loopback in
         // both IP (127.0.0.1, ::1) and hostname (localhost) forms.
-        assert!(validate_remote_endpoint_inner(&parse_url("http://127.0.0.1:9000"), true).is_ok());
-        assert!(validate_remote_endpoint_inner(&parse_url("http://[::1]:9000"), true).is_ok());
-        assert!(validate_remote_endpoint_inner(&parse_url("http://localhost:9000"), true).is_ok());
+        assert!(validate_replication_target_endpoint_inner(&parse_url("http://127.0.0.1:9000"), true).is_ok());
+        assert!(validate_replication_target_endpoint_inner(&parse_url("http://[::1]:9000"), true).is_ok());
+        assert!(validate_replication_target_endpoint_inner(&parse_url("http://localhost:9000"), true).is_ok());
     }
 
     #[test]
     fn replication_endpoint_opt_in_does_not_open_other_ssrf_targets() {
         // The loopback opt-in must not widen into link-local / metadata endpoints.
-        let err = validate_remote_endpoint_inner(&parse_url("http://169.254.169.254/latest/meta-data"), true)
+        let err = validate_replication_target_endpoint_inner(&parse_url("http://169.254.169.254/latest/meta-data"), true)
             .expect_err("metadata endpoint must stay rejected even with loopback opt-in");
         assert!(matches!(
             err,
@@ -2926,7 +3270,7 @@ mod tests {
                 ..
             }
         ));
-        let err = validate_remote_endpoint_inner(&parse_url("http://[fe80::1]:9000"), true)
+        let err = validate_replication_target_endpoint_inner(&parse_url("http://[fe80::1]:9000"), true)
             .expect_err("link-local must stay rejected even with loopback opt-in");
         assert!(matches!(
             err,
@@ -3935,12 +4279,12 @@ mod tests {
 
     #[tokio::test]
     async fn skip_tls_verify_takes_priority_over_invalid_custom_ca_pem() {
-        let client = build_aws_s3_http_client_for_spec(&RemoteS3EndpointSpec::from(&BucketTarget {
+        let client = build_aws_s3_http_client_for_target(&BucketTarget {
             secure: true,
             skip_tls_verify: true,
             ca_cert_pem: "not a pem".to_string(),
             ..Default::default()
-        }))
+        })
         .await
         .expect("skip verification should bypass custom CA parsing");
 

crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs

@@ -1468,7 +1468,7 @@ async fn save_decommission_manifest_checkpoint_if_match(
         }
         let write_data = next_data.to_vec();
         let write = api
-            .run_decommission_capacity_temporary_mutation_with_capacity_lease(
+            .run_decommission_capacity_non_growing_replacement_with_capacity_lease(
                 target.target_pool_index,
                 Some(target.capacity_owner),
                 Some(next_data.len()),

crates/ecstore/src/bucket/metadata.rs

@@ -270,7 +270,6 @@ pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
 pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
 pub const BUCKET_TABLE_CONFIG: &str = "table-bucket.json";
 pub const BUCKET_DURABILITY_CONFIG: &str = "durability.json";
-pub const BUCKET_ON_DEMAND_MIGRATION_CONFIG: &str = "on-demand-migration.json";
 pub const BUCKET_TABLE_RESERVED_PREFIX: &str = ".rustfs-table";
 pub const BUCKET_TABLE_CATALOG_META_PREFIX: &str = "s3tables/catalog";
 pub const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str = "table-buckets";
@@ -322,7 +321,6 @@ pub struct BucketMetadata {
     pub bucket_acl_config_json: Vec<u8>,
     pub table_bucket_config_json: Vec<u8>,
     pub durability_config_json: Vec<u8>,
-    pub on_demand_migration_config_json: Vec<u8>,
 
     pub policy_config_updated_at: OffsetDateTime,
     pub object_lock_config_updated_at: OffsetDateTime,
@@ -344,7 +342,6 @@ pub struct BucketMetadata {
     pub bucket_acl_config_updated_at: OffsetDateTime,
     pub table_bucket_config_updated_at: OffsetDateTime,
     pub durability_config_updated_at: OffsetDateTime,
-    pub on_demand_migration_config_updated_at: OffsetDateTime,
 
     pub new_field_updated_at: OffsetDateTime,
 
@@ -396,7 +393,6 @@ impl Default for BucketMetadata {
             bucket_acl_config_json: Default::default(),
             table_bucket_config_json: Default::default(),
             durability_config_json: Default::default(),
-            on_demand_migration_config_json: Default::default(),
             policy_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH,
@@ -417,7 +413,6 @@ impl Default for BucketMetadata {
             bucket_acl_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             table_bucket_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             durability_config_updated_at: OffsetDateTime::UNIX_EPOCH,
-            on_demand_migration_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             new_field_updated_at: OffsetDateTime::UNIX_EPOCH,
             policy_config: Default::default(),
             notification_config: Default::default(),
@@ -482,23 +477,6 @@ impl BucketMetadata {
     /// Absent/empty/unparsable payloads all mean "no override" (the bucket
     /// follows the global durability mode); a parse failure is logged so a
     /// corrupted entry cannot silently change fsync behavior.
-    /// Parsed on-demand migration config, if one is stored.
-    ///
-    /// `Ok(None)` means no config (absent or cleared). A stored payload that
-    /// does not parse is an error, never a default: the runtime must not
-    /// pull from a source it cannot describe.
-    pub fn on_demand_migration_config(
-        &self,
-    ) -> std::result::Result<
-        Option<super::on_demand_migration::OnDemandMigrationConfig>,
-        super::on_demand_migration::OnDemandMigrationConfigError,
-    > {
-        if self.on_demand_migration_config_json.is_empty() {
-            return Ok(None);
-        }
-        super::on_demand_migration::OnDemandMigrationConfig::from_json(&self.on_demand_migration_config_json).map(Some)
-    }
-
     pub fn durability_config(&self) -> Option<super::durability::BucketDurabilityConfig> {
         if self.durability_config_json.is_empty() {
             return None;
@@ -577,9 +555,6 @@ impl BucketMetadata {
                 "BucketAclConfigJSON" | "BucketAclConfigJson" => self.bucket_acl_config_json = read_msgp_bin(rd)?,
                 "TableBucketConfigJSON" | "TableBucketConfigJson" => self.table_bucket_config_json = read_msgp_bin(rd)?,
                 "DurabilityConfigJSON" | "DurabilityConfigJson" => self.durability_config_json = read_msgp_bin(rd)?,
-                "OnDemandMigrationConfigJSON" | "OnDemandMigrationConfigJson" => {
-                    self.on_demand_migration_config_json = read_msgp_bin(rd)?
-                }
                 "CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?,
                 "LoggingConfigUpdatedAt" => self.logging_config_updated_at = read_msgp_time_value(rd)?,
                 "WebsiteConfigUpdatedAt" => self.website_config_updated_at = read_msgp_time_value(rd)?,
@@ -589,7 +564,6 @@ impl BucketMetadata {
                 "BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?,
                 "TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
                 "DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
-                "OnDemandMigrationConfigUpdatedAt" => self.on_demand_migration_config_updated_at = read_msgp_time_value(rd)?,
                 other => {
                     tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field");
                     skip_msgp_value(rd)?;
@@ -602,8 +576,8 @@ impl BucketMetadata {
 
     /// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
     pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
-        // Map size: MinIO fields (25) + RustFS extensions (21)
-        let map_len: u32 = 46;
+        // Map size: MinIO fields (25) + RustFS extensions (19)
+        let map_len: u32 = 44;
         rmp::encode::write_map_len(wr, map_len)?;
 
         // MinIO field order (same as Go struct)
@@ -663,7 +637,6 @@ impl BucketMetadata {
         write_bin_field(wr, "BucketAclConfigJSON", &self.bucket_acl_config_json)?;
         write_bin_field(wr, "TableBucketConfigJSON", &self.table_bucket_config_json)?;
         write_bin_field(wr, "DurabilityConfigJSON", &self.durability_config_json)?;
-        write_bin_field(wr, "OnDemandMigrationConfigJSON", &self.on_demand_migration_config_json)?;
         rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?;
         write_msgp_time(wr, self.cors_config_updated_at)?;
         rmp::encode::write_str(wr, "LoggingConfigUpdatedAt")?;
@@ -682,8 +655,6 @@ impl BucketMetadata {
         write_msgp_time(wr, self.table_bucket_config_updated_at)?;
         rmp::encode::write_str(wr, "DurabilityConfigUpdatedAt")?;
         write_msgp_time(wr, self.durability_config_updated_at)?;
-        rmp::encode::write_str(wr, "OnDemandMigrationConfigUpdatedAt")?;
-        write_msgp_time(wr, self.on_demand_migration_config_updated_at)?;
 
         Ok(())
     }
@@ -785,9 +756,6 @@ impl BucketMetadata {
         if self.durability_config_updated_at == OffsetDateTime::UNIX_EPOCH {
             self.durability_config_updated_at = self.created
         }
-        if self.on_demand_migration_config_updated_at == OffsetDateTime::UNIX_EPOCH {
-            self.on_demand_migration_config_updated_at = self.created
-        }
     }
 
     pub fn update_config(&mut self, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
@@ -903,17 +871,6 @@ impl BucketMetadata {
                 self.durability_config_json = data;
                 self.durability_config_updated_at = updated;
             }
-            BUCKET_ON_DEMAND_MIGRATION_CONFIG => {
-                // Structural check only (shape, unknown fields); the
-                // deployment-relative rules run in the admin handler with a
-                // `ValidationContext`. A blob this build cannot read must not
-                // be persisted for every later reader to trip over.
-                if !data.is_empty() {
-                    super::on_demand_migration::OnDemandMigrationConfig::from_json(&data).map_err(Error::other)?;
-                }
-                self.on_demand_migration_config_json = data;
-                self.on_demand_migration_config_updated_at = updated;
-            }
             _ => return Err(Error::other(format!("config file not found : {config_file}"))),
         }
 
@@ -1822,117 +1779,6 @@ mod test {
         assert!(!bm.table_bucket_enabled());
     }
 
-    const ODM_JSON: &[u8] = br#"{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
-
-    /// rustfs/backlog#2148: the on-demand migration config is a RustFS
-    /// extension entry that round-trips through `update_config` and the
-    /// msgpack codec, clears on delete, and never parses corruption into a
-    /// default.
-    #[test]
-    fn on_demand_migration_config_round_trips_and_tracks_updates() {
-        use crate::bucket::on_demand_migration::{OnDemandMigrationConfig, OnDemandMigrationConfigError};
-
-        let mut bm = BucketMetadata::new("odm-bucket");
-        assert_eq!(bm.on_demand_migration_config(), Ok(None), "fresh metadata carries no config");
-
-        let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
-        bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .expect("valid config is accepted");
-        assert_ne!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
-        assert_eq!(bm.on_demand_migration_config(), Ok(Some(expected.clone())));
-
-        let back = BucketMetadata::unmarshal(&bm.marshal_msg().unwrap()).unwrap();
-        assert_eq!(back.on_demand_migration_config_json, bm.on_demand_migration_config_json);
-        assert_eq!(
-            back.on_demand_migration_config_updated_at.unix_timestamp(),
-            bm.on_demand_migration_config_updated_at.unix_timestamp()
-        );
-        assert_eq!(back.on_demand_migration_config(), Ok(Some(expected)));
-
-        // A blob this build cannot read is rejected at the write boundary
-        // rather than persisted for every reader to trip over.
-        let before = bm.on_demand_migration_config_json.clone();
-        assert!(
-            bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec())
-                .is_err()
-        );
-        assert_eq!(bm.on_demand_migration_config_json, before, "a rejected update leaves the blob untouched");
-
-        // Delete clears the entry.
-        let stamped = bm.on_demand_migration_config_updated_at;
-        bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, Vec::new()).unwrap();
-        assert!(bm.on_demand_migration_config_json.is_empty());
-        assert_eq!(bm.on_demand_migration_config(), Ok(None));
-        assert!(bm.on_demand_migration_config_updated_at >= stamped);
-
-        // Corruption that bypassed `update_config` (disk, another writer)
-        // is a typed error, never a default.
-        bm.on_demand_migration_config_json = b"not-json".to_vec();
-        assert!(matches!(bm.on_demand_migration_config(), Err(OnDemandMigrationConfigError::Malformed(_))));
-    }
-
-    /// rustfs/backlog#2148: a `.metadata.bin` written before the on-demand
-    /// migration keys existed decodes with an empty blob and an epoch
-    /// timestamp that `default_timestamps` back-fills from `created`.
-    #[test]
-    fn on_demand_migration_config_absent_in_legacy_blob_defaults_to_created() {
-        let blob = decode_hex(include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex"));
-        let mut bm = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal MinIO bucket metadata");
-        assert!(bm.on_demand_migration_config_json.is_empty());
-        assert_eq!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
-        assert_eq!(bm.on_demand_migration_config(), Ok(None));
-
-        bm.default_timestamps();
-        assert_ne!(bm.created, OffsetDateTime::UNIX_EPOCH, "fixture must carry a real creation time");
-        assert_eq!(bm.on_demand_migration_config_updated_at, bm.created);
-
-        // A metadata blob from this build with no config set stays
-        // indistinguishable from the legacy one for these fields.
-        let fresh = BucketMetadata::unmarshal(&BucketMetadata::new("fresh").marshal_msg().unwrap()).unwrap();
-        assert!(fresh.on_demand_migration_config_json.is_empty());
-        assert_eq!(fresh.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
-    }
-
-    /// rustfs/backlog#2148: a reader that predates the two on-demand
-    /// migration keys takes `decode_from`'s unknown-field branch, which is
-    /// `skip_msgp_value`. Walk the new-format blob with exactly that
-    /// primitive and prove both keys are skipped without desynchronising the
-    /// stream, so the fields that follow them still decode.
-    #[test]
-    fn old_decoder_skips_on_demand_migration_fields_without_desync() {
-        let mut bm = BucketMetadata::new("odm-skip");
-        bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .unwrap();
-        bm.update_config(BUCKET_DURABILITY_CONFIG, br#"{"mode":"relaxed"}"#.to_vec())
-            .unwrap();
-        let buf = bm.marshal_msg().unwrap();
-
-        let mut rd = std::io::Cursor::new(buf.as_slice());
-        let fields = rmp::decode::read_map_len(&mut rd).unwrap();
-        let mut skipped = Vec::new();
-        let mut durability_json = Vec::new();
-        for _ in 0..fields {
-            let key_len = rmp::decode::read_str_len(&mut rd).unwrap();
-            let mut key = vec![0u8; key_len as usize];
-            rd.read_exact(&mut key).unwrap();
-            let key = String::from_utf8(key).unwrap();
-            match key.as_str() {
-                // The field an old reader knows that is encoded *after* the
-                // unknown JSON key and *before* the unknown timestamp key.
-                "DurabilityConfigJSON" => durability_json = read_msgp_bin(&mut rd).unwrap(),
-                other => {
-                    if other.starts_with("OnDemandMigration") {
-                        skipped.push(other.to_string());
-                    }
-                    skip_msgp_value(&mut rd).unwrap();
-                }
-            }
-        }
-        assert_eq!(skipped, ["OnDemandMigrationConfigJSON", "OnDemandMigrationConfigUpdatedAt"]);
-        assert_eq!(durability_json, br#"{"mode":"relaxed"}"#);
-        assert_eq!(rd.position() as usize, buf.len(), "old-style walk must consume the blob exactly");
-    }
-
     /// HP-5b (rustfs/backlog#938): the durability override is a RustFS
     /// extension entry and must survive an encode/decode round trip.
     #[test]

crates/ecstore/src/bucket/metadata_sys.rs

@@ -19,7 +19,6 @@ use super::quota::BucketQuota;
 use super::target::BucketTargets;
 use crate::bucket::bucket_target_sys::BucketTargetSys;
 use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_parse_with_presence};
-use crate::bucket::on_demand_migration::{ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig};
 use crate::bucket::utils::is_meta_bucketname;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
@@ -385,42 +384,6 @@ fn clear_bucket_durability(bucket: &str) {
     crate::disk::local::bucket_durability::set(bucket, None);
 }
 
-/// Publish the bucket's on-demand migration config (or its absence) to the
-/// runtime registered in `ON_DEMAND_MIGRATION_CONFIG_HOOK`.
-///
-/// Called from the same five cache-install paths as
-/// [`sync_bucket_durability`]. A stored payload this build cannot parse is
-/// published as `None`: the runtime must stop pulling for that bucket rather
-/// than keep an older config or guess.
-fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
-    let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() else {
-        return;
-    };
-    match bm.on_demand_migration_config() {
-        Ok(config) => hook(bucket, config.as_ref()),
-        Err(err) => {
-            warn!(
-                event = "bucket_metadata_parse_failed",
-                component = "ecstore",
-                subsystem = "bucket_metadata",
-                bucket = %bucket,
-                config = "on_demand_migration",
-                error = %err,
-                "Failed to parse bucket metadata config"
-            );
-            hook(bucket, None);
-        }
-    }
-}
-
-/// Withdraw a bucket's on-demand migration config when its metadata leaves
-/// the cache.
-fn clear_on_demand_migration(bucket: &str) {
-    if let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() {
-        hook(bucket, None);
-    }
-}
-
 pub async fn get(bucket: &str) -> Result<Arc<BucketMetadata>> {
     let sys = get_bucket_metadata_sys()?;
     let lock = sys.read().await;
@@ -1007,16 +970,6 @@ pub async fn get_durability_config(
     Ok((bm.durability_config(), bm.durability_config_updated_at))
 }
 
-/// The bucket's on-demand migration config with its update time, or
-/// `Ok(None)` when the bucket has none. A stored payload that does not parse
-/// is a typed error (`OnDemandMigrationConfigError` inside `Error::Io`).
-pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
-    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
-    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
-
-    bucket_meta_sys.get_on_demand_migration_config(bucket).await
-}
-
 pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -1539,7 +1492,6 @@ impl BucketMetadataSys {
                 if removed {
                     BucketTargetSys::get().delete(bucket).await;
                     clear_bucket_durability(bucket);
-                    clear_on_demand_migration(bucket);
                 }
             }
             return Ok(());
@@ -1577,7 +1529,6 @@ impl BucketMetadataSys {
                 self.missing_buckets.invalidate(bucket).await;
                 sync_bucket_target_sys(bucket, &bm).await;
                 sync_bucket_durability(bucket, &bm);
-                sync_on_demand_migration(bucket, &bm);
             }
             MetadataLoadMode::Initial => {
                 let _publish_guard = self
@@ -1624,7 +1575,6 @@ impl BucketMetadataSys {
             if removed {
                 BucketTargetSys::get().delete(bucket).await;
                 clear_bucket_durability(bucket);
-                clear_on_demand_migration(bucket);
             }
             return Ok(());
         }
@@ -1647,7 +1597,6 @@ impl BucketMetadataSys {
         self.missing_buckets.invalidate(bucket).await;
         sync_bucket_target_sys(bucket, &metadata).await;
         sync_bucket_durability(bucket, &metadata);
-        sync_on_demand_migration(bucket, &metadata);
         Ok(())
     }
 
@@ -1675,7 +1624,6 @@ impl BucketMetadataSys {
             self.missing_buckets.invalidate(&bucket).await;
             sync_bucket_target_sys(&bucket, &bm).await;
             sync_bucket_durability(&bucket, &bm);
-            sync_on_demand_migration(&bucket, &bm);
         }
     }
 
@@ -1696,7 +1644,6 @@ impl BucketMetadataSys {
         if removed {
             BucketTargetSys::get().delete(bucket).await;
             clear_bucket_durability(bucket);
-            clear_on_demand_migration(bucket);
         }
         removed || removed_fabricated
     }
@@ -1986,7 +1933,6 @@ impl BucketMetadataSys {
                 self.missing_buckets.invalidate(bucket).await;
                 sync_bucket_target_sys(bucket, &bm).await;
                 sync_bucket_durability(bucket, &bm);
-                sync_on_demand_migration(bucket, &bm);
             } else {
                 let exists = self
                     .bucket_exists(bucket, &guard, "lazy bucket metadata existence check")
@@ -2325,7 +2271,6 @@ impl BucketMetadataSys {
         self.missing_buckets.invalidate(bucket).await;
         sync_bucket_target_sys(bucket, &metadata).await;
         sync_bucket_durability(bucket, &metadata);
-        sync_on_demand_migration(bucket, &metadata);
         Ok(BucketMetadataAuthority::Authoritative(metadata))
     }
 
@@ -2518,17 +2463,6 @@ impl BucketMetadataSys {
             Err(Error::ConfigNotFound)
         }
     }
-
-    /// See [`get_on_demand_migration_config`].
-    pub async fn get_on_demand_migration_config(
-        &self,
-        bucket: &str,
-    ) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
-        let (bm, _) = self.get_config(bucket).await?;
-
-        let config = bm.on_demand_migration_config().map_err(Error::other)?;
-        Ok(config.map(|config| (config, bm.on_demand_migration_config_updated_at)))
-    }
 }
 
 /// Test-only fixture shared with sibling modules (e.g. the quota checker
@@ -4109,151 +4043,6 @@ mod tests {
         assert_eq!(bucket_durability::lookup(bucket), None);
     }
 
-    const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
-
-    /// Every `(bucket, config)` the recording hook has seen. Tests filter by
-    /// their own bucket name; the hook is process-wide and set once.
-    static ODM_HOOK_CALLS: std::sync::Mutex<Vec<(String, Option<OnDemandMigrationConfig>)>> = std::sync::Mutex::new(Vec::new());
-
-    fn install_recording_odm_hook() {
-        ON_DEMAND_MIGRATION_CONFIG_HOOK.get_or_init(|| {
-            Box::new(|bucket, config| {
-                ODM_HOOK_CALLS.lock().unwrap().push((bucket.to_string(), config.cloned()));
-            })
-        });
-    }
-
-    fn odm_hook_calls(bucket: &str) -> Vec<Option<OnDemandMigrationConfig>> {
-        ODM_HOOK_CALLS
-            .lock()
-            .unwrap()
-            .iter()
-            .filter(|(name, _)| name == bucket)
-            .map(|(_, config)| config.clone())
-            .collect()
-    }
-
-    /// rustfs/backlog#2148: the accessor reports absence as `Ok(None)` and a
-    /// stored payload it cannot parse as a typed error, never as a default
-    /// and never as `ConfigNotFound`.
-    #[tokio::test]
-    async fn get_on_demand_migration_config_distinguishes_absent_from_corrupt() {
-        use crate::bucket::on_demand_migration::OnDemandMigrationConfigError;
-
-        let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
-        let sys = BucketMetadataSys::new(ecstore);
-        let bucket = "odm-accessor";
-
-        sys.set(bucket.to_string(), Arc::new(BucketMetadata::new(bucket))).await;
-        assert_eq!(sys.get_on_demand_migration_config(bucket).await.unwrap(), None);
-
-        let mut corrupt = BucketMetadata::new(bucket);
-        corrupt.on_demand_migration_config_json = br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec();
-        sys.set(bucket.to_string(), Arc::new(corrupt)).await;
-        let err = sys
-            .get_on_demand_migration_config(bucket)
-            .await
-            .expect_err("corrupt config must not read as a default");
-        assert_ne!(err, Error::ConfigNotFound, "corruption must not be reported as absence");
-        let typed = match &err {
-            Error::Io(io) => io
-                .get_ref()
-                .and_then(|source| source.downcast_ref::<OnDemandMigrationConfigError>()),
-            _ => None,
-        };
-        assert!(
-            matches!(typed, Some(OnDemandMigrationConfigError::Malformed(_))),
-            "typed parse error must survive the Result boundary, got: {err:?}"
-        );
-
-        let mut valid = BucketMetadata::new(bucket);
-        valid
-            .update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .unwrap();
-        let stamped = valid.on_demand_migration_config_updated_at;
-        sys.set(bucket.to_string(), Arc::new(valid)).await;
-        let (config, updated_at) = sys
-            .get_on_demand_migration_config(bucket)
-            .await
-            .unwrap()
-            .expect("stored config is returned");
-        assert_eq!(config, OnDemandMigrationConfig::from_json(ODM_JSON).unwrap());
-        assert_eq!(updated_at, stamped);
-    }
-
-    /// rustfs/backlog#2148: the publish hook fires on every path that
-    /// installs bucket metadata into the cache (set, initial load, peer
-    /// reload, refresh loop, lazy load) and withdraws on removal, mirroring
-    /// `sync_bucket_durability`.
-    #[tokio::test]
-    async fn on_demand_migration_hook_fires_on_every_cache_install_path() {
-        install_recording_odm_hook();
-
-        let (dirs, ecstore) = isolated_store_over_temp_disks().await;
-        let bucket = "odm-hook-paths";
-        for dir in &dirs {
-            std::fs::create_dir_all(dir.path().join(bucket)).expect("physical bucket should exist");
-        }
-        let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
-        let expect_publish = |before: usize, label: &str| {
-            let calls = odm_hook_calls(bucket);
-            assert_eq!(calls.len(), before + 1, "{label} must publish exactly once");
-            assert_eq!(calls.last().unwrap().as_ref(), Some(&expected), "{label} must publish the stored config");
-        };
-
-        // set (via persist_new_and_set, which installs through `set`).
-        let mut bm = BucketMetadata::new(bucket);
-        bm.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .unwrap();
-        let writer = BucketMetadataSys::new(ecstore.clone());
-        let before = odm_hook_calls(bucket).len();
-        writer.persist_new_and_set(bm).await.expect("metadata should persist");
-        expect_publish(before, "set");
-
-        // init (initial load on a cold system).
-        let mut cold = BucketMetadataSys::new(ecstore.clone());
-        let before = odm_hook_calls(bucket).len();
-        cold.init(vec![bucket.to_string()]).await;
-        assert!(cold.get(bucket).await.is_ok(), "initial load must cache the bucket");
-        expect_publish(before, "init");
-
-        // peer reload.
-        let before = odm_hook_calls(bucket).len();
-        cold.reload_from_store(bucket).await.expect("peer reload should publish");
-        expect_publish(before, "peer reload");
-
-        // refresh loop.
-        let before = odm_hook_calls(bucket).len();
-        let mut failed = HashSet::new();
-        cold.concurrent_load(&[bucket.to_string()], &mut failed, MetadataLoadMode::Refresh)
-            .await;
-        assert!(failed.is_empty(), "refresh must succeed");
-        expect_publish(before, "refresh loop");
-
-        // lazy load on another cold system.
-        let lazy = BucketMetadataSys::new(ecstore);
-        let before = odm_hook_calls(bucket).len();
-        let (_, loaded) = lazy.get_config(bucket).await.expect("lazy load should publish");
-        assert!(loaded, "the lazy path must have gone to disk");
-        expect_publish(before, "lazy load");
-
-        // Removal withdraws the config.
-        let before = odm_hook_calls(bucket).len();
-        assert!(lazy.remove(bucket).await);
-        let calls = odm_hook_calls(bucket);
-        assert_eq!(calls.len(), before + 1, "remove must withdraw exactly once");
-        assert_eq!(calls.last().unwrap(), &None);
-
-        // A corrupt payload is withdrawn, never published as a config.
-        let mut corrupt = BucketMetadata::new(bucket);
-        corrupt.on_demand_migration_config_json = b"not-json".to_vec();
-        let before = odm_hook_calls(bucket).len();
-        lazy.set(bucket.to_string(), Arc::new(corrupt)).await;
-        let calls = odm_hook_calls(bucket);
-        assert_eq!(calls.len(), before + 1);
-        assert_eq!(calls.last().unwrap(), &None, "unreadable config must publish absence");
-    }
-
     #[tokio::test]
     async fn refresh_wait_exits_when_cancelled() {
         let cancel_token = CancellationToken::new();

crates/ecstore/src/bucket/mod.rs

@@ -26,10 +26,8 @@ mod metadata_test;
 pub mod migration;
 mod msgp_decode;
 pub mod object_lock;
-pub mod on_demand_migration;
 pub mod policy_sys;
 pub mod quota;
-pub mod remote_s3_client;
 pub mod replication;
 pub mod tagging;
 pub mod target;

crates/ecstore/src/bucket/on_demand_migration/mod.rs

@@ -1,28 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! On-Demand Migration (ODM): a bucket can name an external S3-compatible
-//! source bucket; GET misses are served from that source and backfilled
-//! locally. This module owns the bucket-level configuration model
-//! (`on-demand-migration.json` in the bucket metadata file); the runtime is
-//! layered on top of it by later tasks (rustfs/backlog#2147).
-
-pub mod config;
-pub mod source_client;
-
-pub use config::{
-    ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION,
-    OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, SourceConfig,
-    SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
-};

crates/ecstore/src/bucket/remote_s3_client.rs

@@ -1,789 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Shared builder for outbound `aws_sdk_s3::Client`s.
-//!
-//! Replication targets (`bucket_target_sys`) and the on-demand migration
-//! source client build their remote clients from one neutral
-//! [`RemoteS3EndpointSpec`]: endpoint assembly, credential handling, path-style
-//! selection, custom CA / skip-TLS transports and the outbound SSRF gate all
-//! live here so both callers share exactly one policy. The gate keeps the
-//! relaxed replication semantics documented in
-//! `docs/operations/outbound-connection-policy.md`: private addresses are
-//! always allowed, loopback only behind `RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET`.
-
-use aws_credential_types::Credentials as SdkCredentials;
-use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future};
-use aws_sdk_s3::config::Region as SdkRegion;
-use aws_sdk_s3::config::RequestChecksumCalculation;
-use aws_sdk_s3::config::SharedCredentialsProvider;
-use aws_sdk_s3::config::SharedHttpClient;
-use aws_sdk_s3::{Client as S3Client, Config as S3Config};
-use aws_smithy_http_client::{Builder as SmithyHttpClientBuilder, tls as smithy_tls};
-use aws_smithy_runtime_api::box_error::BoxError;
-use aws_smithy_runtime_api::client::http::{
-    HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
-};
-use aws_smithy_runtime_api::client::interceptors::Intercept;
-use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
-use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
-use aws_smithy_runtime_api::client::result::ConnectorError;
-use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
-use aws_smithy_types::body::SdkBody;
-use aws_smithy_types::config_bag::ConfigBag;
-use aws_smithy_types::timeout::TimeoutConfig;
-use http::Uri;
-use hyper_util::client::legacy::Client as HyperClient;
-use hyper_util::rt::{TokioExecutor, TokioTimer};
-use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RUSTFS_CA_CERT, RUSTFS_TLS_CERT};
-use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url};
-use rustls_pki_types::pem::PemObject;
-use std::fmt;
-use std::path::Path;
-use std::sync::Arc;
-use std::time::{Duration, SystemTime};
-use tower::Service;
-use tracing::warn;
-use url::Url;
-
-const REDACTED_CREDENTIAL: &str = "<redacted>";
-pub(crate) const EXPIRED_REMOTE_TARGET_CREDENTIALS: &str = "remote target credentials have expired";
-
-/// Request addressing style for a remote S3-compatible endpoint.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum PathStyle {
-    /// Caller did not choose; the builder defaults to path-style because that
-    /// is what custom S3-compatible endpoints accept most reliably.
-    Auto,
-    /// `https://endpoint/bucket/key`.
-    Path,
-    /// `https://bucket.endpoint/key`.
-    VirtualHost,
-}
-
-impl PathStyle {
-    /// Resolves the style to the SDK `force_path_style` flag. `Auto` keeps
-    /// the historical replication default (path-style).
-    pub fn force_path_style(self) -> bool {
-        !matches!(self, PathStyle::VirtualHost)
-    }
-}
-
-/// Static or temporary credentials for a remote endpoint. `expiration` without
-/// a `session_token` is rejected at build time: only STS-style temporary
-/// credentials expire, so that combination is a corrupted configuration
-/// rather than a static key.
-#[derive(Clone)]
-pub struct RemoteCredentials {
-    pub access_key: String,
-    pub secret_key: String,
-    pub session_token: Option<String>,
-    pub expiration: Option<SystemTime>,
-    /// SDK credential `account_id`; replication targets pass their reset id.
-    pub account_id: String,
-}
-
-impl fmt::Debug for RemoteCredentials {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("RemoteCredentials")
-            .field("access_key", &self.access_key)
-            .field("secret_key", &REDACTED_CREDENTIAL)
-            .field("session_token", &self.session_token.as_ref().map(|_| REDACTED_CREDENTIAL))
-            .field("expiration", &self.expiration)
-            .field("account_id", &self.account_id)
-            .finish()
-    }
-}
-
-/// Neutral description of a remote S3 endpoint from which an
-/// `aws_sdk_s3::Client` is built.
-#[derive(Clone, Debug)]
-pub struct RemoteS3EndpointSpec {
-    /// `host[:port]` without a scheme; `secure` selects `https` or `http`.
-    pub endpoint: String,
-    pub secure: bool,
-    pub region: String,
-    pub path_style: PathStyle,
-    pub credentials: Option<RemoteCredentials>,
-    /// Accept any server certificate. Takes priority over `ca_cert_pem`.
-    pub skip_tls_verify: bool,
-    /// Extra PEM bundle trusted alongside the platform roots and the
-    /// `RUSTFS_TLS_PATH` bundle. `None` and whitespace-only mean "not set".
-    pub ca_cert_pem: Option<String>,
-    pub connect_timeout: Option<Duration>,
-    pub read_timeout: Option<Duration>,
-    /// Appended to the SDK `User-Agent` (space separated) so the remote side
-    /// can identify the caller; empty means no suffix.
-    pub user_agent_suffix: &'static str,
-}
-
-impl RemoteS3EndpointSpec {
-    /// Full endpoint URL (`scheme://host[:port]`) as handed to the SDK.
-    pub fn endpoint_url(&self) -> String {
-        if self.secure {
-            format!("https://{}", self.endpoint)
-        } else {
-            format!("http://{}", self.endpoint)
-        }
-    }
-
-    fn custom_ca_pem(&self) -> Option<&str> {
-        self.ca_cert_pem.as_deref().filter(|pem| !pem.trim().is_empty())
-    }
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum RemoteS3ClientError {
-    #[error("remote endpoint requires credentials")]
-    MissingCredentials,
-    #[error("{0}")]
-    Credentials(&'static str),
-    #[error("invalid target endpoint: {0}")]
-    InvalidEndpoint(String),
-    #[error("target endpoint is not allowed: {0}")]
-    EndpointNotAllowed(#[source] OutboundUrlError),
-    #[error("invalid target CA PEM: {0}")]
-    InvalidCaPem(String),
-}
-
-#[derive(Clone)]
-pub(crate) struct RemoteTargetCredentialsProvider {
-    pub(crate) credentials: SdkCredentials,
-}
-
-impl RemoteTargetCredentialsProvider {
-    pub(crate) fn resolve_at(&self, now: SystemTime) -> aws_credential_types::provider::Result {
-        if self.credentials.expiry().is_some_and(|expiration| expiration <= now) {
-            return Err(CredentialsError::provider_error(std::io::Error::other(EXPIRED_REMOTE_TARGET_CREDENTIALS)));
-        }
-        Ok(self.credentials.clone())
-    }
-}
-
-impl fmt::Debug for RemoteTargetCredentialsProvider {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("RemoteTargetCredentialsProvider")
-            .field("temporary", &self.credentials.session_token().is_some())
-            .field("expiration", &self.credentials.expiry())
-            .finish()
-    }
-}
-
-impl ProvideCredentials for RemoteTargetCredentialsProvider {
-    fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a>
-    where
-        Self: 'a,
-    {
-        future::ProvideCredentials::ready(self.resolve_at(SystemTime::now()))
-    }
-
-    fn fallback_on_interrupt(&self) -> Option<SdkCredentials> {
-        self.resolve_at(SystemTime::now()).ok()
-    }
-}
-
-pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: SystemTime) -> Result<SdkCredentials, &'static str> {
-    if credentials.expiration.is_some() && credentials.session_token.is_none() {
-        return Err("remote target credential expiration requires a session token");
-    }
-    if credentials.expiration.is_some_and(|expiration| expiration <= now) {
-        return Err(EXPIRED_REMOTE_TARGET_CREDENTIALS);
-    }
-
-    let mut builder = SdkCredentials::builder()
-        .access_key_id(credentials.access_key.clone())
-        .secret_access_key(credentials.secret_key.clone())
-        .account_id(credentials.account_id.clone())
-        .provider_name("bucket_target_sys");
-    if let Some(session_token) = &credentials.session_token {
-        builder = builder.session_token(session_token.clone());
-    }
-    if let Some(expiration) = credentials.expiration {
-        builder = builder.expiry(expiration);
-    }
-    Ok(builder.build())
-}
-
-/// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
-/// signing: SigV4 excludes `user-agent` from the canonical request, so the
-/// signature stays valid.
-#[derive(Debug)]
-struct UserAgentSuffixInterceptor {
-    suffix: &'static str,
-}
-
-impl Intercept for UserAgentSuffixInterceptor {
-    fn name(&self) -> &'static str {
-        "RustfsUserAgentSuffix"
-    }
-
-    fn modify_before_transmit(
-        &self,
-        context: &mut BeforeTransmitInterceptorContextMut<'_>,
-        _runtime_components: &RuntimeComponents,
-        _cfg: &mut ConfigBag,
-    ) -> Result<(), BoxError> {
-        let headers = context.request_mut().headers_mut();
-        let user_agent = match headers.get(http::header::USER_AGENT.as_str()) {
-            Some(existing) => format!("{existing} {}", self.suffix),
-            None => self.suffix.to_string(),
-        };
-        headers.try_insert(http::header::USER_AGENT.as_str(), user_agent)?;
-        Ok(())
-    }
-}
-
-/// Builds the SDK config for `spec` without finalizing it, so callers can add
-/// interceptors or (in tests) swap the HTTP client before `build()`.
-pub(crate) async fn build_remote_s3_config(
-    spec: &RemoteS3EndpointSpec,
-) -> Result<aws_sdk_s3::config::Builder, RemoteS3ClientError> {
-    let Some(credentials) = &spec.credentials else {
-        return Err(RemoteS3ClientError::MissingCredentials);
-    };
-    let creds = remote_sdk_credentials(credentials, SystemTime::now()).map_err(RemoteS3ClientError::Credentials)?;
-
-    let endpoint = spec.endpoint_url();
-    let parsed_endpoint = Url::parse(&endpoint).map_err(|err| RemoteS3ClientError::InvalidEndpoint(err.to_string()))?;
-    validate_remote_endpoint(&parsed_endpoint).map_err(RemoteS3ClientError::EndpointNotAllowed)?;
-
-    let mut config_builder = S3Config::builder()
-        .endpoint_url(endpoint)
-        .credentials_provider(SharedCredentialsProvider::new(RemoteTargetCredentialsProvider { credentials: creds }))
-        .region(SdkRegion::new(spec.region.clone()))
-        .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
-        .request_checksum_calculation(replication_request_checksum_calculation());
-
-    if spec.path_style.force_path_style() {
-        config_builder = config_builder.force_path_style(true);
-    }
-
-    if let Some(http_client) = build_aws_s3_http_client_for_spec(spec).await? {
-        config_builder = config_builder.http_client(http_client);
-    }
-
-    if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
-        let mut timeouts = TimeoutConfig::builder();
-        if let Some(connect_timeout) = spec.connect_timeout {
-            timeouts = timeouts.connect_timeout(connect_timeout);
-        }
-        if let Some(read_timeout) = spec.read_timeout {
-            timeouts = timeouts.read_timeout(read_timeout);
-        }
-        config_builder = config_builder.timeout_config(timeouts.build());
-    }
-
-    if !spec.user_agent_suffix.is_empty() {
-        config_builder = config_builder.interceptor(UserAgentSuffixInterceptor {
-            suffix: spec.user_agent_suffix,
-        });
-    }
-
-    Ok(config_builder)
-}
-
-/// Builds an `aws_sdk_s3::Client` for `spec`, applying the outbound endpoint
-/// gate, credential validation and the TLS transport selection.
-pub async fn build_remote_s3_client(spec: &RemoteS3EndpointSpec) -> Result<S3Client, RemoteS3ClientError> {
-    Ok(S3Client::from_conf(build_remote_s3_config(spec).await?.build()))
-}
-
-#[derive(Debug)]
-struct AcceptAnyServerCertVerifier;
-
-impl rustls::client::danger::ServerCertVerifier for AcceptAnyServerCertVerifier {
-    fn verify_server_cert(
-        &self,
-        _end_entity: &rustls_pki_types::CertificateDer<'_>,
-        _intermediates: &[rustls_pki_types::CertificateDer<'_>],
-        _server_name: &rustls_pki_types::ServerName<'_>,
-        _ocsp_response: &[u8],
-        _now: rustls_pki_types::UnixTime,
-    ) -> Result<rustls::client::danger::ServerCertVerified, rustls::Error> {
-        Ok(rustls::client::danger::ServerCertVerified::assertion())
-    }
-
-    fn verify_tls12_signature(
-        &self,
-        _message: &[u8],
-        _cert: &rustls_pki_types::CertificateDer<'_>,
-        _dss: &rustls::DigitallySignedStruct,
-    ) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
-        Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
-    }
-
-    fn verify_tls13_signature(
-        &self,
-        _message: &[u8],
-        _cert: &rustls_pki_types::CertificateDer<'_>,
-        _dss: &rustls::DigitallySignedStruct,
-    ) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
-        Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
-    }
-
-    fn supported_verify_schemes(&self) -> Vec<rustls::SignatureScheme> {
-        rustls::crypto::aws_lc_rs::default_provider()
-            .signature_verification_algorithms
-            .supported_schemes()
-    }
-}
-
-#[derive(Clone)]
-struct TargetHyperHttpConnector<C> {
-    client: HyperClient<C, SdkBody>,
-}
-
-impl<C> fmt::Debug for TargetHyperHttpConnector<C> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("TargetHyperHttpConnector")
-            .field("client", &"** hyper client **")
-            .finish()
-    }
-}
-
-impl<C> SmithyHttpConnector for TargetHyperHttpConnector<C>
-where
-    C: Clone + Send + Sync + 'static,
-    C: Service<Uri>,
-    C::Response:
-        hyper::rt::Read + hyper::rt::Write + hyper_util::client::legacy::connect::Connection + Send + Sync + Unpin + 'static,
-    C::Future: Unpin + Send + 'static,
-    C::Error: Into<BoxError>,
-{
-    fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
-        let request = match request.try_into_http1x() {
-            Ok(request) => request,
-            Err(err) => return HttpConnectorFuture::ready(Err(ConnectorError::user(err.into()))),
-        };
-
-        let mut client = self.client.clone();
-        let fut = client.call(request);
-        HttpConnectorFuture::new(async move {
-            let response = fut
-                .await
-                .map_err(|err| ConnectorError::io(err.into()))?
-                .map(SdkBody::from_body_1_x);
-            HttpResponse::try_from(response).map_err(|err| ConnectorError::other(err.into(), None))
-        })
-    }
-}
-
-pub(crate) fn ensure_rustls_crypto_provider() {
-    if rustls::crypto::CryptoProvider::get_default().is_none() {
-        let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
-    }
-}
-
-/// Env opt-in that re-enables loopback replication targets. Loopback (`127.0.0.1`,
-/// `::1`, `localhost`) is a classic SSRF vector and stays rejected by default, but
-/// single-host multi-instance dev setups and the e2e harness legitimately replicate
-/// over loopback. Never set this in production.
-const ALLOW_LOOPBACK_REPLICATION_TARGET_ENV: &str = "RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET";
-
-fn loopback_replication_targets_allowed() -> bool {
-    std::env::var(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV)
-        .map(|v| v.eq_ignore_ascii_case("true") || v == "1")
-        .unwrap_or(false)
-}
-
-const REPLICATION_STREAMING_CHECKSUMS_ENV: &str = "RUSTFS_REPLICATION_STREAMING_CHECKSUMS";
-
-/// Streaming trailer checksums make the SDK frame request bodies as
-/// `aws-chunked`; a target that does not decode that framing stores the frames
-/// verbatim, silently corrupting every replica while the transfer itself
-/// succeeds (#6853). Plain signed payloads are the compatible default; the env
-/// knob restores trailer checksums for fleets whose targets are all known to
-/// decode them.
-pub(crate) fn replication_request_checksum_calculation() -> RequestChecksumCalculation {
-    if std::env::var(REPLICATION_STREAMING_CHECKSUMS_ENV)
-        .map(|v| v.eq_ignore_ascii_case("true") || v == "1")
-        .unwrap_or(false)
-    {
-        RequestChecksumCalculation::WhenSupported
-    } else {
-        RequestChecksumCalculation::WhenRequired
-    }
-}
-
-/// Outbound gate for operator-configured remote endpoints (replication
-/// targets, on-demand migration sources). See
-/// `docs/operations/outbound-connection-policy.md`.
-pub fn validate_remote_endpoint(url: &Url) -> Result<(), OutboundUrlError> {
-    validate_remote_endpoint_inner(url, loopback_replication_targets_allowed())
-}
-
-pub(crate) fn validate_remote_endpoint_inner(url: &Url, allow_loopback: bool) -> Result<(), OutboundUrlError> {
-    match validate_outbound_url(url) {
-        Ok(()) => Ok(()),
-        // Replication targets are trusted infrastructure the operator configures, and
-        // legitimately live on private networks, so private addresses are always allowed.
-        Err(OutboundUrlError::ForbiddenHost {
-            reason: "private address",
-            ..
-        }) => Ok(()),
-        // Loopback is far higher SSRF risk, so it is allowed only under the explicit,
-        // off-by-default opt-in above (single-host multi-instance / the e2e harness).
-        Err(OutboundUrlError::ForbiddenHost {
-            reason: "loopback address" | "loopback host",
-            ..
-        }) if allow_loopback => Ok(()),
-        Err(err) => Err(err),
-    }
-}
-
-pub(crate) fn build_insecure_aws_s3_http_client() -> SharedHttpClient {
-    ensure_rustls_crypto_provider();
-
-    let tls_config = rustls::ClientConfig::builder()
-        .dangerous()
-        .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCertVerifier))
-        .with_no_client_auth();
-
-    let https = hyper_rustls::HttpsConnectorBuilder::new()
-        .with_tls_config(tls_config)
-        .https_or_http()
-        .enable_http1()
-        .enable_http2()
-        .build();
-    let mut client_builder = HyperClient::builder(TokioExecutor::new());
-    client_builder.pool_timer(TokioTimer::new());
-    let client = client_builder.build(https);
-    let connector = SharedHttpConnector::new(TargetHyperHttpConnector { client });
-
-    http_client_fn(move |_settings, _components| connector.clone())
-}
-
-fn validate_ca_pem_bundle(ca_cert_pem: &[u8]) -> Result<(), String> {
-    let certs = rustls_pki_types::CertificateDer::pem_slice_iter(ca_cert_pem)
-        .collect::<Result<Vec<_>, _>>()
-        .map_err(|err| format!("invalid PEM encoding: {err}"))?;
-
-    if certs.is_empty() {
-        return Err("no certificates found".to_string());
-    }
-
-    // Smithy's rustls adapter defers parsing custom certificates and assumes
-    // they are valid when the HTTPS connector is built. Validate every DER
-    // certificate first so malformed configuration is reported rather than
-    // reaching an `expect` in the dependency.
-    let mut validation_store = rustls::RootCertStore::empty();
-    for cert in certs {
-        validation_store
-            .add(cert)
-            .map_err(|err| format!("invalid X.509 certificate: {err}"))?;
-    }
-
-    Ok(())
-}
-
-pub(crate) fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), RemoteS3ClientError> {
-    validate_ca_pem_bundle(ca_cert_pem.as_bytes()).map_err(RemoteS3ClientError::InvalidCaPem)
-}
-
-pub(crate) fn compose_replication_trust_store(
-    certificate_bundles: impl IntoIterator<Item = Vec<u8>>,
-) -> (smithy_tls::TrustStore, usize) {
-    // `TrustStore::default()` keeps the platform-native roots enabled. Target
-    // and RUSTFS_TLS_PATH certificates extend that baseline instead of
-    // replacing it with a target-specific trust island.
-    let mut trust_store = smithy_tls::TrustStore::default();
-    let mut custom_bundle_count = 0;
-    for pem in certificate_bundles {
-        trust_store.add_pem_certificate(pem);
-        custom_bundle_count += 1;
-    }
-
-    (trust_store, custom_bundle_count)
-}
-
-pub(crate) fn build_aws_s3_http_client_with_trust_store(
-    trust_store: smithy_tls::TrustStore,
-) -> Result<SharedHttpClient, RemoteS3ClientError> {
-    let tls_context = smithy_tls::TlsContext::builder()
-        .with_trust_store(trust_store)
-        .build()
-        .map_err(|err| RemoteS3ClientError::InvalidCaPem(err.to_string()))?;
-
-    Ok(SmithyHttpClientBuilder::new()
-        .tls_provider(smithy_tls::Provider::rustls(smithy_tls::rustls_provider::CryptoMode::AwsLc))
-        .tls_context(tls_context)
-        .build_https())
-}
-
-pub(crate) async fn load_tls_path_ca_bundles(tls_dir: &Path, trust_leaf_cert_as_ca: bool) -> Vec<Vec<u8>> {
-    let mut certificate_bundles = Vec::new();
-
-    let ca_path = tls_dir.join(RUSTFS_CA_CERT);
-    match tokio::fs::read(&ca_path).await {
-        Ok(pem) => match validate_ca_pem_bundle(&pem) {
-            Ok(()) => certificate_bundles.push(pem),
-            Err(err) => warn!("ignoring invalid custom CA bundle {:?} for replication client: {}", ca_path, err),
-        },
-        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
-        Err(e) => warn!("failed to read custom CA bundle {:?} for replication client: {}", ca_path, e),
-    }
-
-    if trust_leaf_cert_as_ca {
-        let leaf_cert_path = tls_dir.join(RUSTFS_TLS_CERT);
-        match tokio::fs::read(&leaf_cert_path).await {
-            Ok(pem) => match validate_ca_pem_bundle(&pem) {
-                Ok(()) => certificate_bundles.push(pem),
-                Err(err) => warn!(
-                    "ignoring invalid leaf certificate {:?} for replication client trust store: {}",
-                    leaf_cert_path, err
-                ),
-            },
-            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
-            Err(e) => warn!("failed to read leaf cert {:?} for replication client trust store: {}", leaf_cert_path, e),
-        }
-    }
-
-    certificate_bundles
-}
-
-async fn load_configured_tls_ca_bundles() -> Vec<Vec<u8>> {
-    let tls_path = rustfs_utils::get_env_str(rustfs_config::ENV_RUSTFS_TLS_PATH, rustfs_config::DEFAULT_RUSTFS_TLS_PATH);
-    if tls_path.is_empty() {
-        return Vec::new();
-    }
-
-    load_tls_path_ca_bundles(
-        Path::new(&tls_path),
-        rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA),
-    )
-    .await
-}
-
-pub(crate) async fn build_aws_s3_http_client_from_target_ca_pem(
-    ca_cert_pem: &str,
-) -> Result<SharedHttpClient, RemoteS3ClientError> {
-    validate_target_ca_pem(ca_cert_pem)?;
-
-    let mut certificate_bundles = load_configured_tls_ca_bundles().await;
-    certificate_bundles.push(ca_cert_pem.as_bytes().to_vec());
-    let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
-
-    build_aws_s3_http_client_with_trust_store(trust_store)
-}
-
-/// Selects the HTTP client for `spec`: `None` keeps the SDK default (plain
-/// HTTP, or HTTPS with platform roots when no custom trust is configured).
-pub(crate) async fn build_aws_s3_http_client_for_spec(
-    spec: &RemoteS3EndpointSpec,
-) -> Result<Option<SharedHttpClient>, RemoteS3ClientError> {
-    if !spec.secure {
-        return Ok(None);
-    }
-
-    if spec.skip_tls_verify {
-        return Ok(Some(build_insecure_aws_s3_http_client()));
-    }
-
-    if let Some(ca_cert_pem) = spec.custom_ca_pem() {
-        return build_aws_s3_http_client_from_target_ca_pem(ca_cert_pem).await.map(Some);
-    }
-
-    Ok(build_aws_s3_http_client_from_tls_path().await)
-}
-
-async fn build_aws_s3_http_client_from_tls_path() -> Option<SharedHttpClient> {
-    let certificate_bundles = load_configured_tls_ca_bundles().await;
-    if certificate_bundles.is_empty() {
-        return None;
-    }
-
-    let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
-    match build_aws_s3_http_client_with_trust_store(trust_store) {
-        Ok(client) => Some(client),
-        Err(e) => {
-            warn!("failed to build AWS SDK TLS context for replication client: {}", e);
-            None
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use aws_smithy_runtime_api::http::StatusCode as SmithyStatusCode;
-    use std::sync::Mutex;
-
-    fn spec(endpoint: &str, secure: bool) -> RemoteS3EndpointSpec {
-        RemoteS3EndpointSpec {
-            endpoint: endpoint.to_string(),
-            secure,
-            region: "us-east-1".to_string(),
-            path_style: PathStyle::Auto,
-            credentials: Some(RemoteCredentials {
-                access_key: "access".to_string(),
-                secret_key: "secret".to_string(),
-                session_token: None,
-                expiration: None,
-                account_id: String::new(),
-            }),
-            skip_tls_verify: false,
-            ca_cert_pem: None,
-            connect_timeout: None,
-            read_timeout: None,
-            user_agent_suffix: "",
-        }
-    }
-
-    type RecordedHeaders = Arc<Mutex<Vec<Vec<(String, String)>>>>;
-
-    #[derive(Clone, Debug)]
-    struct RecordingHeaderConnector {
-        request_headers: RecordedHeaders,
-    }
-
-    impl SmithyHttpConnector for RecordingHeaderConnector {
-        fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
-            self.request_headers
-                .lock()
-                .expect("recorded header lock should not be poisoned")
-                .push(
-                    request
-                        .headers()
-                        .iter()
-                        .map(|(k, v)| (k.to_string(), v.to_string()))
-                        .collect(),
-                );
-            HttpConnectorFuture::ready(Ok(HttpResponse::new(
-                SmithyStatusCode::try_from(200_u16).expect("200 should be a valid response status"),
-                SdkBody::empty(),
-            )))
-        }
-    }
-
-    #[tokio::test]
-    async fn build_remote_s3_client_rejects_loopback_and_metadata_endpoints() {
-        // Default (no loopback opt-in): loopback in IPv4, IPv6 and hostname
-        // forms plus the metadata endpoint all return the typed gate error.
-        for endpoint in ["127.0.0.1:9000", "[::1]:9000", "localhost:9000", "169.254.169.254"] {
-            let err = build_remote_s3_client(&spec(endpoint, false))
-                .await
-                .err()
-                .unwrap_or_else(|| panic!("{endpoint} must be rejected by the outbound gate"));
-            assert!(
-                matches!(err, RemoteS3ClientError::EndpointNotAllowed(OutboundUrlError::ForbiddenHost { .. })),
-                "{endpoint}: unexpected error {err:?}"
-            );
-            assert!(err.to_string().contains("not allowed"), "{endpoint}: {err}");
-        }
-    }
-
-    #[tokio::test]
-    async fn build_remote_s3_client_allows_private_and_public_endpoints() {
-        for endpoint in ["10.0.0.1:9000", "192.168.1.20", "s3.example.com"] {
-            build_remote_s3_client(&spec(endpoint, false))
-                .await
-                .unwrap_or_else(|err| panic!("{endpoint} should be allowed: {err}"));
-        }
-    }
-
-    #[tokio::test]
-    async fn build_remote_s3_client_requires_credentials() {
-        let mut spec = spec("s3.example.com", true);
-        spec.credentials = None;
-        let err = build_remote_s3_client(&spec)
-            .await
-            .expect_err("missing credentials must be a typed error");
-        assert!(matches!(err, RemoteS3ClientError::MissingCredentials));
-    }
-
-    #[tokio::test]
-    async fn build_remote_s3_client_rejects_expiration_without_session_token() {
-        let mut spec = spec("s3.example.com", true);
-        spec.credentials
-            .as_mut()
-            .expect("spec fixture carries credentials")
-            .expiration = Some(SystemTime::now() + Duration::from_secs(3_600));
-        let err = build_remote_s3_client(&spec)
-            .await
-            .expect_err("expiration without session token must be rejected");
-        assert_eq!(err.to_string(), "remote target credential expiration requires a session token");
-    }
-
-    #[tokio::test]
-    async fn build_remote_s3_client_rejects_invalid_custom_ca_pem() {
-        let mut spec = spec("192.168.1.10:9000", true);
-        spec.ca_cert_pem = Some("not a pem".to_string());
-        let err = build_remote_s3_client(&spec)
-            .await
-            .expect_err("invalid custom CA PEM must be rejected");
-        assert!(matches!(err, RemoteS3ClientError::InvalidCaPem(_)));
-        assert!(err.to_string().contains("invalid target CA PEM"));
-    }
-
-    #[test]
-    fn path_style_auto_and_path_force_path_style() {
-        assert!(PathStyle::Auto.force_path_style());
-        assert!(PathStyle::Path.force_path_style());
-        assert!(!PathStyle::VirtualHost.force_path_style());
-    }
-
-    #[test]
-    fn remote_credentials_debug_redacts_secrets() {
-        let credentials = RemoteCredentials {
-            access_key: "access".to_string(),
-            secret_key: "very-secret".to_string(),
-            session_token: Some("session-token".to_string()),
-            expiration: None,
-            account_id: String::new(),
-        };
-        let rendered = format!("{credentials:?}");
-        assert!(rendered.contains("access"));
-        assert!(!rendered.contains("very-secret"));
-        assert!(!rendered.contains("session-token"));
-    }
-
-    #[tokio::test]
-    async fn user_agent_suffix_is_appended_after_signing() {
-        let request_headers: RecordedHeaders = Arc::new(Mutex::new(Vec::new()));
-        let connector = SharedHttpConnector::new(RecordingHeaderConnector {
-            request_headers: Arc::clone(&request_headers),
-        });
-        let http_client = http_client_fn(move |_settings, _components| connector.clone());
-
-        let mut spec = spec("s3.example.com", true);
-        spec.user_agent_suffix = "RustFS-Test/0.0";
-        spec.connect_timeout = Some(Duration::from_secs(5));
-        spec.read_timeout = Some(Duration::from_secs(5));
-        let config = build_remote_s3_config(&spec)
-            .await
-            .expect("spec should build")
-            .http_client(http_client)
-            .build();
-        S3Client::from_conf(config)
-            .head_bucket()
-            .bucket("bucket")
-            .send()
-            .await
-            .expect("recording connector should accept the request");
-
-        let recorded = request_headers.lock().expect("recorded header lock should not be poisoned");
-        let headers = &recorded[0];
-        let user_agent = headers
-            .iter()
-            .find(|(k, _)| k.eq_ignore_ascii_case("user-agent"))
-            .map(|(_, v)| v.as_str())
-            .expect("SDK request must carry a user-agent");
-        assert!(user_agent.ends_with(" RustFS-Test/0.0"), "user-agent was {user_agent}");
-        assert!(user_agent.starts_with("aws-sdk-rust/"), "SDK identity must be preserved: {user_agent}");
-        assert!(
-            headers.iter().any(|(k, _)| k.eq_ignore_ascii_case("authorization")),
-            "request must still be signed"
-        );
-    }
-}

crates/ecstore/src/data_movement/mod.rs

@@ -234,6 +234,7 @@ async fn pause_data_movement_multipart_before_abort(bucket: &str, object: &str)
 }
 
 fn data_movement_abort_opts(
+    object_info: &ObjectInfo,
     src_pool_idx: usize,
     expected_bucket_incarnation_id: Option<uuid::Uuid>,
     lock_lost_signal: Option<&Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
@@ -242,6 +243,9 @@ fn data_movement_abort_opts(
     let mut opts = ObjectOptions {
         data_movement: true,
         src_pool_idx,
+        versioned: object_info.version_id.is_some(),
+        version_id: object_info.version_id.map(|version_id| version_id.to_string()),
+        mod_time: object_info.mod_time,
         expected_bucket_incarnation_id,
         ..Default::default()
     };
@@ -265,16 +269,21 @@ fn insert_data_movement_checksum(user_defined: &mut HashMap<String, String>, obj
     }
 }
 
-fn data_movement_upload_identity(object_info: &ObjectInfo) -> String {
-    let version_id = object_info
-        .version_id
-        .map_or_else(|| "none".to_string(), |version_id| version_id.to_string());
-    let mod_time = object_info
-        .mod_time
-        .map_or_else(|| "none".to_string(), |mod_time| mod_time.unix_timestamp_nanos().to_string());
+fn data_movement_upload_identity_parts(version_id: Option<&str>, mod_time: Option<time::OffsetDateTime>) -> String {
+    let version_id = version_id.unwrap_or("none");
+    let mod_time = mod_time.map_or_else(|| "none".to_string(), |mod_time| mod_time.unix_timestamp_nanos().to_string());
     format!("v1:{version_id}:{mod_time}")
 }
 
+fn data_movement_upload_identity(object_info: &ObjectInfo) -> String {
+    let version_id = object_info.version_id.map(|version_id| version_id.to_string());
+    data_movement_upload_identity_parts(version_id.as_deref(), object_info.mod_time)
+}
+
+pub(crate) fn data_movement_upload_identity_from_options(opts: &ObjectOptions) -> String {
+    data_movement_upload_identity_parts(opts.version_id.as_deref(), opts.mod_time)
+}
+
 fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usize) -> ObjectOptions {
     let mut user_defined = data_movement_user_defined(object_info);
     let upload_identity = data_movement_upload_identity(object_info);
@@ -486,7 +495,7 @@ pub(crate) fn data_movement_target_precondition() -> HTTPPreconditions {
     }
 }
 
-fn is_owned_data_movement_target(target: &ObjectInfo) -> bool {
+pub(crate) fn is_owned_data_movement_target(target: &ObjectInfo) -> bool {
     let rustfs_marker = rustfs_utils::http::internal_key_rustfs(SUFFIX_DATA_MOVED);
     let minio_marker = format!("{}{SUFFIX_DATA_MOVED}", rustfs_utils::http::MINIO_INTERNAL_PREFIX);
     if rustfs_utils::http::get_consistent_str(&target.user_defined, SUFFIX_DATA_MOVED) != Some("true")
@@ -560,9 +569,12 @@ fn resolve_data_movement_abort_result(
     primary_err: Error,
     abort_err: Error,
 ) -> Error {
-    Error::other(format!(
-        "{op_label}: abort_multipart_upload failed for {bucket}/{object} upload {upload_id} after error {primary_err}: {abort_err}"
-    ))
+    data_movement_context_error(
+        format!(
+            "{op_label}: abort_multipart_upload failed for {bucket}/{object} upload {upload_id} after error {primary_err}: {abort_err}"
+        ),
+        abort_err,
+    )
 }
 
 /// A data-movement stage failure that keeps the error it wrapped.
@@ -590,17 +602,23 @@ impl std::error::Error for DataMovementStageError {
     }
 }
 
-fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, object: &str, err: E) -> Error
+pub(crate) fn data_movement_context_error<E>(rendered: String, err: E) -> Error
 where
     E: std::error::Error + Send + Sync + 'static,
 {
-    let rendered = format!("{op_label}: {stage} failed for {bucket}/{object}: {err}");
     Error::other(DataMovementStageError {
         rendered,
         source: Box::new(err),
     })
 }
 
+fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, object: &str, err: E) -> Error
+where
+    E: std::error::Error + Send + Sync + 'static,
+{
+    data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
+}
+
 #[cfg(test)]
 pub(crate) fn data_movement_stage_error_for_test(op_label: &str, stage: &str, bucket: &str, object: &str, err: Error) -> Error {
     data_movement_stage_error(op_label, stage, bucket, object, err)
@@ -1662,8 +1680,13 @@ async fn migrate_object_inner(
                 );
                 return Ok(());
             }
-            let mut cleanup_opts =
-                data_movement_abort_opts(pool_idx, source_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
+            let mut cleanup_opts = data_movement_abort_opts(
+                &object_info,
+                pool_idx,
+                source_bucket_incarnation_id,
+                lock_lost_signal.as_ref(),
+                capacity_owner,
+            );
             if let Some(anchor) = mutation_fence.as_ref() {
                 anchor.guard().add_namespace_lock_fence(&mut cleanup_opts);
             }
@@ -1865,8 +1888,13 @@ async fn migrate_object_inner(
         .await;
 
         if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) {
-            let mut abort_opts =
-                data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
+            let mut abort_opts = data_movement_abort_opts(
+                &object_info,
+                pool_idx,
+                expected_bucket_incarnation_id,
+                lock_lost_signal.as_ref(),
+                capacity_owner,
+            );
             if let Some(anchor) = mutation_fence.as_ref() {
                 anchor.guard().add_namespace_lock_fence(&mut abort_opts);
             }
@@ -1943,8 +1971,13 @@ async fn migrate_object_inner(
             if should_abort_multipart_upload(&abort_multipart_flag) {
                 #[cfg(all(test, feature = "test-util"))]
                 pause_data_movement_multipart_before_abort(&bucket, &object_info.name).await;
-                let mut abort_opts =
-                    data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
+                let mut abort_opts = data_movement_abort_opts(
+                    &object_info,
+                    pool_idx,
+                    expected_bucket_incarnation_id,
+                    lock_lost_signal.as_ref(),
+                    capacity_owner,
+                );
                 if let Some(anchor) = mutation_fence.as_ref() {
                     anchor.guard().add_namespace_lock_fence(&mut abort_opts);
                 }
@@ -2325,6 +2358,7 @@ mod tests {
         assert!(message.contains("bucket-a/object-a"));
         assert!(message.contains("upload upload-1"));
         assert!(message.contains(Error::SlowDown.to_string().as_str()));
+        assert!(matches!(data_movement_stage_source(&err), Some(Error::OperationCanceled)));
     }
 
     #[test]

crates/ecstore/src/data_usage/mod.rs

@@ -2964,7 +2964,6 @@ mod tests {
             decommission_cancelers: RwLock::new(Vec::new()),
             start_gate: TokioMutex::new(()),
             pool_meta_save_gate: TokioMutex::default(),
-            decommission_capacity_entry_gate: TokioMutex::default(),
             ctx,
             bucket_fence_registry: Arc::default(),
         })

crates/ecstore/src/services/notification_sys.rs

@@ -57,18 +57,24 @@ const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
 const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
 const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
 const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3;
+const DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
 type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
 
 fn cross_pool_fence_policy_results(
     peer_epochs: BTreeMap<String, Uuid>,
     minimum_version: u32,
-) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) {
+) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) {
     let journal_result = if minimum_version >= TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION {
         Ok(peer_epochs.clone())
     } else {
         Err(Error::other("tier delete journal v6 policy capability version is unsupported"))
     };
-    (Ok(peer_epochs), journal_result)
+    let decommission_target_fence_result = if minimum_version >= DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION {
+        Ok(peer_epochs.clone())
+    } else {
+        Err(Error::other("decommission target fence policy capability version is unsupported"))
+    };
+    (Ok(peer_epochs), journal_result, decommission_target_fence_result)
 }
 
 #[derive(Clone, Debug)]
@@ -231,6 +237,9 @@ pub(crate) struct RemoteVersionStateFleetProofToken(FleetCapabilityProofToken);
 #[derive(Clone, PartialEq, Eq)]
 pub struct CrossPoolFenceFleetProofToken(FleetCapabilityProofToken);
 
+#[derive(Clone, PartialEq, Eq)]
+pub(crate) struct DecommissionTargetFenceFleetProofToken(FleetCapabilityProofToken);
+
 /// A point-in-time proof that every current storage member implements the v6
 /// dispatch-manifest policy. It intentionally has no `Clone` implementation:
 /// one acquisition authorizes one manifest construction attempt.
@@ -242,6 +251,7 @@ pub(crate) struct TierDeleteJournalFleetProofToken {
 static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
 static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
 static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
+static DECOMMISSION_TARGET_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
 static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
 
 fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
@@ -256,6 +266,10 @@ fn tier_delete_journal_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCap
     TIER_DELETE_JOURNAL_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
 }
 
+fn decommission_target_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
+    DECOMMISSION_TARGET_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
+}
+
 fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
     if let Some(proof) = state.proof.take() {
         proof.generation.revoke();
@@ -368,6 +382,18 @@ pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToke
     fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0)
 }
 
+pub(crate) fn acquire_decommission_target_fence_fleet_proof() -> Option<DecommissionTargetFenceFleetProofToken> {
+    let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
+    let state = decommission_target_fence_fleet_proof_slot()
+        .read()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now()).map(DecommissionTargetFenceFleetProofToken)
+}
+
+pub(crate) fn decommission_target_fence_fleet_proof_matches(proof: &DecommissionTargetFenceFleetProofToken) -> bool {
+    fleet_capability_proof_matches(decommission_target_fence_fleet_proof_slot(), &proof.0)
+}
+
 pub(crate) fn acquire_tier_delete_journal_fleet_proof() -> Option<TierDeleteJournalFleetProofToken> {
     let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
     let state = tier_delete_journal_fleet_proof_slot()
@@ -468,6 +494,19 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
     journal_state.topology_conflict = false;
     journal_state.draining_generation = None;
     journal_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation);
+    drop(journal_state);
+    let mut decommission_state = decommission_target_fence_fleet_proof_slot()
+        .write()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    debug_assert!(
+        decommission_state
+            .proof
+            .as_ref()
+            .is_none_or(|current| current.generation.is_drained())
+    );
+    decommission_state.topology_conflict = false;
+    decommission_state.draining_generation = None;
+    decommission_state.proof = proof.as_ref().map(FleetCapabilityProof::with_fresh_generation);
 }
 
 #[cfg(test)]
@@ -476,6 +515,8 @@ pub(crate) struct CrossPoolFenceFleetProofGuard {
     previous_topology_conflict: bool,
     previous_journal_proof: Option<FleetCapabilityProof>,
     previous_journal_topology_conflict: bool,
+    previous_decommission_proof: Option<FleetCapabilityProof>,
+    previous_decommission_topology_conflict: bool,
 }
 
 #[cfg(test)]
@@ -502,6 +543,17 @@ impl Drop for CrossPoolFenceFleetProofGuard {
             .map(FleetCapabilityProof::with_fresh_generation);
         journal_state.draining_generation = None;
         journal_state.topology_conflict = self.previous_journal_topology_conflict;
+        drop(journal_state);
+        let mut decommission_state = decommission_target_fence_fleet_proof_slot()
+            .write()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        decommission_state.proof = self
+            .previous_decommission_proof
+            .take()
+            .as_ref()
+            .map(FleetCapabilityProof::with_fresh_generation);
+        decommission_state.draining_generation = None;
+        decommission_state.topology_conflict = self.previous_decommission_topology_conflict;
     }
 }
 
@@ -515,11 +567,16 @@ pub(crate) fn without_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceF
     let mut journal_state = tier_delete_journal_fleet_proof_slot()
         .write()
         .unwrap_or_else(std::sync::PoisonError::into_inner);
+    let mut decommission_state = decommission_target_fence_fleet_proof_slot()
+        .write()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
     let guard = CrossPoolFenceFleetProofGuard {
         previous_proof: state.proof.clone(),
         previous_topology_conflict: state.topology_conflict,
         previous_journal_proof: journal_state.proof.clone(),
         previous_journal_topology_conflict: journal_state.topology_conflict,
+        previous_decommission_proof: decommission_state.proof.clone(),
+        previous_decommission_topology_conflict: decommission_state.topology_conflict,
     };
     if let Some(proof) = state.proof.take() {
         proof.generation.revoke();
@@ -535,6 +592,49 @@ pub(crate) fn without_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceF
         }
     }
     journal_state.topology_conflict = true;
+    if let Some(proof) = decommission_state.proof.take() {
+        proof.generation.revoke();
+        if !proof.generation.is_drained() {
+            decommission_state.draining_generation = Some(proof.generation);
+        }
+    }
+    decommission_state.topology_conflict = true;
+    guard
+}
+
+#[cfg(test)]
+pub(crate) struct DecommissionTargetFenceFleetProofGuard {
+    previous_proof: Option<FleetCapabilityProof>,
+    previous_topology_conflict: bool,
+}
+
+#[cfg(test)]
+impl Drop for DecommissionTargetFenceFleetProofGuard {
+    fn drop(&mut self) {
+        let mut state = decommission_target_fence_fleet_proof_slot()
+            .write()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        state.proof = self
+            .previous_proof
+            .take()
+            .as_ref()
+            .map(FleetCapabilityProof::with_fresh_generation);
+        state.draining_generation = None;
+        state.topology_conflict = self.previous_topology_conflict;
+    }
+}
+
+#[cfg(test)]
+pub(crate) fn without_decommission_target_fence_fleet_proof_for_test() -> DecommissionTargetFenceFleetProofGuard {
+    let mut state = decommission_target_fence_fleet_proof_slot()
+        .write()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    let guard = DecommissionTargetFenceFleetProofGuard {
+        previous_proof: state.proof.clone(),
+        previous_topology_conflict: state.topology_conflict,
+    };
+    revoke_fleet_capability_proof_state(&mut state);
+    state.topology_conflict = true;
     guard
 }
 
@@ -573,6 +673,15 @@ pub fn rotate_cross_pool_fence_fleet_proof_for_test() -> bool {
     if journal_state.draining_generation.is_none() {
         journal_state.proof = Some(proof.with_fresh_generation());
     }
+    drop(journal_state);
+    let mut decommission_state = decommission_target_fence_fleet_proof_slot()
+        .write()
+        .unwrap_or_else(std::sync::PoisonError::into_inner);
+    decommission_state.topology_conflict = false;
+    revoke_fleet_capability_proof_state(&mut decommission_state);
+    if decommission_state.draining_generation.is_none() {
+        decommission_state.proof = Some(proof.with_fresh_generation());
+    }
     true
 }
 
@@ -652,6 +761,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                 remote_version_state_fleet_proof_slot(),
                 cross_pool_fence_fleet_proof_slot(),
                 tier_delete_journal_fleet_proof_slot(),
+                decommission_target_fence_fleet_proof_slot(),
             ] {
                 mark_fleet_capability_topology_conflict(slot);
             }
@@ -684,11 +794,15 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                 .unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
                 None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
             };
-            let (fence_result, journal_result) = match fence_probe {
+            let (fence_result, journal_result, decommission_target_fence_result) = match fence_probe {
                 Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version),
                 Err(err) => {
                     let message = err.to_string();
-                    (Err(Error::other(message.clone())), Err(Error::other(message)))
+                    (
+                        Err(Error::other(message.clone())),
+                        Err(Error::other(message.clone())),
+                        Err(Error::other(message)),
+                    )
                 }
             };
             let topology_conflict = remote_version_state_fleet_proof_slot()
@@ -699,6 +813,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                 revoke_fleet_capability_proof(remote_version_state_fleet_proof_slot());
                 revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot());
                 revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
+                revoke_fleet_capability_proof(decommission_target_fence_fleet_proof_slot());
             } else if let Some(err) = publish_fleet_capability_probe_result(
                 remote_version_state_fleet_proof_slot(),
                 &topology_fingerprint,
@@ -743,6 +858,24 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
                     "notification capability probe"
                 );
             }
+            if !topology_conflict
+                && let Some(err) = publish_fleet_capability_probe_result(
+                    decommission_target_fence_fleet_proof_slot(),
+                    &topology_fingerprint,
+                    decommission_target_fence_result,
+                    Instant::now(),
+                )
+            {
+                debug!(
+                    event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_NOTIFICATION,
+                    capability = "decommission_target_fence_v2",
+                    state = "failed_closed",
+                    error = %err,
+                    "notification capability probe"
+                );
+            }
             sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
         }
     });
@@ -834,7 +967,7 @@ impl NotificationSys {
         // A single-node deployment has no remote member to lower the local
         // policy version advertised by this binary.
         if minimum_version == u32::MAX {
-            minimum_version = TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION;
+            minimum_version = DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION;
         }
         Ok((peer_epochs, minimum_version))
     }
@@ -2804,15 +2937,22 @@ mod tests {
     use super::*;
 
     #[test]
-    fn cross_pool_v2_remains_generic_but_cannot_authorize_v6_journal() {
+    fn cross_pool_policy_versions_authorize_only_their_supported_protocols() {
         let peers = BTreeMap::from([("node-b:9000".to_string(), Uuid::new_v4())]);
-        let (generic_v2, journal_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
+        let (generic_v2, journal_v2, decommission_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
         assert!(generic_v2.is_ok(), "v2 remains valid for existing cross-pool fencing");
         assert!(journal_v2.is_err(), "a mixed v2/v3 fleet must fail closed for journal-v6 deletion");
+        assert!(decommission_v2.is_err(), "v2 cannot authorize the sticky per-target decommission fence");
 
-        let (generic_v3, journal_v3) = cross_pool_fence_policy_results(peers, 3);
+        let (generic_v3, journal_v3, decommission_v3) = cross_pool_fence_policy_results(peers.clone(), 3);
         assert!(generic_v3.is_ok());
         assert!(journal_v3.is_ok(), "an all-v3 fleet may authorize journal-v6 deletion");
+        assert!(decommission_v3.is_err(), "v3 members do not understand the per-target decommission fence");
+
+        let (generic_v4, journal_v4, decommission_v4) = cross_pool_fence_policy_results(peers, 4);
+        assert!(generic_v4.is_ok());
+        assert!(journal_v4.is_ok());
+        assert!(decommission_v4.is_ok(), "an all-v4 fleet may create sticky per-target reservations");
     }
 
     #[test]

crates/ecstore/src/services/rebalance/mod.rs

@@ -83,7 +83,6 @@ pub async fn test_store_with_persisted_rebalance_meta(
         decommission_cancelers: tokio::sync::RwLock::new(vec![None]),
         start_gate: tokio::sync::Mutex::new(()),
         pool_meta_save_gate: tokio::sync::Mutex::default(),
-        decommission_capacity_entry_gate: tokio::sync::Mutex::default(),
         ctx,
         bucket_fence_registry: std::sync::Arc::default(),
     });
@@ -233,7 +232,6 @@ async fn test_pool_stores_with_contexts(
             decommission_cancelers: tokio::sync::RwLock::new(vec![None; pool_count]),
             start_gate: tokio::sync::Mutex::new(()),
             pool_meta_save_gate: tokio::sync::Mutex::new(pool_meta_write_state.independent_clone_for_test()),
-            decommission_capacity_entry_gate: tokio::sync::Mutex::default(),
             ctx: store_ctx,
             bucket_fence_registry: std::sync::Arc::default(),
         })

crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs

@@ -3009,7 +3009,6 @@ fn test_store_with_rebalance_meta(meta: RebalanceMeta) -> Arc<crate::store::ECSt
         decommission_cancelers: tokio::sync::RwLock::new(Vec::new()),
         start_gate: tokio::sync::Mutex::new(()),
         pool_meta_save_gate: tokio::sync::Mutex::default(),
-        decommission_capacity_entry_gate: tokio::sync::Mutex::default(),
         ctx: crate::runtime::instance::bootstrap_ctx(),
         bucket_fence_registry: std::sync::Arc::default(),
     })

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -66,6 +66,7 @@ use crate::disk::new_disk;
 use crate::multipart_listing::paginate_multipart_listing;
 #[cfg(test)]
 use crate::object_api::ObjectLockConfigSnapshot;
+use crate::set_disk::core::io_primitives::finish_rename_tail_heal;
 use crate::set_disk::mem;
 use crate::set_disk::metadata_sys;
 use crate::set_disk::runtime_sources;
@@ -836,7 +837,7 @@ impl SetDisks {
         orig_bucket: &str,
         error_path: &str,
         root_prefix: &str,
-    ) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize)> {
+    ) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize, bool)> {
         let disks = self.disks.read().await.clone();
         if disks.is_empty() {
             return Err(Error::ErasureReadQuorum);
@@ -881,16 +882,24 @@ impl SetDisks {
             return Err(to_object_err(err.into(), vec![orig_bucket, error_path]));
         }
 
+        let mut has_minority_candidate = false;
         let mut candidate_paths = candidate_counts
             .into_iter()
-            .filter_map(|(path, count)| (count >= discovery_quorum).then_some(path))
+            .filter_map(|(path, count)| {
+                if count >= discovery_quorum {
+                    Some(path)
+                } else {
+                    has_minority_candidate = true;
+                    None
+                }
+            })
             .collect::<Vec<_>>();
         candidate_paths.sort_unstable();
-        Ok((disks, candidate_paths, discovery_quorum))
+        Ok((disks, candidate_paths, discovery_quorum, has_minority_candidate))
     }
 
     pub(crate) async fn first_multipart_upload_path_for_decommission(&self, bucket: &str) -> Result<Option<String>> {
-        let (_, paths, _) = self
+        let (_, paths, _, _) = self
             .discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "")
             .await?;
         Ok(paths.into_iter().next())
@@ -904,7 +913,14 @@ impl SetDisks {
         upload_identity: &str,
     ) -> Result<Vec<String>> {
         let expected_parent = format!("{DATA_MOVEMENT_MULTIPART_PREFIX}/{}", Self::get_multipart_sha_dir(bucket, object));
-        let (_, candidate_paths, _) = self.discover_multipart_upload_paths(bucket, object, &expected_parent).await?;
+        let (_, candidate_paths, _, has_minority_candidate) =
+            self.discover_multipart_upload_paths(bucket, object, &expected_parent).await?;
+        if has_minority_candidate {
+            return Err(Error::DecommissionCapacityBlocked {
+                message: "data movement multipart cleanup found an upload path on fewer than the discovery quorum of disks"
+                    .to_string(),
+            });
+        }
         let mut upload_ids = Vec::new();
         for upload_path in candidate_paths {
             let Some((parent, raw_upload_id)) = upload_path.rsplit_once('/') else {
@@ -920,7 +936,11 @@ impl SetDisks {
             {
                 Ok((file_info, _)) => file_info,
                 Err(err) if crate::error::is_err_invalid_upload_id(&err) || crate::error::is_err_object_not_found(&err) => {
-                    continue;
+                    return Err(Error::DecommissionCapacityBlocked {
+                        message: format!(
+                            "data movement multipart cleanup found quorum-visible upload path {upload_path} without verifiable metadata: {err}"
+                        ),
+                    });
                 }
                 Err(err) => return Err(err),
             };
@@ -1189,7 +1209,7 @@ impl SetDisks {
         max_uploads: usize,
         expected_incarnation_id: Option<Uuid>,
     ) -> Result<ListMultipartsInfo> {
-        let (disks, candidate_paths, discovery_quorum) = self.discover_multipart_upload_paths(bucket, prefix, "").await?;
+        let (disks, candidate_paths, discovery_quorum, _) = self.discover_multipart_upload_paths(bucket, prefix, "").await?;
         let listed_uploads = stream::iter(candidate_paths)
             .map(|upload_path| {
                 let disks = &disks;
@@ -3123,11 +3143,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         let commit_object_lock_guard = object_lock_guard.take();
         let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
         let commit_decommission_capacity_guard = decommission_capacity_guard.take();
-        // CompleteMultipartUpload is an S3 publication boundary: after a
-        // successful response, the object must be immediately readable and
-        // usable as a CopyObject source. Do not return on rename quorum while
-        // a tail owner may still hold the object guard and finish shard moves.
-        let commit_allows_early_ack = false;
+        let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation())
+            && (commit_object_lock_guard.is_some() || commit_decommission_object_lock_guard.is_some());
         let detach_commit_owner = commit_allows_early_ack || upload_guard.is_some() || quota_mutation_fence;
         let commit = async move {
             let mut _object_lock_guard = commit_object_lock_guard;
@@ -3258,15 +3275,105 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                 commit_allows_early_ack,
             )
             .await;
+            let mut rename_guard_release = None;
+            let mut needs_immediate_heal = false;
+            let mut tail_owns_staging_cleanup = false;
             if let Ok(rename_commit) = rename_result.as_mut() {
                 commit_set.record_capacity_scope_if_needed(commit_capacity_scope_token, &rename_commit.capacity_disks);
-                debug_assert!(
-                    rename_commit.tail_drain.is_none(),
-                    "multipart completion disables early ACK and must not detach a rename tail"
-                );
+                // Install the tail watcher before any post-commit await. The
+                // latch keeps namespace guards through their prior handoff point.
+                needs_immediate_heal = rename_commit.needs_immediate_heal();
+                if let Some(rename_tail_drain) = rename_commit.tail_drain.take() {
+                    tail_owns_staging_cleanup = true;
+                    let mut request = rustfs_heal_contracts::heal_channel::create_heal_request_with_options(
+                        commit_bucket.clone(),
+                        Some(commit_object.clone()),
+                        false,
+                        Some(HealChannelPriority::Normal),
+                        Some(commit_set.pool_index),
+                        Some(commit_set.set_index),
+                    );
+                    request.object_version_id = fi
+                        .version_id
+                        .or_else(|| commit_version_suspended.then(Uuid::nil))
+                        .map(|version_id| version_id.to_string());
+                    let object_lock_guard = _object_lock_guard.take();
+                    let upload_guard = _upload_guard.take();
+                    let decommission_object_lock_guard = _decommission_object_lock_guard.take();
+                    let decommission_capacity_guard = _decommission_capacity_guard.take();
+                    let cleanup_bucket = commit_bucket.clone();
+                    let cleanup_object = commit_object.clone();
+                    let heal_set = commit_set.clone();
+                    let cleanup_set = commit_set.clone();
+                    let committed_data_dir = fi.data_dir;
+                    let cleanup_parts = parts.clone();
+                    let cleanup_upload_path = commit_upload_id_path.clone();
+                    let cleanup_upload_id = commit_upload_id.clone();
+                    let fence_disks = commit_disks.clone();
+                    let fence_tokens = quota_fence_tokens.clone();
+                    let fence_bucket = commit_bucket.clone();
+                    let fence_object = commit_object.clone();
+                    let (guard_release_tx, guard_release_rx) = tokio::sync::oneshot::channel();
+                    rename_guard_release = Some(guard_release_tx);
+                    tokio::spawn(finish_rename_tail_heal(
+                        rename_tail_drain,
+                        guard_release_rx,
+                        (
+                            object_lock_guard,
+                            upload_guard,
+                            decommission_object_lock_guard,
+                            decommission_capacity_guard,
+                        ),
+                        request,
+                        move || async move {
+                            if quota_mutation_fence {
+                                let _ = SetDisks::release_quota_mutation_fences(
+                                    &fence_disks,
+                                    &fence_tokens,
+                                    &fence_bucket,
+                                    &fence_object,
+                                    write_quorum,
+                                )
+                                .await;
+                            }
+                        },
+                        move |(object_lock_guard, upload_guard, decommission_object_lock_guard, decommission_capacity_guard),
+                              targets| async move {
+                            drop(object_lock_guard);
+                            cleanup_set.cleanup_multipart_path(&cleanup_parts).await;
+                            cleanup_set
+                                .cleanup_rename_tail(
+                                    targets,
+                                    &cleanup_bucket,
+                                    &cleanup_object,
+                                    committed_data_dir,
+                                    transaction_epoch,
+                                )
+                                .await;
+                            if let Err(err) = cleanup_set
+                                .delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &cleanup_upload_path, write_quorum)
+                                .await
+                            {
+                                warn!(
+                                    bucket = %cleanup_bucket,
+                                    object = %cleanup_object,
+                                    upload_id = %cleanup_upload_id,
+                                    error = ?err,
+                                    "completed multipart upload staging cleanup did not reach write quorum"
+                                );
+                            }
+                            drop(upload_guard);
+                            drop(decommission_object_lock_guard);
+                            drop(decommission_capacity_guard);
+                        },
+                        |request| async move { heal_set.submit_rename_tail_heal(request).await },
+                    ));
+                }
             }
-            drop(_decommission_capacity_guard.take());
-            if quota_mutation_fence {
+            if !tail_owns_staging_cleanup {
+                drop(_decommission_capacity_guard.take());
+            }
+            if quota_mutation_fence && !tail_owns_staging_cleanup {
                 let _ = SetDisks::release_quota_mutation_fences(
                     &commit_disks,
                     &quota_fence_tokens,
@@ -3283,7 +3390,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                 Ok(result) => result,
                 Err(err) => return Err(err.into()),
             };
-            let needs_immediate_heal = rename_commit.needs_immediate_heal();
             let op_old_dir = rename_commit.data_dir;
             let cleanup_disks = rename_commit.cleanup_disks;
             let committed_file_info = rename_commit.committed_file_info;
@@ -3326,6 +3432,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             // parts are swept by a retried completion or upload GC (rustfs/backlog#946).
             // Compiles to a no-op outside `#[cfg(test)]`.
             if crash_inject::should_crash_at(CrashPoint::MultipartAfterCommitBeforePartsCleanup, &commit_object) {
+                if let Some(release) = rename_guard_release.take() {
+                    let _ = release.send(false);
+                }
                 return Err(StorageError::Unexpected);
             }
 
@@ -3341,7 +3450,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                 .invalidate_get_object_metadata_cache(&commit_bucket, &commit_object)
                 .await;
 
-            drop(_object_lock_guard.take()); // release the object lock before multipart cleanup IO.
+            if let Some(release) = rename_guard_release.take() {
+                let _ = release.send(true);
+            }
+            drop(_object_lock_guard.take()); // release the object lock before multipart cleanup tail IO.
 
             #[cfg(test)]
             pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::AfterObjectPublication).await;
@@ -3354,7 +3466,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             // parts; deleting them before the commit would strand the upload
             // permanently. This mirrors the "clean up only after commit" pattern
             // already used for the old data-dir GC and the upload-dir delete_all below.
-            commit_set.cleanup_multipart_path(&parts).await;
+            if !tail_owns_staging_cleanup {
+                commit_set.cleanup_multipart_path(&parts).await;
+            }
 
             if let Some(old_dir) = op_old_dir {
                 // backlog#898: best-effort reclaim of the dereferenced old data dir.
@@ -3385,9 +3499,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             #[cfg(test)]
             pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::AfterRename).await;
 
-            if let Err(err) = commit_set
-                .delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &commit_upload_id_path, write_quorum)
-                .await
+            if !tail_owns_staging_cleanup
+                && let Err(err) = commit_set
+                    .delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &commit_upload_id_path, write_quorum)
+                    .await
             {
                 warn!(
                     bucket = %commit_bucket,
@@ -3914,7 +4029,7 @@ mod tests {
 
     #[tokio::test]
     #[serial(capacity_dirty_scope)]
-    async fn complete_multipart_waits_for_tail_before_releasing_guards_and_marking_capacity() {
+    async fn early_ack_multipart_holds_quota_fences_and_re_marks_capacity_after_tail_drain() {
         use rustfs_object_capacity::capacity_scope::drain_global_dirty_scopes;
 
         temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
@@ -3960,9 +4075,10 @@ mod tests {
                 .collect::<HashSet<_>>();
             let _ = drain_global_dirty_scopes();
 
+            let rename_tasks = rename_fanout_barrier::observe_tasks(object);
             let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
             let complete_store = Arc::clone(&set_disks);
-            let mut complete = tokio::spawn(async move {
+            let complete = tokio::spawn(async move {
                 let mut opts = ObjectOptions::default();
                 assert!(opts.set_quota_admission(0, u64::MAX));
                 complete_store
@@ -3972,15 +4088,20 @@ mod tests {
             tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
                 .await
                 .expect("multipart completion should pause one tail disk during rename");
+            let cleanup_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_CLEANUP);
+            complete
+                .await
+                .expect("early-ACK multipart task should join before tail release")
+                .expect("multipart completion should return after write quorum");
             assert!(
-                tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
-                "multipart completion must not publish success while a tail rename is still paused"
+                rename_tasks.running() >= 1,
+                "the paused multipart tail disk must remain in flight after quorum ACK"
             );
 
             let initial = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
             assert!(
-                initial.is_empty(),
-                "capacity must not be marked as committed before the full multipart rename finishes"
+                expected.is_subset(&initial),
+                "the multipart quorum ACK must mark every candidate disk dirty"
             );
 
             let abort_store = Arc::clone(&set_disks);
@@ -3990,7 +4111,7 @@ mod tests {
                     .await
             });
             signaling.wait_for_attempts(2).await;
-            assert!(!abort.is_finished(), "the in-flight completion must retain the multipart upload guard");
+            assert!(!abort.is_finished(), "the detached tail owner must retain the multipart upload guard");
 
             let retained_staging = futures::future::join_all(
                 disk_stores
@@ -4015,20 +4136,25 @@ mod tests {
                     .await
             });
             signaling.wait_for_attempts(object_attempt).await;
-            assert!(!object_probe.is_finished(), "the in-flight completion must retain the object guard");
+            assert!(!object_probe.is_finished(), "the detached tail owner must retain the object guard");
 
             rename_barrier.release();
-            complete
-                .await
-                .expect("multipart task should join after tail release")
-                .expect("multipart completion should return after every tail rename finishes");
             object_probe
                 .await
-                .expect("object guard probe should join after completion releases")
-                .expect("object guard probe should acquire after completion releases");
+                .expect("object guard probe should join after the tail releases")
+                .expect("object guard probe should acquire after the tail releases");
+            tokio::time::timeout(Duration::from_secs(30), cleanup_barrier.wait_until_paused())
+                .await
+                .expect("the multipart tail should pause before reclaiming its old body");
+            let after_tail = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
+            assert!(
+                expected.is_subset(&after_tail),
+                "the multipart rename tail must re-mark capacity after the first scope was drained"
+            );
+            cleanup_barrier.release();
             let abort_err = abort
                 .await
-                .expect("abort task should join after completion releases")
+                .expect("abort task should join after the tail releases")
                 .expect_err("the committed upload should no longer exist");
             assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
 
@@ -4041,7 +4167,7 @@ mod tests {
             let after_cleanup = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
             assert!(
                 expected.is_subset(&after_cleanup),
-                "the completed multipart commit must mark every candidate disk dirty"
+                "the multipart tail cleanup must re-mark capacity after its preceding scope was drained"
             );
         })
         .await;
@@ -4176,26 +4302,23 @@ mod tests {
             ],
             async {
                 let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
-                let complete_store = Arc::clone(&set_disks);
-                let mut complete = tokio::spawn(async move {
-                    complete_store
-                        .complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &ObjectOptions::default())
-                        .await
-                });
+                set_disks
+                    .clone()
+                    .complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &ObjectOptions::default())
+                    .await
+                    .expect("fenced multipart completion should commit with a live proof");
 
                 tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
                     .await
-                    .expect("multipart completion should pause one tail disk during rename");
+                    .expect("multipart completion should leave one rename tail in flight after quorum ACK");
+                let disks = disk_stores.clone();
+                let mut epochs = tokio::spawn(async move { object_transaction_epochs(&disks, bucket, object).await });
                 assert!(
-                    tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
-                    "fenced multipart completion must wait for every rename tail before returning"
+                    tokio::time::timeout(Duration::from_millis(100), &mut epochs).await.is_err(),
+                    "epoch read-back should wait for the lagging rename tail"
                 );
                 rename_barrier.release();
-                complete
-                    .await
-                    .expect("fenced multipart task should join after tail release")
-                    .expect("fenced multipart completion should commit with a live proof");
-                object_transaction_epochs(&disk_stores, bucket, object).await
+                epochs.await.expect("epoch read-back should finish after the rename tail")
             },
         )
         .await;
@@ -7447,6 +7570,101 @@ mod tests {
         assert_eq!(bucket_wide.uploads[0].object, "blobs/data/layer.bin");
     }
 
+    #[tokio::test]
+    async fn data_movement_cleanup_discovery_rejects_quorum_visible_upload_without_metadata() {
+        let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "data-movement-unverifiable-upload";
+        let object = "staged/object.bin";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let upload_identity = format!("v1:{}:{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp_nanos());
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, upload_identity.clone());
+        let opts = ObjectOptions {
+            data_movement: true,
+            user_defined: metadata,
+            ..Default::default()
+        };
+        let upload = set_disks
+            .new_multipart_upload(bucket, object, &opts)
+            .await
+            .expect("data movement upload should be created");
+        let upload_path = SetDisks::get_multipart_upload_dir(bucket, object, &upload.upload_id, true);
+        for temp_dir in &temp_dirs {
+            tokio::fs::remove_file(
+                temp_dir
+                    .path()
+                    .join(RUSTFS_META_MULTIPART_BUCKET)
+                    .join(&upload_path)
+                    .join("xl.meta"),
+            )
+            .await
+            .expect("upload metadata should be removable while preserving its quorum-visible directory");
+        }
+
+        let err = set_disks
+            .data_movement_multipart_upload_ids(bucket, object, None, &upload_identity)
+            .await
+            .expect_err("cleanup discovery must fail closed on an unverifiable upload path");
+        assert!(matches!(err, Error::DecommissionCapacityBlocked { .. }));
+    }
+
+    #[tokio::test]
+    async fn data_movement_cleanup_discovery_rejects_minority_upload_until_disks_recover() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "data-movement-minority-upload";
+        let object = "staged/object.bin";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        {
+            let mut disks = set_disks.disks.write().await;
+            disks[3] = None;
+        }
+        let upload_identity = format!("v1:{}:{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp_nanos());
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD, upload_identity.clone());
+        let upload = set_disks
+            .new_multipart_upload(
+                bucket,
+                object,
+                &ObjectOptions {
+                    data_movement: true,
+                    user_defined: metadata,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("write quorum should create the upload while one disk is offline");
+
+        {
+            let mut disks = set_disks.disks.write().await;
+            disks[1] = None;
+            disks[2] = None;
+            disks[3] = Some(disk_stores[3].clone());
+        }
+        let err = set_disks
+            .data_movement_multipart_upload_ids(bucket, object, None, &upload_identity)
+            .await
+            .expect_err("a minority-observed upload must block a destructive absence proof");
+        assert!(matches!(err, Error::DecommissionCapacityBlocked { .. }));
+
+        {
+            let mut disks = set_disks.disks.write().await;
+            disks[1] = Some(disk_stores[1].clone());
+            disks[2] = Some(disk_stores[2].clone());
+        }
+        let recovered = set_disks
+            .data_movement_multipart_upload_ids(bucket, object, None, &upload_identity)
+            .await
+            .expect("recovered quorum should make the staged upload verifiable");
+        assert_eq!(recovered.len(), 1);
+        assert_eq!(upload_uuid_suffix(&recovered[0]), upload_uuid_suffix(&upload.upload_id));
+    }
+
     /// Regression (issue #5716): a single upload directory whose `xl.meta` was
     /// destroyed (crash mid-write, torn disk state) must degrade to that upload
     /// alone. Failing the whole ListMultipartUploads turns one piece of stale
@@ -8288,18 +8506,29 @@ mod tests {
             let new = payload(0xC3);
             let (u_new, parts_new) = stage_upload(&set_disks, bucket, object, &new).await;
             let parts_retry = parts_new.clone();
+            let rename_tasks = rename_fanout_barrier::observe_tasks(object);
+            let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
             crash_inject::arm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
             let crashed = complete(&set_disks, bucket, object, &u_new, parts_new).await;
             assert!(
                 matches!(crashed, Err(StorageError::Unexpected)),
                 "the armed post-commit crash point must be the failure that surfaced, got {crashed:?}"
             );
+            assert!(rename_tasks.running() >= 1, "the crash must interrupt an actual early-ACK tail handoff");
             crash_inject::disarm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
+            rename_barrier.release();
+            tokio::time::timeout(Duration::from_secs(30), async {
+                while rename_tasks.running() != 0 {
+                    tokio::task::yield_now().await;
+                }
+            })
+            .await
+            .expect("the crash-interrupted rename tail should drain after release");
             drop(
                 set_disks
                     .acquire_write_lock_diag("post_commit_crash_tail_probe", bucket, object)
                     .await
-                    .expect("the failed post-commit completion should release its object guard"),
+                    .expect("the crash-interrupted tail should release its object guard"),
             );
 
             // The commit landed: the new version reads back whole and correct.
@@ -8372,18 +8601,29 @@ mod tests {
 
                     let new = payload(0x52);
                     let (u_new, parts_new) = stage_upload(&set_disks, bucket, object, &new).await;
+                    let rename_tasks = rename_fanout_barrier::observe_tasks(object);
+                    let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
                     crash_inject::arm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
                     let crashed = complete(&set_disks, bucket, object, &u_new, parts_new).await;
                     assert!(
                         matches!(crashed, Err(StorageError::Unexpected)),
                         "the post-commit crash point must surface as unexpected, got {crashed:?}"
                     );
+                    assert!(rename_tasks.running() >= 1, "the crash must interrupt an actual early-ACK tail handoff");
                     crash_inject::disarm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
+                    rename_barrier.release();
+                    tokio::time::timeout(Duration::from_secs(30), async {
+                        while rename_tasks.running() != 0 {
+                            tokio::task::yield_now().await;
+                        }
+                    })
+                    .await
+                    .expect("the crash-interrupted rename tail should drain after release");
                     drop(
                         set_disks
                             .acquire_write_lock_diag("post_commit_receipt_tail_probe", bucket, object)
                             .await
-                            .expect("the failed post-commit completion should release its object guard"),
+                            .expect("the crash-interrupted tail should release its object guard"),
                     );
 
                     let (body, _) = read_object(&set_disks, bucket, object).await;
@@ -8397,8 +8637,8 @@ mod tests {
                         );
                     }
                     assert_eq!(
-                        receipts, 4,
-                        "the completed rename must persist old-data cleanup receipts on every disk before surfacing the post-commit crash"
+                        receipts, 3,
+                        "the committed quorum must persist receipts while the crash-interrupted tail preserves staging"
                     );
 
                     let restarted_endpoints = temp_dirs
@@ -8444,12 +8684,12 @@ mod tests {
                         .reconcile_old_data_cleanup_receipts(bucket, object)
                         .await
                         .expect("restart receipt reconciliation should succeed");
-                    assert_eq!(removed, 4, "restart receipt reconciliation should delete every committed target");
+                    assert_eq!(removed, 3, "restart receipt reconciliation should delete the committed quorum's targets");
                     let reclaimed = restarted_set
                         .reclaim_orphan_data_dirs(bucket, object)
                         .await
                         .expect("restart orphan reconciliation should succeed");
-                    assert_eq!(reclaimed, 0, "the post-commit crash should leave no receipt-less late commit orphan");
+                    assert_eq!(reclaimed, 1, "the late commit without a receipt must remain reclaimable as an orphan");
                     for disk in &reloaded {
                         assert!(
                             !data_dir_exists(disk, bucket, object, old_dir).await,

crates/ecstore/src/store/heal.rs

@@ -821,7 +821,6 @@ mod tests {
             decommission_cancelers: RwLock::new(Vec::new()),
             start_gate: Mutex::new(()),
             pool_meta_save_gate: Mutex::default(),
-            decommission_capacity_entry_gate: Mutex::default(),
             ctx: crate::runtime::instance::bootstrap_ctx(),
             bucket_fence_registry: std::sync::Arc::default(),
         }
@@ -2380,7 +2379,6 @@ mod tests {
             decommission_cancelers: RwLock::new(Vec::new()),
             start_gate: Mutex::new(()),
             pool_meta_save_gate: Mutex::default(),
-            decommission_capacity_entry_gate: Mutex::default(),
             ctx: crate::runtime::instance::bootstrap_ctx(),
             bucket_fence_registry: std::sync::Arc::default(),
         };

crates/ecstore/src/store/init.rs

@@ -580,7 +580,6 @@ impl ECStore {
             decommission_cancelers,
             start_gate: Mutex::new(()),
             pool_meta_save_gate: Mutex::new(pool_meta_write_state),
-            decommission_capacity_entry_gate: Mutex::default(),
             // Adopt the caller's context (the process bootstrap one on the
             // legacy path) so startup writes (erasure type recorded before
             // this point) and later reads share one cell.
@@ -918,6 +917,35 @@ mod tests {
     use time::OffsetDateTime;
     use tokio::io::AsyncReadExt;
     use tokio_util::sync::CancellationToken;
+
+    fn run_large_stack_async_test<C, F>(name: &str, case: C)
+    where
+        C: FnOnce() -> F + Send + 'static,
+        F: Future<Output = ()> + 'static,
+    {
+        const STACK_SIZE: usize = if cfg!(debug_assertions) {
+            8 * rustfs_config::DEFAULT_THREAD_STACK_SIZE
+        } else if cfg!(target_os = "macos") {
+            2 * rustfs_config::DEFAULT_THREAD_STACK_SIZE
+        } else {
+            rustfs_config::DEFAULT_THREAD_STACK_SIZE
+        };
+        std::thread::Builder::new()
+            .name(name.to_string())
+            .stack_size(STACK_SIZE)
+            .spawn(move || {
+                let runtime = tokio::runtime::Builder::new_multi_thread()
+                    .enable_all()
+                    .worker_threads(2)
+                    .thread_stack_size(STACK_SIZE)
+                    .build()
+                    .expect("large-stack store test runtime should build");
+                runtime.block_on(case());
+            })
+            .expect("large-stack store test thread should spawn")
+            .join()
+            .expect("large-stack store test thread should complete");
+    }
     use uuid::Uuid;
 
     #[test]
@@ -2306,203 +2334,17 @@ mod tests {
     }
 
     #[cfg(feature = "test-util")]
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    #[test]
     #[serial_test::serial(storage_class_env)]
-    async fn copy_object_immediately_reads_small_completed_multipart_source() {
-        let temp_dir = tempfile::tempdir().expect("create small multipart copy store dir");
-        let (_ctx, store, shutdown) =
-            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "small-multipart-copy", &[1])).await;
-        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
-
-        let bucket = format!("small-multipart-copy-{}", Uuid::new_v4());
-        let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
-        let target_object = "docker/registry/v2/blobs/sha256/c0/digest/data";
-        let payload = vec![0xAB; 273];
-
-        store
-            .make_bucket(&bucket, &MakeBucketOptions::default())
-            .await
-            .expect("create bucket for small multipart copy");
-        let upload = store
-            .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
-            .await
-            .expect("create source multipart upload");
-        let mut part_reader = PutObjReader::from_vec(payload.clone());
-        let part = store
-            .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
-            .await
-            .expect("stage small multipart source part");
-        let completed = store
-            .clone()
-            .complete_multipart_upload(
-                &bucket,
-                source_object,
-                &upload.upload_id,
-                vec![crate::storage_api_contracts::multipart::CompletePart {
-                    part_num: part.part_num,
-                    etag: part.etag,
-                    ..Default::default()
-                }],
-                &ObjectOptions::default(),
-            )
-            .await
-            .expect("complete the small multipart source");
-        assert_eq!(completed.get_actual_size().expect("completed object logical size"), payload.len() as i64);
-
-        let source_reader = store
-            .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
-            .await
-            .expect("completed multipart source should be immediately readable");
-        let mut copy_info = source_reader.object_info.clone();
-        let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
-        assert_eq!(actual_size, payload.len() as i64);
-        let copy_reader = rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
-            .expect("copy source hash reader should build");
-        copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
-
-        store
-            .copy_object(
-                &bucket,
-                source_object,
-                &bucket,
-                target_object,
-                &mut copy_info,
-                &ObjectOptions::default(),
-                &ObjectOptions::default(),
-            )
-            .await
-            .expect("CopyObject should accept a freshly completed multipart source");
-
-        let mut target_reader = store
-            .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
-            .await
-            .expect("copied target should be readable");
-        let mut target_body = Vec::new();
-        target_reader
-            .stream
-            .read_to_end(&mut target_body)
-            .await
-            .expect("target body should stream");
-        assert_eq!(target_body, payload);
-        shutdown.cancel();
+    fn data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease() {
+        run_large_stack_async_test(
+            "multipart-part-staging-publication-fence",
+            data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease_case,
+        );
     }
 
     #[cfg(feature = "test-util")]
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    #[serial_test::serial(storage_class_env)]
-    async fn complete_multipart_waits_for_tail_rename_before_copy_source_visibility() {
-        let temp_dir = tempfile::tempdir().expect("create early-ack multipart copy store dir");
-        let (_ctx, store, shutdown) =
-            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "early-ack-multipart-copy", &[4])).await;
-        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
-
-        let bucket = format!("early-ack-multipart-copy-{}", Uuid::new_v4());
-        let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
-        let target_object = "docker/registry/v2/blobs/sha256/c0/digest/data";
-        let payload = vec![0xCD; 273];
-
-        store
-            .make_bucket(&bucket, &MakeBucketOptions::default())
-            .await
-            .expect("create bucket for early-ack multipart copy");
-        let upload = store
-            .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
-            .await
-            .expect("create source multipart upload");
-        let mut part_reader = PutObjReader::from_vec(payload.clone());
-        let part = store
-            .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
-            .await
-            .expect("stage small multipart source part");
-        let completed_parts = vec![crate::storage_api_contracts::multipart::CompletePart {
-            part_num: part.part_num,
-            etag: part.etag,
-            ..Default::default()
-        }];
-
-        temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
-            let rename_tasks = crate::set_disk::rename_fanout_barrier::observe_tasks(source_object);
-            let rename_barrier = crate::set_disk::rename_fanout_barrier::arm(
-                source_object,
-                0,
-                crate::set_disk::rename_fanout_barrier::PHASE_RENAME,
-            );
-            let complete_store = Arc::clone(&store);
-            let complete_bucket = bucket.clone();
-            let complete_upload_id = upload.upload_id.clone();
-            let mut complete = tokio::spawn(async move {
-                complete_store
-                    .complete_multipart_upload(
-                        &complete_bucket,
-                        source_object,
-                        &complete_upload_id,
-                        completed_parts,
-                        &ObjectOptions::default(),
-                    )
-                    .await
-            });
-            tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
-                .await
-                .expect("multipart completion should pause one tail disk during rename");
-            assert!(
-                rename_tasks.running() >= 1,
-                "the paused multipart tail disk must remain in flight before completion returns"
-            );
-            assert!(
-                tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
-                "CompleteMultipartUpload must not return while a copy source rename tail is still pending"
-            );
-            rename_barrier.release();
-            complete
-                .await
-                .expect("multipart completion task should join")
-                .expect("multipart completion should return after every rename tail finishes");
-
-            let source_reader = store
-                .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
-                .await
-                .expect("completed multipart source should be immediately readable after success");
-            let mut copy_info = source_reader.object_info.clone();
-            let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
-            assert_eq!(actual_size, payload.len() as i64);
-            let copy_reader =
-                rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
-                    .expect("copy source hash reader should build");
-            copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
-
-            store
-                .copy_object(
-                    &bucket,
-                    source_object,
-                    &bucket,
-                    target_object,
-                    &mut copy_info,
-                    &ObjectOptions::default(),
-                    &ObjectOptions::default(),
-                )
-                .await
-                .expect("CopyObject should accept a freshly completed multipart source");
-        })
-        .await;
-
-        let mut target_reader = store
-            .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
-            .await
-            .expect("copied target should be readable after tail release");
-        let mut target_body = Vec::new();
-        target_reader
-            .stream
-            .read_to_end(&mut target_body)
-            .await
-            .expect("target body should stream");
-        assert_eq!(target_body, payload);
-        shutdown.cancel();
-    }
-
-    #[cfg(feature = "test-util")]
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    #[serial_test::serial(storage_class_env)]
-    async fn data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease() {
+    async fn data_movement_multipart_part_staging_holds_no_publication_lock_or_tier_lease_case() {
         let temp_dir = tempfile::tempdir().expect("create multipart staging-fence store dir");
         let (ctx, store, shutdown) =
             without_storage_class_env(build_isolated_test_store(temp_dir.path(), "mpu-staging-publication", &[4, 4])).await;
@@ -2582,19 +2424,23 @@ mod tests {
         let capacity_owner = test_decommission_capacity_owner(store.as_ref(), 0)
             .await
             .with_mutation_id(Uuid::new_v4());
-        let mut upload_metadata = source.user_defined.as_ref().clone();
-        rustfs_utils::http::insert_str(
-            &mut upload_metadata,
-            rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
-            "mpu-staging-test".to_string(),
-        );
+        let upload_metadata = source.user_defined.as_ref().clone();
         let mut staging_opts = ObjectOptions {
             data_movement: true,
             src_pool_idx: 0,
+            versioned: source.version_id.is_some(),
+            version_id: source.version_id.map(|version_id| version_id.to_string()),
+            mod_time: source.mod_time,
             user_defined: upload_metadata,
             expected_bucket_incarnation_id: Some(bucket_incarnation_id),
             ..Default::default()
         };
+        let upload_identity = crate::data_movement::data_movement_upload_identity_from_options(&staging_opts);
+        rustfs_utils::http::insert_str(
+            &mut staging_opts.user_defined,
+            rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
+            upload_identity,
+        );
         capacity_owner.apply_to(&mut staging_opts);
         let (upload, target_pool_idx, staged_incarnation_id) = store
             .handle_new_multipart_upload_with_pool_idx(&bucket, object, &staging_opts, None)
@@ -2664,6 +2510,9 @@ mod tests {
         let mut abort_opts = ObjectOptions {
             data_movement: true,
             src_pool_idx: 0,
+            versioned: source.version_id.is_some(),
+            version_id: source.version_id.map(|version_id| version_id.to_string()),
+            mod_time: source.mod_time,
             expected_bucket_incarnation_id: Some(bucket_incarnation_id),
             ..Default::default()
         };
@@ -5166,8 +5015,9 @@ mod tests {
                     let ordinary_faults_for_hook = Arc::clone(&ordinary_faults);
                     let fault_bucket = other_bucket.clone();
                     let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
-                        move |stage, bucket, object, attempt| {
-                            let injected = stage == DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
+                        move |stage, bucket, object, attempt, succeeded| {
+                            let injected = succeeded
+                                && stage == DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
                                 && bucket == fault_bucket.as_str()
                                 && object == other_object
                                 && attempt < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS;
@@ -5436,8 +5286,9 @@ mod tests {
                     let fault_calls_for_hook = Arc::clone(&fault_calls);
                     let fault_bucket = bucket.clone();
                     let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
-                        move |stage, called_bucket, called_object, attempt| {
-                            let injected = stage == DECOMMISSION_TEST_FAULT_STAGE_DELETE_MARKER
+                        move |stage, called_bucket, called_object, attempt, succeeded| {
+                            let injected = succeeded
+                                && stage == DECOMMISSION_TEST_FAULT_STAGE_DELETE_MARKER
                                 && called_bucket == fault_bucket.as_str()
                                 && called_object == object
                                 && attempt < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS;
@@ -5556,8 +5407,9 @@ mod tests {
                     let fault_calls_for_hook = Arc::clone(&fault_calls);
                     let fault_bucket = bucket.clone();
                     let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
-                        move |stage, called_bucket, called_object, attempt| {
-                            let injected = stage == DECOMMISSION_TEST_FAULT_STAGE_TIERED
+                        move |stage, called_bucket, called_object, attempt, succeeded| {
+                            let injected = succeeded
+                                && stage == DECOMMISSION_TEST_FAULT_STAGE_TIERED
                                 && called_bucket == fault_bucket.as_str()
                                 && called_object == object
                                 && attempt < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS;
@@ -5662,9 +5514,16 @@ mod tests {
         shutdown.cancel();
     }
 
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    #[test]
     #[serial_test::serial(storage_class_env)]
-    async fn decommission_outer_fence_loss_blocks_multipart_commits() {
+    fn decommission_outer_fence_loss_blocks_multipart_commits() {
+        run_large_stack_async_test(
+            "decommission-multipart-outer-fence-loss",
+            decommission_outer_fence_loss_blocks_multipart_commits_case,
+        );
+    }
+
+    async fn decommission_outer_fence_loss_blocks_multipart_commits_case() {
         for (object, pause) in [
             ("complete.bin", crate::set_disk::MultipartCommitPause::BeforeLockLost),
             ("new-upload.bin", crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost),
@@ -10213,6 +10072,29 @@ mod tests {
             .expect("the active decommission should own the checkpoint target");
         assert_eq!(targets.len(), 1);
         let target = targets[0].clone();
+        let consumed_before_checkpoint = store.pool_meta.read().await.pools[0]
+            .decommission
+            .as_ref()
+            .and_then(|info| info.capacity_reservation.as_ref())
+            .expect("checkpoint shutdown capacity reservation should exist")
+            .consumed_target_physical_bytes;
+        let precommit_error = store
+            .run_decommission_capacity_non_growing_replacement_with_capacity_lease(
+                target.target_pool_index,
+                Some(target.capacity_owner),
+                Some(aborting_data.len()),
+                |_| async { Err::<(), Error>(Error::other("injected checkpoint failure before commit")) },
+            )
+            .await
+            .expect_err("the first checkpoint attempt should fail before writing its target")
+            .to_string();
+        assert!(precommit_error.contains("injected checkpoint failure before commit"));
+        assert!(
+            store
+                .has_decommission_capacity_temporary_mutation_state(target.target_pool_index, target.capacity_owner)
+                .await,
+            "a failed checkpoint attempt must retain exact retry state"
+        );
         let barrier = crate::set_disk::PutObjectCommitBarrier::install(
             RUSTFS_META_BUCKET,
             &manifest_name,
@@ -10263,6 +10145,27 @@ mod tests {
                 .await,
             "the admitted target PUT must drain its capacity transaction before releasing the recovery fences"
         );
+        {
+            let pool_meta = store.pool_meta.read().await;
+            let reservation = pool_meta.pools[0]
+                .decommission
+                .as_ref()
+                .and_then(|info| info.capacity_reservation.as_ref())
+                .expect("checkpoint shutdown capacity reservation should remain active");
+            let capacity_target = reservation
+                .targets
+                .iter()
+                .find(|candidate| candidate.pool_index == target.target_pool_index)
+                .expect("checkpoint shutdown capacity target should remain allocated");
+            assert_eq!(
+                reservation.consumed_target_physical_bytes, consumed_before_checkpoint,
+                "a non-growing checkpoint replacement must not consume durable migration capacity"
+            );
+            assert_eq!(reservation.pending_target_physical_bytes, 0);
+            assert_eq!(reservation.inflight_target_physical_bytes, 0);
+            assert_eq!(capacity_target.pending_physical_bytes, 0);
+            assert!(capacity_target.temporary_mutations.is_empty());
+        }
         drop(barrier);
         let mut reloaded_pool_meta = PoolMeta::default();
         reloaded_pool_meta

crates/ecstore/src/store/mod.rs

@@ -468,12 +468,6 @@ pub struct ECStore {
     /// Lock order: acquire `pool_meta_save_gate`, then the distributed
     /// `pool.bin` fence, then clone `pool_meta` under a short read lock.
     pub(crate) pool_meta_save_gate: Mutex<PoolMetaWriteState>,
-    /// Serializes decommission entries while the durable capacity ledger has
-    /// one target mutation intent slot.
-    ///
-    /// Lock order: acquire this gate before object namespaces or
-    /// `pool_meta_save_gate`.
-    pub(crate) decommission_capacity_entry_gate: Mutex<()>,
     /// Per-instance runtime state (Phase 5, backlog#939).
     ///
     /// Carries this instance's identity/runtime out of the process globals so
@@ -1728,7 +1722,6 @@ mod tests {
             decommission_cancelers: RwLock::new(Vec::new()),
             start_gate: Mutex::new(()),
             pool_meta_save_gate: Mutex::default(),
-            decommission_capacity_entry_gate: Mutex::default(),
             ctx,
             bucket_fence_registry: Arc::default(),
         };
@@ -1804,7 +1797,6 @@ mod tests {
             decommission_cancelers: RwLock::new(Vec::new()),
             start_gate: Mutex::new(()),
             pool_meta_save_gate: Mutex::default(),
-            decommission_capacity_entry_gate: Mutex::default(),
             ctx,
             bucket_fence_registry: Arc::default(),
         })

crates/ecstore/src/store/multipart.rs

@@ -17,11 +17,52 @@ use crate::core::pools::{DecommissionCapacityOwner, ensure_decommission_capacity
 use crate::multipart_listing::paginate_multipart_listing;
 use crate::set_disk::get_lock_acquire_timeout;
 use crate::storage_api_contracts::multipart::MultipartOperations as _;
+use crate::storage_api_contracts::object::ObjectOperations as _;
 use futures::{StreamExt, stream};
 use std::collections::HashSet;
 
 const MULTIPART_LIST_SET_CONCURRENCY: usize = 4;
 
+#[cfg(test)]
+static DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<Uuid, usize>>> =
+    std::sync::OnceLock::new();
+
+#[cfg(test)]
+fn data_movement_multipart_discovery_counts() -> &'static std::sync::Mutex<std::collections::HashMap<Uuid, usize>> {
+    DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
+}
+
+fn decommission_multipart_target_clear_pending(opts: &ObjectOptions, target: Option<&ObjectInfo>) -> Result<bool> {
+    let expected_mod_time = opts.mod_time.ok_or_else(|| Error::DecommissionCapacityBlocked {
+        message: "multipart cleanup cannot prove exact target absence without a modification time".to_string(),
+    })?;
+    let expected_version_id = opts
+        .version_id
+        .as_deref()
+        .map(Uuid::parse_str)
+        .transpose()
+        .map_err(|err| Error::DecommissionCapacityBlocked {
+            message: format!("multipart cleanup exact target version is invalid: {err}"),
+        })?
+        .filter(|version_id| !version_id.is_nil());
+    let Some(target) = target else {
+        return Ok(true);
+    };
+    if target.version_id.filter(|version_id| !version_id.is_nil()) != expected_version_id
+        || target.mod_time != Some(expected_mod_time)
+    {
+        return Err(Error::DecommissionCapacityBlocked {
+            message: "multipart cleanup found a target but cannot prove the exact staged identity is absent".to_string(),
+        });
+    }
+    if !crate::data_movement::is_owned_data_movement_target(target) {
+        return Err(Error::DecommissionCapacityBlocked {
+            message: "multipart cleanup found an exact target without its ownership proof".to_string(),
+        });
+    }
+    Ok(false)
+}
+
 #[derive(Clone, Debug)]
 pub(super) struct MultipartUploadListRequest {
     pub(super) prefix: String,
@@ -197,6 +238,24 @@ async fn list_pool_multipart_uploads_for_incarnation(
 }
 
 impl ECStore {
+    #[cfg(test)]
+    pub(crate) fn reset_data_movement_multipart_discovery_count_for_test(&self) {
+        data_movement_multipart_discovery_counts()
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .insert(self.id, 0);
+    }
+
+    #[cfg(test)]
+    pub(crate) fn data_movement_multipart_discovery_count_for_test(&self) -> usize {
+        data_movement_multipart_discovery_counts()
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .get(&self.id)
+            .copied()
+            .unwrap_or_default()
+    }
+
     pub(crate) async fn acquire_decommission_multipart_mutation_fence(
         &self,
         owner: DecommissionCapacityOwner,
@@ -728,8 +787,16 @@ impl ECStore {
         upload_id: &str,
         opts: &ObjectOptions,
     ) -> Result<()> {
-        self.abort_multipart_uploads_for_data_movement(target_pool_idx, bucket, object, &[upload_id.to_owned()], None, opts)
-            .await
+        let upload_identity = crate::data_movement::data_movement_upload_identity_from_options(opts);
+        self.abort_multipart_uploads_for_data_movement(
+            target_pool_idx,
+            bucket,
+            object,
+            &[upload_id.to_owned()],
+            &upload_identity,
+            opts,
+        )
+        .await
     }
 
     pub(crate) async fn reconcile_multipart_uploads_for_data_movement(
@@ -740,26 +807,37 @@ impl ECStore {
         upload_identity: &str,
         opts: &ObjectOptions,
     ) -> Result<()> {
-        let pool = self
-            .pools
+        self.pools
             .get(target_pool_idx)
             .ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?;
-        let owner = DecommissionCapacityOwner::from_options(opts);
-        let has_capacity_state = match owner {
-            Some(owner) => {
-                self.has_decommission_capacity_temporary_mutation_state(target_pool_idx, owner)
-                    .await
-            }
-            None => false,
-        };
-        if !has_capacity_state {
+        let owner = DecommissionCapacityOwner::from_options(opts)
+            .ok_or_else(|| Error::other("data movement multipart cleanup is missing its capacity owner"))?;
+        if !self
+            .decommission_capacity_cleanup_target_indices(owner)
+            .await?
+            .contains(&target_pool_idx)
+        {
+            return Err(Error::DecommissionCapacityBlocked {
+                message: format!(
+                    "data movement multipart cleanup target pool {target_pool_idx} is outside its capacity reservation"
+                ),
+            });
+        }
+        if !self
+            .has_decommission_capacity_temporary_mutation_state(target_pool_idx, owner)
+            .await
+        {
             return Ok(());
         }
-        let set = pool.get_disks_by_key(object);
-        let upload_ids = set
-            .data_movement_multipart_upload_ids(bucket, object, opts.expected_bucket_incarnation_id, upload_identity)
-            .await?;
-        self.abort_multipart_uploads_for_data_movement(target_pool_idx, bucket, object, &upload_ids, Some(upload_identity), opts)
+        #[cfg(test)]
+        {
+            *data_movement_multipart_discovery_counts()
+                .lock()
+                .unwrap_or_else(std::sync::PoisonError::into_inner)
+                .entry(self.id)
+                .or_default() += 1;
+        }
+        self.abort_multipart_uploads_for_data_movement(target_pool_idx, bucket, object, &[], upload_identity, opts)
             .await
     }
 
@@ -769,7 +847,7 @@ impl ECStore {
         bucket: &str,
         object: &str,
         upload_ids: &[String],
-        expected_upload_identity: Option<&str>,
+        expected_upload_identity: &str,
         opts: &ObjectOptions,
     ) -> Result<()> {
         check_new_multipart_args(bucket, object)?;
@@ -781,42 +859,116 @@ impl ECStore {
         }
         let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
         ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
+        let capacity_owner = DecommissionCapacityOwner::from_options(&opts);
         let pool = self
             .pools
             .get(target_pool_idx)
             .ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?;
-        let set = pool.get_disks_by_key(object);
-        let mut guards = Vec::with_capacity(upload_ids.len());
-        for upload_id in upload_ids {
-            if let Some(guard) = set
-                .lock_data_movement_multipart_abort(bucket, object, upload_id, expected_upload_identity, &opts)
-                .await?
-            {
-                guard.add_namespace_lock_fence(&mut opts);
-                guards.push(guard);
-            }
-        }
         opts.no_lock = true;
-        let capacity_owner = DecommissionCapacityOwner::from_options(&opts);
-        // Keep every upload namespace guard alive through the final capacity progress save.
-        let result = self
+        let set = pool.get_disks_by_key(object);
+        // Discover and lock uploads only after the target capacity gate is held.
+        // Return the guards so they remain alive through the final capacity save.
+        let (cleanup_decision_error, guards) = self
             .run_decommission_capacity_temporary_release_with_capacity_lease(target_pool_idx, capacity_owner, |capacity_lease| {
                 let mut delete_opts = opts.clone();
-                let guards = &guards;
                 let set = &set;
+                let pool = &pool;
                 async move {
-                    if let Some(capacity_lease) = capacity_lease {
-                        delete_opts.add_namespace_lock_lost_signal(capacity_lease);
+                    if let Some(capacity_lease) = capacity_lease.as_ref() {
+                        delete_opts.add_namespace_lock_lost_signal(Arc::clone(capacity_lease));
                     }
-                    for guard in guards {
-                        guard.delete(set, bucket, object, &delete_opts).await?;
+                    let mut candidate_upload_ids = upload_ids.to_vec();
+                    candidate_upload_ids.extend(
+                        set.data_movement_multipart_upload_ids(
+                            bucket,
+                            object,
+                            delete_opts.expected_bucket_incarnation_id,
+                            expected_upload_identity,
+                        )
+                        .await?,
+                    );
+                    candidate_upload_ids.sort_unstable();
+                    candidate_upload_ids.dedup();
+
+                    let mut guards = Vec::with_capacity(candidate_upload_ids.len());
+                    for upload_id in &candidate_upload_ids {
+                        match set
+                            .lock_data_movement_multipart_abort(
+                                bucket,
+                                object,
+                                upload_id,
+                                Some(expected_upload_identity),
+                                &delete_opts,
+                            )
+                            .await
+                        {
+                            Ok(Some(guard)) => {
+                                guard.add_namespace_lock_fence(&mut delete_opts);
+                                guards.push(guard);
+                            }
+                            Ok(None) => {}
+                            Err(err) => return Err(err),
+                        }
                     }
-                    Ok(())
+                    for guard in &guards {
+                        match guard.delete(set, bucket, object, &delete_opts).await {
+                            Ok(()) => {}
+                            Err(err) if is_err_invalid_upload_id(&err) => {}
+                            Err(err) => return Err(err),
+                        }
+                    }
+
+                    if !set
+                        .data_movement_multipart_upload_ids(
+                            bucket,
+                            object,
+                            delete_opts.expected_bucket_incarnation_id,
+                            expected_upload_identity,
+                        )
+                        .await?
+                        .is_empty()
+                    {
+                        return Err(Error::DecommissionCapacityBlocked {
+                            message: "multipart cleanup could not prove the exact staged uploads are absent".to_string(),
+                        });
+                    }
+
+                    // The target capacity gate makes the exact target proof,
+                    // upload absence proof, and pending-ledger decision one
+                    // critical section with cleanup finalize.
+                    let (clear_pending, cleanup_decision_error) = if capacity_owner.is_some() {
+                        let mut lookup_opts = ObjectOptions {
+                            versioned: delete_opts.versioned,
+                            version_suspended: delete_opts.version_suspended,
+                            version_id: delete_opts.version_id.clone(),
+                            metadata_chg: delete_opts.version_id.is_some(),
+                            no_lock: true,
+                            ..Default::default()
+                        };
+                        if let Some(capacity_lease) = capacity_lease {
+                            lookup_opts.add_namespace_lock_lost_signal(capacity_lease);
+                        }
+                        let target = match pool.get_object_info(bucket, object, &lookup_opts).await {
+                            Ok(target) => Some(target),
+                            Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => None,
+                            Err(err) => return Err(err),
+                        };
+                        match decommission_multipart_target_clear_pending(&delete_opts, target.as_ref()) {
+                            Ok(clear_pending) => (clear_pending, None),
+                            Err(err) => (false, Some(err)),
+                        }
+                    } else {
+                        (true, None)
+                    };
+                    Ok(((cleanup_decision_error, guards), clear_pending))
                 }
             })
-            .await;
+            .await?;
         drop(guards);
-        result
+        if let Some(err) = cleanup_decision_error {
+            return Err(err);
+        }
+        Ok(())
     }
 
     #[instrument(skip(self))]
@@ -1024,6 +1176,66 @@ mod tests {
         }
     }
 
+    #[test]
+    fn decommission_multipart_cleanup_requires_exact_target_evidence() {
+        let version_id = Uuid::new_v4();
+        let mod_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(7);
+        let opts = ObjectOptions {
+            versioned: true,
+            version_id: Some(version_id.to_string()),
+            mod_time: Some(mod_time),
+            ..Default::default()
+        };
+
+        assert!(
+            decommission_multipart_target_clear_pending(&opts, None)
+                .expect("an exact target miss should authorize pending cleanup")
+        );
+
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_DATA_MOVED, "true".to_string());
+        rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_DATA_MOVED_TAGS, "v1:".to_string());
+        let owned_target = ObjectInfo {
+            version_id: Some(version_id),
+            mod_time: Some(mod_time),
+            user_defined: Arc::new(metadata),
+            ..Default::default()
+        };
+        assert!(
+            !decommission_multipart_target_clear_pending(&opts, Some(&owned_target))
+                .expect("an exact owned target should preserve pending capacity")
+        );
+
+        let missing_identity = ObjectOptions {
+            mod_time: None,
+            ..opts.clone()
+        };
+        assert!(matches!(
+            decommission_multipart_target_clear_pending(&missing_identity, None),
+            Err(Error::DecommissionCapacityBlocked { .. })
+        ));
+
+        let mismatched_target = ObjectInfo {
+            version_id: Some(Uuid::new_v4()),
+            mod_time: Some(mod_time),
+            ..Default::default()
+        };
+        assert!(matches!(
+            decommission_multipart_target_clear_pending(&opts, Some(&mismatched_target)),
+            Err(Error::DecommissionCapacityBlocked { .. })
+        ));
+
+        let unowned_target = ObjectInfo {
+            version_id: Some(version_id),
+            mod_time: Some(mod_time),
+            ..Default::default()
+        };
+        assert!(matches!(
+            decommission_multipart_target_clear_pending(&opts, Some(&unowned_target)),
+            Err(Error::DecommissionCapacityBlocked { .. })
+        ));
+    }
+
     /// Models a single pool's `list_multipart_uploads`: returns uploads strictly
     /// after the `(key, upload_id)` marker in `(key, upload_id)` order, capped at
     /// `max_uploads` (mirroring the per-pool page cap).
@@ -1171,7 +1383,6 @@ mod tests {
             decommission_cancelers: RwLock::new(Vec::new()),
             start_gate: Mutex::new(()),
             pool_meta_save_gate: Mutex::default(),
-            decommission_capacity_entry_gate: Mutex::default(),
             ctx: crate::runtime::instance::bootstrap_ctx(),
             bucket_fence_registry: std::sync::Arc::default(),
         }

crates/ecstore/src/store/object.rs

@@ -1531,6 +1531,10 @@ fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> Object
     writer_pool_lookup_opts(opts, no_lock)
 }
 
+fn uses_data_movement_pool_selection(opts: &ObjectOptions) -> bool {
+    opts.data_movement && (opts.version_id.is_some() || DecommissionCapacityOwner::from_options(opts).is_some())
+}
+
 fn writer_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
     let mut lookup_opts = version_aware_lookup_opts(opts, no_lock);
     lookup_opts.skip_decommissioned = true;
@@ -3467,7 +3471,12 @@ impl ECStore {
     }
 
     fn resolve_decommission_tiered_object_result(result: Result<()>, bucket: &str, object: &str) -> Result<()> {
-        result.map_err(|err| Error::other(format!("failed to decommission tiered object for {bucket}/{object}: {err}")))
+        result.map_err(|err| {
+            crate::data_movement::data_movement_context_error(
+                format!("failed to decommission tiered object for {bucket}/{object}: {err}"),
+                err,
+            )
+        })
     }
 
     #[instrument(skip(self, fi, opts))]
@@ -3515,7 +3524,7 @@ impl ECStore {
             );
         }
 
-        let idx = if opts.data_movement && opts.version_id.is_some() {
+        let idx = if uses_data_movement_pool_selection(&opts) {
             Self::resolve_decommission_target_pool_idx_result(
                 self.select_data_movement_pool_idx(bucket, &object, fi.size, &opts, true)
                     .await,
@@ -3713,7 +3722,7 @@ impl ECStore {
             return Ok(0);
         }
 
-        let idx = if opts.data_movement && opts.version_id.is_some() {
+        let idx = if uses_data_movement_pool_selection(opts) {
             self.select_data_movement_pool_idx(bucket, object, size, opts, false).await?
         } else if opts.no_lock {
             self.get_pool_idx_no_lock(bucket, object, size).await?
@@ -7277,6 +7286,23 @@ mod tests {
         assert!(rendered.contains("boom"), "{rendered}");
     }
 
+    #[test]
+    fn resolve_decommission_tiered_object_result_preserves_typed_capacity_error() {
+        let err = ECStore::resolve_decommission_tiered_object_result(
+            Err(Error::DecommissionCapacityBlocked {
+                message: "target gate busy".to_string(),
+            }),
+            "bucket",
+            "object",
+        )
+        .expect_err("expected contextual error");
+
+        assert!(matches!(
+            crate::data_movement::data_movement_stage_source(&err),
+            Some(Error::DecommissionCapacityBlocked { message }) if message == "target gate busy"
+        ));
+    }
+
     #[test]
     fn version_aware_lookup_opts_enables_version_aware_lookup() {
         let opts = ObjectOptions {
@@ -7502,6 +7528,26 @@ mod tests {
         assert!(lookup_opts.skip_rebalancing);
     }
 
+    #[test]
+    fn capacity_owned_unversioned_move_uses_data_movement_pool_selection() {
+        let mut opts = ObjectOptions {
+            data_movement: true,
+            ..Default::default()
+        };
+        assert!(!uses_data_movement_pool_selection(&opts));
+
+        DecommissionCapacityOwner {
+            source_pool_index: 1,
+            operation_id: Uuid::new_v4(),
+            generation: 2,
+            owner_nonce: Uuid::new_v4(),
+            mutation_id: None,
+        }
+        .apply_to(&mut opts);
+
+        assert!(uses_data_movement_pool_selection(&opts));
+    }
+
     #[test]
     fn transition_restore_pool_opts_skips_decommissioned_and_preserves_locking() {
         let lookup_opts = transition_restore_pool_opts(&ObjectOptions {
@@ -7576,7 +7622,6 @@ mod tests {
             decommission_cancelers: RwLock::new(Vec::new()),
             start_gate: Mutex::new(()),
             pool_meta_save_gate: Mutex::default(),
-            decommission_capacity_entry_gate: Mutex::default(),
             ctx: crate::runtime::instance::bootstrap_ctx(),
             bucket_fence_registry: std::sync::Arc::default(),
         }
@@ -7669,7 +7714,6 @@ mod tests {
             decommission_cancelers: RwLock::new(Vec::new()),
             start_gate: Mutex::new(()),
             pool_meta_save_gate: Mutex::default(),
-            decommission_capacity_entry_gate: Mutex::default(),
             ctx,
             bucket_fence_registry: std::sync::Arc::default(),
         }

crates/protocols/src/common/client/s3.rs

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use async_trait::async_trait;
-use rustfs_credentials::Credentials;
 use s3s::dto::*;
 
 #[cfg(feature = "webdav")]
@@ -28,33 +27,49 @@ pub trait StorageBackend: Send + Sync {
         &self,
         bucket: &str,
         key: &str,
-        credentials: &Credentials,
+        access_key: &str,
+        secret_key: &str,
         start_pos: Option<u64>,
     ) -> Result<GetObjectOutput, Self::Error>;
     async fn get_object_range(
         &self,
         bucket: &str,
         key: &str,
-        credentials: &Credentials,
+        access_key: &str,
+        secret_key: &str,
         start_pos: u64,
         length: u64,
     ) -> Result<GetObjectOutput, Self::Error>;
     /// Put object content with metadata
-    async fn put_object(&self, input: PutObjectInput, credentials: &Credentials) -> Result<PutObjectOutput, Self::Error>;
+    async fn put_object(&self, input: PutObjectInput, access_key: &str, secret_key: &str)
+    -> Result<PutObjectOutput, Self::Error>;
     /// Delete an object
-    async fn delete_object(&self, bucket: &str, key: &str, credentials: &Credentials) -> Result<DeleteObjectOutput, Self::Error>;
+    async fn delete_object(
+        &self,
+        bucket: &str,
+        key: &str,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<DeleteObjectOutput, Self::Error>;
     /// Get object metadata without content
-    async fn head_object(&self, bucket: &str, key: &str, credentials: &Credentials) -> Result<HeadObjectOutput, Self::Error>;
+    async fn head_object(
+        &self,
+        bucket: &str,
+        key: &str,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<HeadObjectOutput, Self::Error>;
     /// Check if bucket exists and get metadata
-    async fn head_bucket(&self, bucket: &str, credentials: &Credentials) -> Result<HeadBucketOutput, Self::Error>;
+    async fn head_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<HeadBucketOutput, Self::Error>;
     /// List objects in a bucket with pagination
     async fn list_objects_v2(
         &self,
         input: ListObjectsV2Input,
-        credentials: &Credentials,
+        access_key: &str,
+        secret_key: &str,
     ) -> Result<ListObjectsV2Output, Self::Error>;
     /// List all buckets (requires authentication).
-    async fn list_buckets(&self, credentials: &Credentials) -> Result<ListBucketsOutput, Self::Error>;
+    async fn list_buckets(&self, access_key: &str, secret_key: &str) -> Result<ListBucketsOutput, Self::Error>;
     /// List buckets visible to the authenticated session.
     ///
     /// Backends that implement this must apply per-bucket authorization. The default denies the
@@ -72,15 +87,20 @@ pub trait StorageBackend: Send + Sync {
         ))
     }
     /// Create a new bucket
-    async fn create_bucket(&self, bucket: &str, credentials: &Credentials) -> Result<CreateBucketOutput, Self::Error>;
+    async fn create_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<CreateBucketOutput, Self::Error>;
     /// Delete a bucket (must be empty)
-    async fn delete_bucket(&self, bucket: &str, credentials: &Credentials) -> Result<DeleteBucketOutput, Self::Error>;
+    async fn delete_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<DeleteBucketOutput, Self::Error>;
     /// Server-side copy of an object from one bucket+key to another.
     /// The input carries the full S3 surface (content type, metadata map,
     /// metadata directive, storage class, SSE config, conditional-copy
     /// headers) so protocol drivers can map client-supplied metadata
     /// onto the destination object.
-    async fn copy_object(&self, input: CopyObjectInput, credentials: &Credentials) -> Result<CopyObjectOutput, Self::Error>;
+    async fn copy_object(
+        &self,
+        input: CopyObjectInput,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<CopyObjectOutput, Self::Error>;
     /// Initiate a multipart upload. Returns an upload_id that identifies
     /// the in-progress upload for subsequent UploadPart, CompleteMultipartUpload,
     /// and AbortMultipartUpload calls. The input carries the full S3 surface
@@ -90,18 +110,25 @@ pub trait StorageBackend: Send + Sync {
     async fn create_multipart_upload(
         &self,
         input: CreateMultipartUploadInput,
-        credentials: &Credentials,
+        access_key: &str,
+        secret_key: &str,
     ) -> Result<CreateMultipartUploadOutput, Self::Error>;
     /// Upload one part of a multipart upload. The part_number must be in
     /// the range 1 to the 10 000-part S3 limit. The returned ETag
     /// identifies the part in the subsequent CompleteMultipartUpload call.
-    async fn upload_part(&self, input: UploadPartInput, credentials: &Credentials) -> Result<UploadPartOutput, Self::Error>;
+    async fn upload_part(
+        &self,
+        input: UploadPartInput,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<UploadPartOutput, Self::Error>;
     /// Assemble the parts listed in the input into the final object.
     /// The parts list must be sorted by part_number with no duplicates.
     async fn complete_multipart_upload(
         &self,
         input: CompleteMultipartUploadInput,
-        credentials: &Credentials,
+        access_key: &str,
+        secret_key: &str,
     ) -> Result<CompleteMultipartUploadOutput, Self::Error>;
     /// Abort an in-progress multipart upload. Releases any storage
     /// associated with the upload_id. Idempotent: calling abort on an
@@ -111,7 +138,8 @@ pub trait StorageBackend: Send + Sync {
     async fn abort_multipart_upload(
         &self,
         input: AbortMultipartUploadInput,
-        credentials: &Credentials,
+        access_key: &str,
+        secret_key: &str,
     ) -> Result<AbortMultipartUploadOutput, Self::Error>;
     /// Copy a byte range from an existing object into a part of an
     /// in-progress multipart upload. Used by rename for objects larger
@@ -119,6 +147,7 @@ pub trait StorageBackend: Send + Sync {
     async fn upload_part_copy(
         &self,
         input: UploadPartCopyInput,
-        credentials: &Credentials,
+        access_key: &str,
+        secret_key: &str,
     ) -> Result<UploadPartCopyOutput, Self::Error>;
 }

crates/protocols/src/common/dummy_storage.rs

@@ -35,7 +35,6 @@ use crate::common::session::SessionContext;
 use async_trait::async_trait;
 use bytes::Bytes;
 use futures_util::stream::{self, StreamExt};
-use rustfs_credentials::Credentials;
 use s3s::dto::{
     AbortMultipartUploadInput, AbortMultipartUploadOutput, CompleteMultipartUploadInput, CompleteMultipartUploadOutput,
     CopyObjectInput, CopyObjectOutput, CopyPartResult, CreateBucketOutput, CreateMultipartUploadInput,
@@ -606,7 +605,8 @@ impl StorageBackend for DummyBackend {
         &self,
         bucket: &str,
         key: &str,
-        _credentials: &Credentials,
+        _ak: &str,
+        _sk: &str,
         _start_pos: Option<u64>,
     ) -> Result<GetObjectOutput, Self::Error> {
         match self.inner.lock().expect("lock").get_object.pop_front() {
@@ -619,7 +619,8 @@ impl StorageBackend for DummyBackend {
         &self,
         bucket: &str,
         key: &str,
-        _credentials: &Credentials,
+        _ak: &str,
+        _sk: &str,
         _start_pos: u64,
         _length: u64,
     ) -> Result<GetObjectOutput, Self::Error> {
@@ -629,7 +630,7 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn put_object(&self, input: PutObjectInput, _credentials: &Credentials) -> Result<PutObjectOutput, Self::Error> {
+    async fn put_object(&self, input: PutObjectInput, _ak: &str, _sk: &str) -> Result<PutObjectOutput, Self::Error> {
         // Decide control flow while holding the lock. Release before
         // awaiting so the stall path does not hold the Mutex across
         // an await point.
@@ -658,12 +659,7 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn delete_object(
-        &self,
-        bucket: &str,
-        key: &str,
-        _credentials: &Credentials,
-    ) -> Result<DeleteObjectOutput, Self::Error> {
+    async fn delete_object(&self, bucket: &str, key: &str, _ak: &str, _sk: &str) -> Result<DeleteObjectOutput, Self::Error> {
         let mut inner = self.inner.lock().expect("lock");
         inner.delete_object_calls.push(DeleteObjectCall {
             bucket: bucket.to_string(),
@@ -675,7 +671,7 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn head_object(&self, bucket: &str, key: &str, _credentials: &Credentials) -> Result<HeadObjectOutput, Self::Error> {
+    async fn head_object(&self, bucket: &str, key: &str, _ak: &str, _sk: &str) -> Result<HeadObjectOutput, Self::Error> {
         {
             let mut inner = self.inner.lock().expect("lock");
             inner.head_object_calls.push(HeadObjectCall {
@@ -689,7 +685,7 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn head_bucket(&self, bucket: &str, _credentials: &Credentials) -> Result<HeadBucketOutput, Self::Error> {
+    async fn head_bucket(&self, bucket: &str, _ak: &str, _sk: &str) -> Result<HeadBucketOutput, Self::Error> {
         match self.inner.lock().expect("lock").head_bucket.pop_front() {
             Some(r) => r,
             None => Err(DummyError::NoSuchBucket(bucket.to_string())),
@@ -699,7 +695,8 @@ impl StorageBackend for DummyBackend {
     async fn list_objects_v2(
         &self,
         _input: ListObjectsV2Input,
-        _credentials: &Credentials,
+        _ak: &str,
+        _sk: &str,
     ) -> Result<ListObjectsV2Output, Self::Error> {
         // Decide control flow while holding the lock. Release before
         // awaiting so the stall path does not hold the Mutex across
@@ -724,7 +721,7 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn list_buckets(&self, _credentials: &Credentials) -> Result<ListBucketsOutput, Self::Error> {
+    async fn list_buckets(&self, _access_key: &str, _secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
         match self.inner.lock().expect("lock").list_buckets.pop_front() {
             Some(r) => r,
             None => Ok(ListBucketsOutput::default()),
@@ -747,14 +744,14 @@ impl StorageBackend for DummyBackend {
             .unwrap_or_else(|| Ok(ListBucketsOutput::default()))
     }
 
-    async fn create_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<CreateBucketOutput, Self::Error> {
+    async fn create_bucket(&self, _bucket: &str, _ak: &str, _sk: &str) -> Result<CreateBucketOutput, Self::Error> {
         match self.inner.lock().expect("lock").create_bucket.pop_front() {
             Some(r) => r,
             None => Err(DummyError::Unconfigured("create_bucket")),
         }
     }
 
-    async fn delete_bucket(&self, bucket: &str, _credentials: &Credentials) -> Result<DeleteBucketOutput, Self::Error> {
+    async fn delete_bucket(&self, bucket: &str, _ak: &str, _sk: &str) -> Result<DeleteBucketOutput, Self::Error> {
         let mut inner = self.inner.lock().expect("lock");
         inner.delete_bucket_calls.push(bucket.to_string());
         match inner.delete_bucket.pop_front() {
@@ -763,7 +760,7 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn copy_object(&self, _input: CopyObjectInput, _credentials: &Credentials) -> Result<CopyObjectOutput, Self::Error> {
+    async fn copy_object(&self, _input: CopyObjectInput, _ak: &str, _sk: &str) -> Result<CopyObjectOutput, Self::Error> {
         match self.inner.lock().expect("lock").copy_object.pop_front() {
             Some(r) => r,
             None => Err(DummyError::Unconfigured("copy_object")),
@@ -773,7 +770,8 @@ impl StorageBackend for DummyBackend {
     async fn create_multipart_upload(
         &self,
         input: CreateMultipartUploadInput,
-        _credentials: &Credentials,
+        _ak: &str,
+        _sk: &str,
     ) -> Result<CreateMultipartUploadOutput, Self::Error> {
         {
             let mut inner = self.inner.lock().expect("lock");
@@ -789,7 +787,7 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn upload_part(&self, input: UploadPartInput, _credentials: &Credentials) -> Result<UploadPartOutput, Self::Error> {
+    async fn upload_part(&self, input: UploadPartInput, _ak: &str, _sk: &str) -> Result<UploadPartOutput, Self::Error> {
         // Record the call and decide the control flow while holding the
         // lock. Release the lock before awaiting so the stall path does
         // not hold the Mutex across an await point.
@@ -823,7 +821,8 @@ impl StorageBackend for DummyBackend {
     async fn complete_multipart_upload(
         &self,
         input: CompleteMultipartUploadInput,
-        _credentials: &Credentials,
+        _ak: &str,
+        _sk: &str,
     ) -> Result<CompleteMultipartUploadOutput, Self::Error> {
         let part_count = input
             .multipart_upload
@@ -848,7 +847,8 @@ impl StorageBackend for DummyBackend {
     async fn abort_multipart_upload(
         &self,
         input: AbortMultipartUploadInput,
-        _credentials: &Credentials,
+        _ak: &str,
+        _sk: &str,
     ) -> Result<AbortMultipartUploadOutput, Self::Error> {
         {
             let mut inner = self.inner.lock().expect("lock");
@@ -867,7 +867,8 @@ impl StorageBackend for DummyBackend {
     async fn upload_part_copy(
         &self,
         _input: UploadPartCopyInput,
-        _credentials: &Credentials,
+        _ak: &str,
+        _sk: &str,
     ) -> Result<UploadPartCopyOutput, Self::Error> {
         match self.inner.lock().expect("lock").upload_part_copy.pop_front() {
             Some(r) => r,
@@ -883,8 +884,7 @@ mod tests {
     #[tokio::test]
     async fn dummy_backend_reports_not_found_by_default() {
         let backend = DummyBackend::new();
-        let credentials = Credentials::default();
-        let result = backend.head_object("b", "k", &credentials).await;
+        let result = backend.head_object("b", "k", "ak", "sk").await;
         let Err(err) = result else {
             panic!("default head_object must return an error");
         };
@@ -897,23 +897,21 @@ mod tests {
     #[tokio::test]
     async fn dummy_backend_returns_queued_head_object_response() {
         let backend = DummyBackend::new();
-        let credentials = Credentials::default();
         backend.queue_head_object_ok(42, None);
-        let out = backend.head_object("b", "k", &credentials).await.expect("queued Ok");
+        let out = backend.head_object("b", "k", "ak", "sk").await.expect("queued Ok");
         assert_eq!(out.content_length, Some(42));
     }
 
     #[tokio::test]
     async fn dummy_backend_logs_abort_multipart_calls() {
         let backend = Arc::new(DummyBackend::new());
-        let credentials = Credentials::default();
         let input = AbortMultipartUploadInput::builder()
             .bucket("b".to_string())
             .key("k".to_string())
             .upload_id("UP-1".to_string())
             .build()
             .expect("build");
-        backend.abort_multipart_upload(input, &credentials).await.expect("Ok");
+        backend.abort_multipart_upload(input, "ak", "sk").await.expect("Ok");
         let calls = backend.abort_multipart_calls();
         assert_eq!(calls.len(), 1);
         assert_eq!(calls[0].upload_id, "UP-1");
@@ -922,7 +920,6 @@ mod tests {
     #[tokio::test]
     async fn dummy_backend_unconfigured_errors_loudly() {
         let backend = DummyBackend::new();
-        let credentials = Credentials::default();
         let err = backend
             .create_multipart_upload(
                 CreateMultipartUploadInput::builder()
@@ -930,7 +927,8 @@ mod tests {
                     .key("k".to_string())
                     .build()
                     .expect("build"),
-                &credentials,
+                "ak",
+                "sk",
             )
             .await
             .expect_err("default create_multipart_upload must error");

crates/protocols/src/common/gateway.rs

@@ -288,7 +288,12 @@ pub async fn is_authorized(
         }
     };
 
-    let claims = policy_claims_for_session(session_context);
+    // Create policy arguments
+    let mut claims = HashMap::new();
+    claims.insert(
+        "principal".to_string(),
+        serde_json::Value::String(session_context.principal.access_key().to_string()),
+    );
 
     let policy_action: rustfs_policy::policy::action::Action = action.clone().into();
 
@@ -310,21 +315,6 @@ pub async fn is_authorized(
     Ok(iam_sys.is_allowed(&args).await)
 }
 
-fn policy_claims_for_session(session_context: &SessionContext) -> HashMap<String, serde_json::Value> {
-    let mut claims = session_context
-        .principal
-        .user_identity
-        .credentials
-        .claims
-        .clone()
-        .unwrap_or_default();
-    claims.insert(
-        "principal".to_string(),
-        serde_json::Value::String(session_context.principal.access_key().to_string()),
-    );
-    claims
-}
-
 /// Authorize an operation and return an error if not authorized.
 /// AccessDenied covers both the protocol-not-supported case and the
 /// policy-denies case. IamUnavailable propagates from is_authorized
@@ -467,9 +457,7 @@ pub use test_auth_override::{with_test_auth_override, with_test_iam_unavailable}
 mod tests {
     use super::*;
     use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext};
-    use rustfs_credentials::{IAM_POLICY_CLAIM_NAME_SA, INHERITED_POLICY_TYPE};
     use rustfs_policy::auth::UserIdentity;
-    use serde_json::Value;
     use std::net::{IpAddr, Ipv4Addr};
     use std::sync::Arc;
 
@@ -478,44 +466,6 @@ mod tests {
         SessionContext::new(principal, Protocol::Sftp, IpAddr::V4(Ipv4Addr::LOCALHOST))
     }
 
-    fn session_with_claims(access_key: &str, claims: HashMap<String, Value>) -> SessionContext {
-        let identity = UserIdentity::new(rustfs_credentials::Credentials {
-            access_key: access_key.to_string(),
-            secret_key: "secret".to_string(),
-            claims: Some(claims),
-            ..Default::default()
-        });
-        let principal = ProtocolPrincipal::new(Arc::new(identity));
-        SessionContext::new(principal, Protocol::WebDav, IpAddr::V4(Ipv4Addr::LOCALHOST))
-    }
-
-    #[test]
-    fn policy_claims_preserve_authenticated_service_account_claims() {
-        let parent = "parent-user";
-        let mut stored_claims = HashMap::new();
-        stored_claims.insert("parent".to_string(), Value::String(parent.to_string()));
-        stored_claims.insert(IAM_POLICY_CLAIM_NAME_SA.to_string(), Value::String(INHERITED_POLICY_TYPE.to_string()));
-        let session = session_with_claims("service-account", stored_claims);
-
-        let claims = policy_claims_for_session(&session);
-
-        assert_eq!(claims.get("parent").and_then(Value::as_str), Some(parent));
-        assert_eq!(claims.get(IAM_POLICY_CLAIM_NAME_SA).and_then(Value::as_str), Some(INHERITED_POLICY_TYPE));
-        assert_eq!(claims.get("principal").and_then(Value::as_str), Some("service-account"));
-    }
-
-    #[test]
-    fn policy_claims_overwrite_untrusted_principal_claim() {
-        let session = session_with_claims(
-            "authenticated-service-account",
-            HashMap::from([("principal".to_string(), Value::String("forged-principal".to_string()))]),
-        );
-
-        let claims = policy_claims_for_session(&session);
-
-        assert_eq!(claims.get("principal").and_then(Value::as_str), Some("authenticated-service-account"));
-    }
-
     #[tokio::test]
     async fn with_test_auth_override_allow_returns_ok() {
         let session = test_session();

crates/protocols/src/common/session.rs

@@ -84,11 +84,6 @@ impl SessionContext {
     pub fn access_key(&self) -> &str {
         self.principal.access_key()
     }
-
-    /// Get the authenticated credentials for this session.
-    pub fn credentials(&self) -> &Credentials {
-        &self.principal.user_identity.credentials
-    }
 }
 
 /// Build a SessionContext suitable for driver-level unit tests. The

crates/protocols/src/ftps/driver.rs

@@ -129,7 +129,14 @@ where
         }
 
         let mut list_result = Vec::new();
-        match self.storage.list_buckets(session_context.credentials()).await {
+        match self
+            .storage
+            .list_buckets(
+                &session_context.principal.user_identity.credentials.access_key,
+                &session_context.principal.user_identity.credentials.secret_key,
+            )
+            .await
+        {
             Ok(output) => {
                 if let Some(buckets) = output.buckets {
                     for bucket in buckets {
@@ -183,7 +190,15 @@ where
                 Error::new(ErrorKind::PermanentFileNotAvailable, format!("Failed to build ListObjectsV2Input: {}", e))
             })?;
 
-            if let Ok(output) = self.storage.list_objects_v2(list_input, session_context.credentials()).await {
+            if let Ok(output) = self
+                .storage
+                .list_objects_v2(
+                    list_input,
+                    &session_context.principal.user_identity.credentials.access_key,
+                    &session_context.principal.user_identity.credentials.secret_key,
+                )
+                .await
+            {
                 // Delete all objects in this page
                 if let Some(objects) = output.contents {
                     for obj in objects {
@@ -194,7 +209,12 @@ where
 
                             let _ = self
                                 .storage
-                                .delete_object(bucket, &obj_key, session_context.credentials())
+                                .delete_object(
+                                    bucket,
+                                    &obj_key,
+                                    &session_context.principal.user_identity.credentials.access_key,
+                                    &session_context.principal.user_identity.credentials.secret_key,
+                                )
                                 .await;
                         }
                     }
@@ -211,7 +231,15 @@ where
         }
 
         // Then delete the bucket
-        match self.storage.delete_bucket(bucket, session_context.credentials()).await {
+        match self
+            .storage
+            .delete_bucket(
+                bucket,
+                &session_context.principal.user_identity.credentials.access_key,
+                &session_context.principal.user_identity.credentials.secret_key,
+            )
+            .await
+        {
             Ok(_) => Ok(()),
             Err(e) if e.to_string().contains("NoSuchBucket") => Ok(()),
             Err(e) => {
@@ -249,7 +277,16 @@ where
                 .await
                 .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
 
-            match self.storage.head_object(&bucket, &key, session_context.credentials()).await {
+            match self
+                .storage
+                .head_object(
+                    &bucket,
+                    &key,
+                    &session_context.principal.user_identity.credentials.access_key,
+                    &session_context.principal.user_identity.credentials.secret_key,
+                )
+                .await
+            {
                 Ok(output) => {
                     let size = output.content_length.unwrap_or(0) as u64;
                     let modified = output.last_modified.map(|dt| {
@@ -286,7 +323,15 @@ where
                 .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
 
             let bucket_clone = bucket.clone();
-            match self.storage.head_bucket(&bucket, session_context.credentials()).await {
+            match self
+                .storage
+                .head_bucket(
+                    &bucket,
+                    &session_context.principal.user_identity.credentials.access_key,
+                    &session_context.principal.user_identity.credentials.secret_key,
+                )
+                .await
+            {
                 Ok(_) => Ok(FtpsMetadata {
                     size: 0,
                     modified: Some(std::time::SystemTime::now()),
@@ -345,7 +390,15 @@ where
                 Error::new(ErrorKind::PermanentFileNotAvailable, format!("Failed to build ListObjectsV2Input: {}", e))
             })?;
 
-        match self.storage.list_objects_v2(list_input, session_context.credentials()).await {
+        match self
+            .storage
+            .list_objects_v2(
+                list_input,
+                &session_context.principal.user_identity.credentials.access_key,
+                &session_context.principal.user_identity.credentials.secret_key,
+            )
+            .await
+        {
             Ok(output) => {
                 let mut fileinfos = Vec::new();
 
@@ -462,7 +515,8 @@ where
             .get_object(
                 &bucket,
                 &key,
-                session_context.credentials(),
+                &session_context.principal.user_identity.credentials.access_key,
+                &session_context.principal.user_identity.credentials.secret_key,
                 Some(start_pos), // Pass start_pos for range request
             )
             .await
@@ -570,7 +624,15 @@ where
             .build()
             .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Failed to build PutObjectInput"))?;
 
-        match self.storage.put_object(put_input, session_context.credentials()).await {
+        match self
+            .storage
+            .put_object(
+                put_input,
+                &session_context.principal.user_identity.credentials.access_key,
+                &session_context.principal.user_identity.credentials.secret_key,
+            )
+            .await
+        {
             Ok(_output) => {
                 Ok(file_size as u64) // Return the size of the uploaded object
             }
@@ -619,7 +681,16 @@ where
                 .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
 
             // Delete file
-            match self.storage.delete_object(&bucket, &key, session_context.credentials()).await {
+            match self
+                .storage
+                .delete_object(
+                    &bucket,
+                    &key,
+                    &session_context.principal.user_identity.credentials.access_key,
+                    &session_context.principal.user_identity.credentials.secret_key,
+                )
+                .await
+            {
                 Ok(_) => Ok(()),
                 Err(e) => {
                     error!(
@@ -677,7 +748,15 @@ where
             .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
 
         // Create bucket for directory
-        match self.storage.create_bucket(&bucket, session_context.credentials()).await {
+        match self
+            .storage
+            .create_bucket(
+                &bucket,
+                &session_context.principal.user_identity.credentials.access_key,
+                &session_context.principal.user_identity.credentials.secret_key,
+            )
+            .await
+        {
             Ok(_) => {
                 debug!(
                     event = EVENT_FTPS_DIRECTORY_STATE,
@@ -777,7 +856,15 @@ where
             .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
 
         // Check if bucket exists
-        match self.storage.head_bucket(&bucket, session_context.credentials()).await {
+        match self
+            .storage
+            .head_bucket(
+                &bucket,
+                &session_context.principal.user_identity.credentials.access_key,
+                &session_context.principal.user_identity.credentials.secret_key,
+            )
+            .await
+        {
             Ok(_) => Ok(()),
             Err(e) => {
                 error!(

crates/protocols/src/sftp/attrs.rs

@@ -137,7 +137,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             // on success. Size and mtime are not returned by HeadBucket.
             None => {
                 self.authorize(&S3Action::HeadBucket, &bucket, None).await?;
-                self.run_backend("head_bucket", self.storage.head_bucket(&bucket, self.credentials()))
+                self.run_backend("head_bucket", self.storage.head_bucket(&bucket, self.access_key(), self.secret_key()))
                     .await?;
                 Ok(s3_attrs_to_sftp(0, None, true))
             }
@@ -154,7 +154,11 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             Some(object_key) => {
                 self.authorize(&S3Action::HeadObject, &bucket, Some(&object_key)).await?;
                 match self
-                    .run_backend_with_err("head_object", self.storage.head_object(&bucket, &object_key, self.credentials()))
+                    .run_backend_with_err(
+                        "head_object",
+                        self.storage
+                            .head_object(&bucket, &object_key, self.access_key(), self.secret_key()),
+                    )
                     .await?
                 {
                     Ok(out) => {
@@ -179,7 +183,10 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
                             .build()
                             .map_err(|e| s3_error_to_sftp("build_list_objects", e))?;
                         let out = self
-                            .run_backend("list_objects_v2", self.storage.list_objects_v2(input, self.credentials()))
+                            .run_backend(
+                                "list_objects_v2",
+                                self.storage.list_objects_v2(input, self.access_key(), self.secret_key()),
+                            )
                             .await?;
 
                         let has_contents = out.contents.map(|c| !c.is_empty()).unwrap_or(false);

crates/protocols/src/sftp/dir.rs

@@ -102,7 +102,10 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
         let input = builder.build().map_err(|e| s3_error_to_sftp("build_list_objects", e))?;
 
         let out = self
-            .run_backend("list_objects_v2", self.storage.list_objects_v2(input, self.credentials()))
+            .run_backend(
+                "list_objects_v2",
+                self.storage.list_objects_v2(input, self.access_key(), self.secret_key()),
+            )
             .await?;
 
         let mut entries = Vec::new();
@@ -193,7 +196,10 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
         // Issue list_objects_v2. On Err the destructive caller never
         // runs because validate_directory_empty returns the Err.
         let out = self
-            .run_backend("list_objects_v2", self.storage.list_objects_v2(input, self.credentials()))
+            .run_backend(
+                "list_objects_v2",
+                self.storage.list_objects_v2(input, self.access_key(), self.secret_key()),
+            )
             .await?;
 
         // Count content entries that are not the directory's own marker.
@@ -228,7 +234,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
         self.authorize(&S3Action::ListBuckets, "", None).await?;
 
         let out = self
-            .run_backend("list_buckets", self.storage.list_buckets(self.credentials()))
+            .run_backend("list_buckets", self.storage.list_buckets(self.access_key(), self.secret_key()))
             .await?;
 
         let mut entries = Vec::new();
@@ -274,7 +280,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
     /// MKDIR for a bucket-level path: authorise and issue CreateBucket.
     pub(super) async fn mkdir_bucket(&self, bucket: &str) -> Result<(), SftpError> {
         self.authorize(&S3Action::CreateBucket, bucket, None).await?;
-        self.run_backend("create_bucket", self.storage.create_bucket(bucket, self.credentials()))
+        self.run_backend("create_bucket", self.storage.create_bucket(bucket, self.access_key(), self.secret_key()))
             .await?;
         Ok(())
     }
@@ -296,7 +302,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             .body(Some(streaming))
             .build()
             .map_err(|e| s3_error_to_sftp("build_put_object", e))?;
-        self.run_backend("put_object", self.storage.put_object(input, self.credentials()))
+        self.run_backend("put_object", self.storage.put_object(input, self.access_key(), self.secret_key()))
             .await?;
         Ok(())
     }
@@ -306,7 +312,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
     pub(super) async fn rmdir_bucket(&self, bucket: &str) -> Result<(), SftpError> {
         self.validate_directory_empty(bucket, "").await?;
         self.authorize(&S3Action::DeleteBucket, bucket, None).await?;
-        self.run_backend("delete_bucket", self.storage.delete_bucket(bucket, self.credentials()))
+        self.run_backend("delete_bucket", self.storage.delete_bucket(bucket, self.access_key(), self.secret_key()))
             .await?;
         Ok(())
     }
@@ -320,8 +326,12 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
 
         let marker_key = path::encode_dir_object(&prefix);
         self.authorize(&S3Action::DeleteObject, bucket, Some(&marker_key)).await?;
-        self.run_backend("delete_object", self.storage.delete_object(bucket, &marker_key, self.credentials()))
-            .await?;
+        self.run_backend(
+            "delete_object",
+            self.storage
+                .delete_object(bucket, &marker_key, self.access_key(), self.secret_key()),
+        )
+        .await?;
         Ok(())
     }
 
@@ -388,7 +398,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
                 if prefix.is_empty() { None } else { Some(prefix.as_str()) },
             )
             .await?;
-            self.run_backend("head_bucket", self.storage.head_bucket(&bucket, self.credentials()))
+            self.run_backend("head_bucket", self.storage.head_bucket(&bucket, self.access_key(), self.secret_key()))
                 .await?;
             DirCursor::Listing {
                 bucket,

crates/protocols/src/sftp/driver.rs

@@ -34,7 +34,6 @@ use crate::common::client::s3::StorageBackend;
 use crate::common::gateway::{AuthorizationError, S3Action, authorize_operation};
 use crate::common::session::SessionContext;
 use russh_sftp::protocol::{Attrs, Data, File, FileAttributes, Handle, Name, OpenFlags, Packet, Status, StatusCode, Version};
-use rustfs_credentials::Credentials;
 use rustfs_utils::MaskedAccessKey;
 use s3s::dto::{AbortMultipartUploadInput, CopyObjectInput, CopySource};
 use std::collections::HashMap;
@@ -166,14 +165,16 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
         super::read_cache::ReadCache::new(Arc::clone(&self.read_cache_in_use))
     }
 
-    /// Borrow the authenticated principal's S3 access key for diagnostics.
+    /// Borrow the authenticated principal's S3 access key. Each StorageBackend
+    /// call needs this alongside the secret key for signing.
     pub(super) fn access_key(&self) -> &str {
-        &self.credentials().access_key
+        &self.session_context.principal.user_identity.credentials.access_key
     }
 
-    /// Borrow the authenticated principal credentials for backend calls.
-    pub(super) fn credentials(&self) -> &Credentials {
-        self.session_context.credentials()
+    /// Borrow the authenticated principal's S3 secret key. Used together with
+    /// access_key for signing every backend call.
+    pub(super) fn secret_key(&self) -> &str {
+        &self.session_context.principal.user_identity.credentials.secret_key
     }
 
     /// Returns Err(PermissionDenied) when the driver is read-only,
@@ -786,8 +787,12 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
 
         self.authorize(&S3Action::DeleteObject, &bucket, Some(&object_key)).await?;
 
-        self.run_backend("delete_object", self.storage.delete_object(&bucket, &object_key, self.credentials()))
-            .await?;
+        self.run_backend(
+            "delete_object",
+            self.storage
+                .delete_object(&bucket, &object_key, self.access_key(), self.secret_key()),
+        )
+        .await?;
         Ok(ok_status(id))
     }
 
@@ -893,7 +898,11 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
         // single-shot vs multipart-copy branch below.
         self.authorize(&S3Action::HeadObject, &src_bucket, Some(&src_object)).await?;
         let head = self
-            .run_backend("head_object", self.storage.head_object(&src_bucket, &src_object, self.credentials()))
+            .run_backend(
+                "head_object",
+                self.storage
+                    .head_object(&src_bucket, &src_object, self.access_key(), self.secret_key()),
+            )
             .await?;
         let content_length = head.content_length.unwrap_or(0).max(0) as u64;
 
@@ -911,7 +920,7 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
                 .key(dst_object.clone())
                 .build()
                 .map_err(|e| s3_error_to_sftp("build_copy_object", e))?;
-            self.run_backend("copy_object", self.storage.copy_object(input, self.credentials()))
+            self.run_backend("copy_object", self.storage.copy_object(input, self.access_key(), self.secret_key()))
                 .await?;
         } else {
             self.multipart_copy(&src_bucket, &src_object, &dst_bucket, &dst_object, content_length)
@@ -923,8 +932,12 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
         // delete separately.
         self.authorize(&S3Action::DeleteObject, &src_bucket, Some(&src_object))
             .await?;
-        self.run_backend("delete_object", self.storage.delete_object(&src_bucket, &src_object, self.credentials()))
-            .await?;
+        self.run_backend(
+            "delete_object",
+            self.storage
+                .delete_object(&src_bucket, &src_object, self.access_key(), self.secret_key()),
+        )
+        .await?;
 
         Ok(ok_status(id))
     }
@@ -1016,12 +1029,14 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
 impl<S: StorageBackend + Send + Sync + 'static> Drop for SftpDriver<S> {
     fn drop(&mut self) {
         // Snapshot credentials, peer IP, and the per-call backend
-        // timeout before draining the handle table. Borrowing
-        // self.session_context inside the loop would conflict with the
-        // mutable borrow of self.handles. The timeout is copied into each
-        // spawned abort task so the deadline applies uniformly to inline
-        // calls and Drop-time aborts.
-        let credentials = self.session_context.credentials().clone();
+        // timeout before draining the handle table. self.access_key()
+        // and self.secret_key() borrow self.session_context immutably,
+        // which conflicts with the mutable borrow of self.handles
+        // inside the loop. The timeout is copied into each spawned
+        // abort task so the deadline applies uniformly to inline calls
+        // and Drop-time aborts.
+        let access_key = self.session_context.principal.user_identity.credentials.access_key.clone();
+        let secret_key = self.session_context.principal.user_identity.credentials.secret_key.clone();
         let peer = self.session_context.source_ip;
         let backend_op_timeout_secs = self.backend_op_timeout_secs;
 
@@ -1041,7 +1056,7 @@ impl<S: StorageBackend + Send + Sync + 'static> Drop for SftpDriver<S> {
                             key = %key,
                             upload_id = %upload_id,
                             peer = %peer,
-                            access_key = %MaskedAccessKey(&credentials.access_key),
+                            access_key = %access_key,
                             "skipped abort of orphaned multipart upload on session drop, principal lacks s3:AbortMultipartUpload, bucket lifecycle rules must reclaim parts",
                         );
                     }
@@ -1050,7 +1065,8 @@ impl<S: StorageBackend + Send + Sync + 'static> Drop for SftpDriver<S> {
             };
 
             let storage = Arc::clone(&self.storage);
-            let credentials = credentials.clone();
+            let access_key = access_key.clone();
+            let secret_key = secret_key.clone();
             let upload_id = upload_id_owned;
 
             // Cap the global abort fan-out so a burst of session
@@ -1106,7 +1122,7 @@ impl<S: StorageBackend + Send + Sync + 'static> Drop for SftpDriver<S> {
                     };
                     match tokio::time::timeout(
                         std::time::Duration::from_secs(backend_op_timeout_secs),
-                        storage.abort_multipart_upload(input, &credentials),
+                        storage.abort_multipart_upload(input, &access_key, &secret_key),
                     )
                     .await
                     {

crates/protocols/src/sftp/read.rs

@@ -46,7 +46,11 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
         // the body. These are cached on the handle so READ can detect EOF
         // and FSTAT can answer without another backend call.
         let head = self
-            .run_backend("head_object", self.storage.head_object(&bucket, &object_key, self.credentials()))
+            .run_backend(
+                "head_object",
+                self.storage
+                    .head_object(&bucket, &object_key, self.access_key(), self.secret_key()),
+            )
             .await?;
         let size = head.content_length.unwrap_or(0).max(0) as u64;
         let mtime = timestamp_to_mtime(head.last_modified);
@@ -162,7 +166,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             .run_backend(
                 "get_object_range",
                 self.storage
-                    .get_object_range(bucket, key, self.credentials(), offset, fetch_len),
+                    .get_object_range(bucket, key, self.access_key(), self.secret_key(), offset, fetch_len),
             )
             .await?;
 

crates/protocols/src/sftp/write.rs

@@ -293,7 +293,11 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             // not-found error means the key is free. Any other error is
             // propagated rather than misinterpreted as "does not exist".
             match self
-                .run_backend_with_err("head_object", self.storage.head_object(&bucket, &object_key, self.credentials()))
+                .run_backend_with_err(
+                    "head_object",
+                    self.storage
+                        .head_object(&bucket, &object_key, self.access_key(), self.secret_key()),
+                )
                 .await?
             {
                 Ok(_) => return Err(SftpError::code(StatusCode::Failure)),
@@ -381,7 +385,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
                 .map_err(|e| s3_error_to_sftp("build_put_object", e))?;
 
             let outcome = self
-                .run_backend_with_err("put_object", self.storage.put_object(input, self.credentials()))
+                .run_backend_with_err("put_object", self.storage.put_object(input, self.access_key(), self.secret_key()))
                 .await?;
 
             let backend_err = match outcome {
@@ -444,7 +448,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             .map_err(|e| s3_error_to_sftp("build_upload_part", e))?;
 
         let out = self
-            .run_backend("upload_part", self.storage.upload_part(input, self.credentials()))
+            .run_backend("upload_part", self.storage.upload_part(input, self.access_key(), self.secret_key()))
             .await?;
 
         let e_tag = out.e_tag.ok_or_else(|| {
@@ -524,7 +528,11 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             .map_err(|e| s3_error_to_sftp("build_create_multipart_upload", e))?;
 
         let out = self
-            .run_backend("create_multipart_upload", self.storage.create_multipart_upload(input, self.credentials()))
+            .run_backend(
+                "create_multipart_upload",
+                self.storage
+                    .create_multipart_upload(input, self.access_key(), self.secret_key()),
+            )
             .await?;
 
         let upload_id = out.upload_id.ok_or_else(|| {
@@ -577,7 +585,8 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
         let result = self
             .run_backend(
                 "complete_multipart_upload",
-                self.storage.complete_multipart_upload(input, self.credentials()),
+                self.storage
+                    .complete_multipart_upload(input, self.access_key(), self.secret_key()),
             )
             .await;
         result?;
@@ -843,8 +852,12 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             .build()
             .map_err(|e| s3_error_to_sftp("build_abort_multipart_upload", e))?;
 
-        self.run_backend("abort_multipart_upload", self.storage.abort_multipart_upload(input, self.credentials()))
-            .await?;
+        self.run_backend(
+            "abort_multipart_upload",
+            self.storage
+                .abort_multipart_upload(input, self.access_key(), self.secret_key()),
+        )
+        .await?;
         Ok(())
     }
 
@@ -1053,7 +1066,10 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
                     .map_err(|e| s3_error_to_sftp("build_upload_part_copy", e))?;
 
                 let out = self
-                    .run_backend("upload_part_copy", self.storage.upload_part_copy(input, self.credentials()))
+                    .run_backend(
+                        "upload_part_copy",
+                        self.storage.upload_part_copy(input, self.access_key(), self.secret_key()),
+                    )
                     .await?;
 
                 let e_tag = out.copy_part_result.and_then(|r| r.e_tag).ok_or_else(|| {
@@ -1109,7 +1125,6 @@ mod tests {
     use crate::common::dummy_storage::{AbortCall, DummyBackend, DummyError};
     use crate::common::gateway::with_test_auth_override;
     use russh_sftp::protocol::{FileAttributes, OpenFlags, StatusCode};
-    use rustfs_credentials::Credentials;
     use s3s::dto::ETag;
     use std::sync::Arc;
     use std::time::Duration;
@@ -2309,10 +2324,9 @@ mod tests {
         let backend = Arc::new(DummyBackend::new());
         backend.queue_head_object_err(DummyError::AccessDenied("pinned".to_string()));
         let driver = build_driver(backend, TEST_PART_SIZE);
-        let credentials = Credentials::default();
 
         let result = driver
-            .run_backend_with_err("head_object", driver.storage.head_object("b", "k", &credentials))
+            .run_backend_with_err("head_object", driver.storage.head_object("b", "k", "ak", "sk"))
             .await;
 
         match result {

crates/protocols/src/webdav/driver.rs

@@ -22,7 +22,6 @@ use dav_server::fs::{
 };
 use futures_util::{FutureExt, StreamExt, stream};
 use percent_encoding::percent_decode_str;
-use rustfs_credentials::Credentials;
 use rustfs_utils::MaskedAccessKey;
 use rustfs_utils::path;
 use s3s::S3ErrorCode;
@@ -199,7 +198,15 @@ where
         let key = self.key.clone();
 
         async move {
-            match storage.head_object(&bucket, &key, session_context.credentials()).await {
+            match storage
+                .head_object(
+                    &bucket,
+                    &key,
+                    &session_context.principal.user_identity.credentials.access_key,
+                    &session_context.principal.user_identity.credentials.secret_key,
+                )
+                .await
+            {
                 Ok(output) => {
                     let size = output.content_length.unwrap_or(0) as u64;
                     let modified = output
@@ -281,7 +288,14 @@ where
         async move {
             let start_pos = *position.read().await;
             match storage
-                .get_object_range(&bucket, &key, session_context.credentials(), start_pos, count as u64)
+                .get_object_range(
+                    &bucket,
+                    &key,
+                    &session_context.principal.user_identity.credentials.access_key,
+                    &session_context.principal.user_identity.credentials.secret_key,
+                    start_pos,
+                    count as u64,
+                )
                 .await
             {
                 Ok(output) => {
@@ -393,7 +407,14 @@ where
                 .build()
                 .map_err(|_| FsError::GeneralFailure)?;
 
-            match storage.put_object(put_input, session_context.credentials()).await {
+            match storage
+                .put_object(
+                    put_input,
+                    &session_context.principal.user_identity.credentials.access_key,
+                    &session_context.principal.user_identity.credentials.secret_key,
+                )
+                .await
+            {
                 Ok(_) => {
                     debug!(
                         event = EVENT_WEBDAV_OBJECT_WRITE_STATE,
@@ -501,8 +522,11 @@ where
         self
     }
 
-    fn credentials(&self) -> &Credentials {
-        self.session_context.credentials()
+    fn credentials(&self) -> (&str, &str) {
+        (
+            &self.session_context.principal.user_identity.credentials.access_key,
+            &self.session_context.principal.user_identity.credentials.secret_key,
+        )
     }
 
     fn is_missing_head_object_error(error: &str) -> bool {
@@ -514,7 +538,7 @@ where
     }
 
     async fn prefix_has_entries(&self, bucket: &str, prefix: &str) -> FsResult<bool> {
-        let credentials = self.credentials();
+        let (access_key, secret_key) = self.credentials();
         let list_input = ListObjectsV2Input::builder()
             .bucket(bucket.to_string())
             .prefix(Some(prefix.to_string()))
@@ -522,28 +546,32 @@ where
             .build()
             .map_err(|_| FsError::GeneralFailure)?;
 
-        let output = self.storage.list_objects_v2(list_input, credentials).await.map_err(|e| {
-            error!(
-                event = EVENT_WEBDAV_LIST_FAILED,
-                component = LOG_COMPONENT_PROTOCOLS,
-                subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
-                bucket = %bucket,
-                prefix = %prefix,
-                error = %e,
-                "webdav list failed"
-            );
-            FsError::GeneralFailure
-        })?;
+        let output = self
+            .storage
+            .list_objects_v2(list_input, access_key, secret_key)
+            .await
+            .map_err(|e| {
+                error!(
+                    event = EVENT_WEBDAV_LIST_FAILED,
+                    component = LOG_COMPONENT_PROTOCOLS,
+                    subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
+                    bucket = %bucket,
+                    prefix = %prefix,
+                    error = %e,
+                    "webdav list failed"
+                );
+                FsError::GeneralFailure
+            })?;
 
         Ok(output.contents.map(|c| !c.is_empty()).unwrap_or(false)
             || output.common_prefixes.map(|c| !c.is_empty()).unwrap_or(false))
     }
 
     async fn copy_object_streaming(&self, src_bucket: &str, src_key: &str, dst_bucket: &str, dst_key: &str) -> FsResult<()> {
-        let credentials = self.credentials();
+        let (access_key, secret_key) = self.credentials();
         let get_output = self
             .storage
-            .get_object(src_bucket, src_key, credentials, None)
+            .get_object(src_bucket, src_key, access_key, secret_key, None)
             .await
             .map_err(|e| {
                 error!(
@@ -597,21 +625,24 @@ where
 
         let put_input = put_builder.build().map_err(|_| FsError::GeneralFailure)?;
 
-        self.storage.put_object(put_input, credentials).await.map_err(|e| {
-            error!(
-                event = EVENT_WEBDAV_COPY_FAILED,
-                component = LOG_COMPONENT_PROTOCOLS,
-                subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
-                state = "destination_write_failed",
-                src_bucket = %src_bucket,
-                src_object = %src_key,
-                dst_bucket = %dst_bucket,
-                dst_object = %dst_key,
-                error = %e,
-                "webdav copy failed"
-            );
-            FsError::GeneralFailure
-        })?;
+        self.storage
+            .put_object(put_input, access_key, secret_key)
+            .await
+            .map_err(|e| {
+                error!(
+                    event = EVENT_WEBDAV_COPY_FAILED,
+                    component = LOG_COMPONENT_PROTOCOLS,
+                    subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
+                    state = "destination_write_failed",
+                    src_bucket = %src_bucket,
+                    src_object = %src_key,
+                    dst_bucket = %dst_bucket,
+                    dst_object = %dst_key,
+                    error = %e,
+                    "webdav copy failed"
+                );
+                FsError::GeneralFailure
+            })?;
 
         Ok(())
     }
@@ -622,7 +653,7 @@ where
         dst_bucket: &str,
         rename_pairs: &[(String, String)],
     ) -> FsResult<()> {
-        let credentials = self.credentials();
+        let (access_key, secret_key) = self.credentials();
 
         for (src_obj_key, dst_obj_key) in rename_pairs {
             self.copy_object_streaming(src_bucket, src_obj_key, dst_bucket, dst_obj_key)
@@ -631,7 +662,7 @@ where
 
         for (src_obj_key, _) in rename_pairs {
             self.storage
-                .delete_object(src_bucket, src_obj_key, credentials)
+                .delete_object(src_bucket, src_obj_key, access_key, secret_key)
                 .await
                 .map_err(|e| {
                     error!(
@@ -652,7 +683,7 @@ where
     }
 
     async fn probe_head_object(&self, bucket: &str, key: &str) -> FsResult<HeadObjectProbe> {
-        let credentials = self.credentials();
+        let (access_key, secret_key) = self.credentials();
 
         if authorize_operation(&self.session_context, &S3Action::HeadObject, bucket, Some(key))
             .await
@@ -661,7 +692,7 @@ where
             return Ok(HeadObjectProbe::Forbidden);
         }
 
-        match self.storage.head_object(bucket, key, credentials).await {
+        match self.storage.head_object(bucket, key, access_key, secret_key).await {
             Ok(output) => Ok(HeadObjectProbe::Found(Box::new(output))),
             Err(e) => {
                 let err_msg = e.to_string();
@@ -785,8 +816,8 @@ where
     async fn list_buckets(&self) -> FsResult<Vec<WebDavDirEntry>> {
         match authorize_operation(&self.session_context, &S3Action::ListBuckets, "", None).await {
             Ok(()) => {
-                let credentials = self.credentials();
-                return match self.storage.list_buckets(credentials).await {
+                let (access_key, secret_key) = self.credentials();
+                return match self.storage.list_buckets(access_key, secret_key).await {
                     Ok(output) => Ok(Self::bucket_entries(output)),
                     Err(error) => {
                         error!(
@@ -794,7 +825,7 @@ where
                             component = LOG_COMPONENT_PROTOCOLS,
                             subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
                             error = %error,
-                            access_key = %MaskedAccessKey(credentials.access_key.as_str()),
+                            access_key = %MaskedAccessKey(access_key),
                             "webdav bucket list failed"
                         );
                         Err(FsError::GeneralFailure)
@@ -877,7 +908,15 @@ where
             .build()
             .map_err(|_| FsError::GeneralFailure)?;
 
-        match self.storage.list_objects_v2(list_input, self.credentials()).await {
+        match self
+            .storage
+            .list_objects_v2(
+                list_input,
+                &self.session_context.principal.user_identity.credentials.access_key,
+                &self.session_context.principal.user_identity.credentials.secret_key,
+            )
+            .await
+        {
             Ok(output) => {
                 let mut entries = Vec::new();
 
@@ -1015,7 +1054,15 @@ where
 
             let list_input = list_input.build().map_err(|_| FsError::GeneralFailure)?;
 
-            if let Ok(output) = self.storage.list_objects_v2(list_input, self.credentials()).await {
+            if let Ok(output) = self
+                .storage
+                .list_objects_v2(
+                    list_input,
+                    &self.session_context.principal.user_identity.credentials.access_key,
+                    &self.session_context.principal.user_identity.credentials.secret_key,
+                )
+                .await
+            {
                 // Delete all objects in this page
                 if let Some(objects) = output.contents {
                     for obj in objects {
@@ -1024,7 +1071,15 @@ where
                                 .await
                                 .map_err(|_| FsError::Forbidden)?;
 
-                            let _ = self.storage.delete_object(bucket, &obj_key, self.credentials()).await;
+                            let _ = self
+                                .storage
+                                .delete_object(
+                                    bucket,
+                                    &obj_key,
+                                    &self.session_context.principal.user_identity.credentials.access_key,
+                                    &self.session_context.principal.user_identity.credentials.secret_key,
+                                )
+                                .await;
                         }
                     }
                 }
@@ -1040,7 +1095,15 @@ where
         }
 
         // Then delete the bucket
-        match self.storage.delete_bucket(bucket, self.credentials()).await {
+        match self
+            .storage
+            .delete_bucket(
+                bucket,
+                &self.session_context.principal.user_identity.credentials.access_key,
+                &self.session_context.principal.user_identity.credentials.secret_key,
+            )
+            .await
+        {
             Ok(_) => Ok(()),
             Err(e) if e.to_string().contains("NoSuchBucket") => Ok(()),
             Err(e) => {
@@ -1187,7 +1250,15 @@ where
                     .await
                     .map_err(|_| FsError::Forbidden)?;
 
-                match self.storage.head_bucket(&bucket, self.credentials()).await {
+                match self
+                    .storage
+                    .head_bucket(
+                        &bucket,
+                        &self.session_context.principal.user_identity.credentials.access_key,
+                        &self.session_context.principal.user_identity.credentials.secret_key,
+                    )
+                    .await
+                {
                     Ok(_) => Ok(Box::new(WebDavMetaData {
                         size: 0,
                         modified: SystemTime::now(),
@@ -1247,7 +1318,15 @@ where
                     .build()
                     .map_err(|_| FsError::GeneralFailure)?;
 
-                match self.storage.put_object(put_input, self.credentials()).await {
+                match self
+                    .storage
+                    .put_object(
+                        put_input,
+                        &self.session_context.principal.user_identity.credentials.access_key,
+                        &self.session_context.principal.user_identity.credentials.secret_key,
+                    )
+                    .await
+                {
                     Ok(_) => {
                         debug!(
                             event = EVENT_WEBDAV_DIRECTORY_STATE,
@@ -1281,7 +1360,15 @@ where
                 .await
                 .map_err(|_| FsError::Forbidden)?;
 
-            match self.storage.create_bucket(&bucket, self.credentials()).await {
+            match self
+                .storage
+                .create_bucket(
+                    &bucket,
+                    &self.session_context.principal.user_identity.credentials.access_key,
+                    &self.session_context.principal.user_identity.credentials.secret_key,
+                )
+                .await
+            {
                 Ok(_) => {
                     debug!(
                         event = EVENT_WEBDAV_DIRECTORY_STATE,
@@ -1351,7 +1438,15 @@ where
 
                     let list_input = list_input.build().map_err(|_| FsError::GeneralFailure)?;
 
-                    if let Ok(output) = self.storage.list_objects_v2(list_input, self.credentials()).await {
+                    if let Ok(output) = self
+                        .storage
+                        .list_objects_v2(
+                            list_input,
+                            &self.session_context.principal.user_identity.credentials.access_key,
+                            &self.session_context.principal.user_identity.credentials.secret_key,
+                        )
+                        .await
+                    {
                         if let Some(objects) = output.contents {
                             for obj in objects {
                                 if let Some(obj_key) = obj.key {
@@ -1359,7 +1454,15 @@ where
                                         .await
                                         .map_err(|_| FsError::Forbidden)?;
 
-                                    let _ = self.storage.delete_object(&bucket, &obj_key, self.credentials()).await;
+                                    let _ = self
+                                        .storage
+                                        .delete_object(
+                                            &bucket,
+                                            &obj_key,
+                                            &self.session_context.principal.user_identity.credentials.access_key,
+                                            &self.session_context.principal.user_identity.credentials.secret_key,
+                                        )
+                                        .await;
                                 }
                             }
                         }
@@ -1376,7 +1479,12 @@ where
                 // Also delete the directory marker itself
                 let _ = self
                     .storage
-                    .delete_object(&bucket, &prefix_with_slash, self.credentials())
+                    .delete_object(
+                        &bucket,
+                        &prefix_with_slash,
+                        &self.session_context.principal.user_identity.credentials.access_key,
+                        &self.session_context.principal.user_identity.credentials.secret_key,
+                    )
                     .await;
 
                 return Ok(());
@@ -1407,7 +1515,16 @@ where
                 .await
                 .map_err(|_| FsError::Forbidden)?;
 
-            match self.storage.delete_object(&bucket, &key, self.credentials()).await {
+            match self
+                .storage
+                .delete_object(
+                    &bucket,
+                    &key,
+                    &self.session_context.principal.user_identity.credentials.access_key,
+                    &self.session_context.principal.user_identity.credentials.secret_key,
+                )
+                .await
+            {
                 Ok(_) => {
                     debug!(
                         event = EVENT_WEBDAV_OBJECT_DELETE_STATE,
@@ -1449,7 +1566,7 @@ where
 
             let src_key = src_key.ok_or(FsError::Forbidden)?;
             let dst_key = dst_key.ok_or(FsError::Forbidden)?;
-            let credentials = self.credentials();
+            let (access_key, secret_key) = self.credentials();
             let resolved_src = self.resolve_path(&src_bucket, &src_key).await?;
             let (src_prefix, include_src_marker) = match resolved_src {
                 ResolvedPath::File(_) => {
@@ -1467,7 +1584,7 @@ where
                         .await?;
 
                     self.storage
-                        .delete_object(&src_bucket, &src_key, credentials)
+                        .delete_object(&src_bucket, &src_key, access_key, secret_key)
                         .await
                         .map_err(|e| {
                             error!(
@@ -1539,21 +1656,25 @@ where
                 }
 
                 let list_input = list_builder.build().map_err(|_| FsError::GeneralFailure)?;
-                let output = self.storage.list_objects_v2(list_input, credentials).await.map_err(|e| {
-                    error!(
-                        event = EVENT_WEBDAV_RENAME_STATE,
-                        component = LOG_COMPONENT_PROTOCOLS,
-                        subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
-                        state = "directory_list_failed",
-                        src_bucket = %src_bucket,
-                        src_prefix = %src_prefix,
-                        dst_bucket = %dst_bucket,
-                        dst_prefix = %dst_prefix,
-                        error = %e,
-                        "WebDAV rename directory listing failed"
-                    );
-                    FsError::GeneralFailure
-                })?;
+                let output = self
+                    .storage
+                    .list_objects_v2(list_input, access_key, secret_key)
+                    .await
+                    .map_err(|e| {
+                        error!(
+                            event = EVENT_WEBDAV_RENAME_STATE,
+                            component = LOG_COMPONENT_PROTOCOLS,
+                            subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
+                            state = "directory_list_failed",
+                            src_bucket = %src_bucket,
+                            src_prefix = %src_prefix,
+                            dst_bucket = %dst_bucket,
+                            dst_prefix = %dst_prefix,
+                            error = %e,
+                            "WebDAV rename directory listing failed"
+                        );
+                        FsError::GeneralFailure
+                    })?;
 
                 let mut page_pairs: Vec<(String, String)> = Vec::new();
                 if let Some(objects) = output.contents {
@@ -1664,7 +1785,8 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
             _start_pos: Option<u64>,
         ) -> Result<GetObjectOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
@@ -1674,14 +1796,20 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
             _start_pos: u64,
             _length: u64,
         ) -> Result<GetObjectOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
-        async fn put_object(&self, _input: PutObjectInput, _credentials: &Credentials) -> Result<PutObjectOutput, Self::Error> {
+        async fn put_object(
+            &self,
+            _input: PutObjectInput,
+            _access_key: &str,
+            _secret_key: &str,
+        ) -> Result<PutObjectOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
@@ -1689,7 +1817,8 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<DeleteObjectOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
@@ -1698,39 +1827,57 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<HeadObjectOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
-        async fn head_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<HeadBucketOutput, Self::Error> {
+        async fn head_bucket(
+            &self,
+            _bucket: &str,
+            _access_key: &str,
+            _secret_key: &str,
+        ) -> Result<HeadBucketOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
         async fn list_objects_v2(
             &self,
             _input: ListObjectsV2Input,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<ListObjectsV2Output, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
-        async fn list_buckets(&self, _credentials: &Credentials) -> Result<ListBucketsOutput, Self::Error> {
+        async fn list_buckets(&self, _access_key: &str, _secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
-        async fn create_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<CreateBucketOutput, Self::Error> {
+        async fn create_bucket(
+            &self,
+            _bucket: &str,
+            _access_key: &str,
+            _secret_key: &str,
+        ) -> Result<CreateBucketOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
-        async fn delete_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<DeleteBucketOutput, Self::Error> {
+        async fn delete_bucket(
+            &self,
+            _bucket: &str,
+            _access_key: &str,
+            _secret_key: &str,
+        ) -> Result<DeleteBucketOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
         async fn copy_object(
             &self,
             _input: CopyObjectInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<CopyObjectOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
@@ -1738,7 +1885,8 @@ mod tests {
         async fn create_multipart_upload(
             &self,
             _input: CreateMultipartUploadInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<CreateMultipartUploadOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
@@ -1746,7 +1894,8 @@ mod tests {
         async fn upload_part(
             &self,
             _input: UploadPartInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<UploadPartOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
@@ -1754,7 +1903,8 @@ mod tests {
         async fn complete_multipart_upload(
             &self,
             _input: CompleteMultipartUploadInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<CompleteMultipartUploadOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
@@ -1762,7 +1912,8 @@ mod tests {
         async fn abort_multipart_upload(
             &self,
             _input: AbortMultipartUploadInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<AbortMultipartUploadOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
@@ -1770,7 +1921,8 @@ mod tests {
         async fn upload_part_copy(
             &self,
             _input: UploadPartCopyInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<UploadPartCopyOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
@@ -1947,7 +2099,8 @@ mod tests {
             &self,
             bucket: &str,
             key: &str,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
             _start_pos: Option<u64>,
         ) -> Result<GetObjectOutput, Self::Error> {
             let data = self
@@ -1974,7 +2127,8 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
             _start_pos: u64,
             _length: u64,
         ) -> Result<GetObjectOutput, Self::Error> {
@@ -1984,7 +2138,8 @@ mod tests {
         async fn put_object(
             &self,
             mut input: PutObjectInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<PutObjectOutput, Self::Error> {
             let bucket = input.bucket.clone();
             let key = input.key.clone();
@@ -2008,7 +2163,8 @@ mod tests {
             &self,
             bucket: &str,
             key: &str,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<DeleteObjectOutput, Self::Error> {
             let mut state = self.state.lock().expect("recording storage lock poisoned");
             state.delete_keys.push(key.to_string());
@@ -2023,19 +2179,26 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<HeadObjectOutput, Self::Error> {
             unreachable!("head_object is not used in rename regression tests")
         }
 
-        async fn head_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<HeadBucketOutput, Self::Error> {
+        async fn head_bucket(
+            &self,
+            _bucket: &str,
+            _access_key: &str,
+            _secret_key: &str,
+        ) -> Result<HeadBucketOutput, Self::Error> {
             unreachable!("head_bucket is not used in rename regression tests")
         }
 
         async fn list_objects_v2(
             &self,
             input: ListObjectsV2Input,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<ListObjectsV2Output, Self::Error> {
             let prefix = input.prefix.unwrap_or_default();
             let mut keys: Vec<String> = self
@@ -2063,15 +2226,25 @@ mod tests {
             })
         }
 
-        async fn list_buckets(&self, _credentials: &Credentials) -> Result<ListBucketsOutput, Self::Error> {
+        async fn list_buckets(&self, _access_key: &str, _secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
             unreachable!("list_buckets is not used in rename regression tests")
         }
 
-        async fn create_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<CreateBucketOutput, Self::Error> {
+        async fn create_bucket(
+            &self,
+            _bucket: &str,
+            _access_key: &str,
+            _secret_key: &str,
+        ) -> Result<CreateBucketOutput, Self::Error> {
             unreachable!("create_bucket is not used in rename regression tests")
         }
 
-        async fn delete_bucket(&self, bucket: &str, _credentials: &Credentials) -> Result<DeleteBucketOutput, Self::Error> {
+        async fn delete_bucket(
+            &self,
+            bucket: &str,
+            _access_key: &str,
+            _secret_key: &str,
+        ) -> Result<DeleteBucketOutput, Self::Error> {
             self.state
                 .lock()
                 .expect("recording storage lock poisoned")
@@ -2083,7 +2256,8 @@ mod tests {
         async fn copy_object(
             &self,
             _input: CopyObjectInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<CopyObjectOutput, Self::Error> {
             unreachable!("copy_object is not used in rename regression tests")
         }
@@ -2091,7 +2265,8 @@ mod tests {
         async fn create_multipart_upload(
             &self,
             _input: CreateMultipartUploadInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<CreateMultipartUploadOutput, Self::Error> {
             unreachable!("create_multipart_upload is not used in rename regression tests")
         }
@@ -2099,7 +2274,8 @@ mod tests {
         async fn upload_part(
             &self,
             _input: UploadPartInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<UploadPartOutput, Self::Error> {
             unreachable!("upload_part is not used in rename regression tests")
         }
@@ -2107,7 +2283,8 @@ mod tests {
         async fn complete_multipart_upload(
             &self,
             _input: CompleteMultipartUploadInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<CompleteMultipartUploadOutput, Self::Error> {
             unreachable!("complete_multipart_upload is not used in rename regression tests")
         }
@@ -2115,7 +2292,8 @@ mod tests {
         async fn abort_multipart_upload(
             &self,
             _input: AbortMultipartUploadInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<AbortMultipartUploadOutput, Self::Error> {
             unreachable!("abort_multipart_upload is not used in rename regression tests")
         }
@@ -2123,7 +2301,8 @@ mod tests {
         async fn upload_part_copy(
             &self,
             _input: UploadPartCopyInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<UploadPartCopyOutput, Self::Error> {
             unreachable!("upload_part_copy is not used in rename regression tests")
         }

crates/protocols/src/webdav/server.rs

@@ -687,7 +687,6 @@ mod tests {
     use futures_util::stream;
     use http_body_util::StreamBody;
     use hyper::body::Frame;
-    use rustfs_credentials::Credentials;
     use s3s::dto::*;
     use std::fmt::{Debug, Formatter};
     use std::net::{Ipv4Addr, SocketAddr};
@@ -716,7 +715,8 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
             _start_pos: Option<u64>,
         ) -> Result<GetObjectOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
@@ -726,14 +726,20 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
             _start_pos: u64,
             _length: u64,
         ) -> Result<GetObjectOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
-        async fn put_object(&self, _input: PutObjectInput, _credentials: &Credentials) -> Result<PutObjectOutput, Self::Error> {
+        async fn put_object(
+            &self,
+            _input: PutObjectInput,
+            _access_key: &str,
+            _secret_key: &str,
+        ) -> Result<PutObjectOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
@@ -741,7 +747,8 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<DeleteObjectOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
@@ -750,39 +757,57 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<HeadObjectOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
-        async fn head_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<HeadBucketOutput, Self::Error> {
+        async fn head_bucket(
+            &self,
+            _bucket: &str,
+            _access_key: &str,
+            _secret_key: &str,
+        ) -> Result<HeadBucketOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
         async fn list_objects_v2(
             &self,
             _input: ListObjectsV2Input,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<ListObjectsV2Output, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
-        async fn list_buckets(&self, _credentials: &Credentials) -> Result<ListBucketsOutput, Self::Error> {
+        async fn list_buckets(&self, _access_key: &str, _secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
-        async fn create_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<CreateBucketOutput, Self::Error> {
+        async fn create_bucket(
+            &self,
+            _bucket: &str,
+            _access_key: &str,
+            _secret_key: &str,
+        ) -> Result<CreateBucketOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
-        async fn delete_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<DeleteBucketOutput, Self::Error> {
+        async fn delete_bucket(
+            &self,
+            _bucket: &str,
+            _access_key: &str,
+            _secret_key: &str,
+        ) -> Result<DeleteBucketOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
         async fn copy_object(
             &self,
             _input: CopyObjectInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<CopyObjectOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
@@ -790,7 +815,8 @@ mod tests {
         async fn create_multipart_upload(
             &self,
             _input: CreateMultipartUploadInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<CreateMultipartUploadOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
@@ -798,7 +824,8 @@ mod tests {
         async fn upload_part(
             &self,
             _input: UploadPartInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<UploadPartOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
@@ -806,7 +833,8 @@ mod tests {
         async fn complete_multipart_upload(
             &self,
             _input: CompleteMultipartUploadInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<CompleteMultipartUploadOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
@@ -814,7 +842,8 @@ mod tests {
         async fn abort_multipart_upload(
             &self,
             _input: AbortMultipartUploadInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<AbortMultipartUploadOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
@@ -822,7 +851,8 @@ mod tests {
         async fn upload_part_copy(
             &self,
             _input: UploadPartCopyInput,
-            _credentials: &Credentials,
+            _access_key: &str,
+            _secret_key: &str,
         ) -> Result<UploadPartCopyOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }

crates/rio/Cargo.toml

@@ -90,15 +90,8 @@ s3s = { workspace = true, features = ["minio"] }
 hex-simd.workspace = true
 
 [dev-dependencies]
-tokio = { workspace = true, features = ["test-util"] }
 tokio-test = { workspace = true }
-criterion = { workspace = true, features = ["html_reports"] }
-proptest = { workspace = true }
 axum = { workspace = true }
 hyper = { workspace = true, features = ["http2", "server"] }
 hyper-util = { workspace = true, features = ["tokio"] }
 http-body-util = { workspace = true }
-
-[[bench]]
-name = "tee_reader"
-harness = false

crates/rio/benches/tee_reader.rs

@@ -1,103 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Throughput of `tee_reader` versus reading the same source directly:
-//! 64 MiB of data served in 1 MiB chunks, consumed with 1 MiB reads.
-
-use bytes::Bytes;
-use criterion::{Criterion, Throughput, criterion_group, criterion_main};
-use rustfs_rio::tee_reader;
-use std::hint::black_box;
-use std::io;
-use std::pin::Pin;
-use std::task::{Context, Poll};
-use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf};
-
-const CHUNK_BYTES: usize = 1024 * 1024;
-const TOTAL_BYTES: usize = 64 * 1024 * 1024;
-const TEE_BUFFER_BYTES: usize = 4 * CHUNK_BYTES;
-
-/// In-memory source that serves at most `CHUNK_BYTES` per poll.
-struct ChunkedSource {
-    data: Bytes,
-    pos: usize,
-}
-
-impl AsyncRead for ChunkedSource {
-    fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
-        let remaining = self.data.len() - self.pos;
-        let n = CHUNK_BYTES.min(remaining).min(buf.remaining());
-        buf.put_slice(&self.data[self.pos..self.pos + n]);
-        self.pos += n;
-        Poll::Ready(Ok(()))
-    }
-}
-
-async fn consume<R: AsyncRead + Unpin>(mut reader: R) -> usize {
-    let mut buf = vec![0u8; CHUNK_BYTES];
-    let mut total = 0;
-    loop {
-        let n = reader.read(&mut buf).await.expect("read");
-        if n == 0 {
-            return total;
-        }
-        total += n;
-    }
-}
-
-fn bench_tee_reader(c: &mut Criterion) {
-    let runtime = tokio::runtime::Builder::new_multi_thread()
-        .worker_threads(2)
-        .enable_all()
-        .build()
-        .expect("build tokio runtime for tee_reader benchmark");
-    let data = Bytes::from(vec![0xA5u8; TOTAL_BYTES]);
-
-    let mut group = c.benchmark_group("tee_reader_64mib_1mib_chunks");
-    group.throughput(Throughput::Bytes(TOTAL_BYTES as u64));
-    group.sample_size(10);
-
-    group.bench_function("direct_read", |b| {
-        b.iter(|| {
-            let source = ChunkedSource {
-                data: data.clone(),
-                pos: 0,
-            };
-            let total = runtime.block_on(consume(source));
-            black_box(total)
-        })
-    });
-
-    group.bench_function("tee_primary_plus_secondary", |b| {
-        b.iter(|| {
-            let source = ChunkedSource {
-                data: data.clone(),
-                pos: 0,
-            };
-            let (primary, secondary) = tee_reader(source, TEE_BUFFER_BYTES);
-            let totals = runtime.block_on(async {
-                let secondary_task = tokio::spawn(consume(secondary));
-                let primary_total = consume(primary).await;
-                let secondary_total = secondary_task.await.expect("secondary task");
-                (primary_total, secondary_total)
-            });
-            black_box(totals)
-        })
-    });
-
-    group.finish();
-}
-
-criterion_group!(benches, bench_tee_reader);
-criterion_main!(benches);

crates/rio/src/lib.rs

@@ -118,12 +118,6 @@ pub use hardlimit_reader::HardLimitReader;
 
 mod hash_reader;
 pub use hash_reader::*;
-
-mod tee_reader;
-pub use tee_reader::{
-    DEFAULT_TEE_MAX_DRAIN_BYTES, TeeDrainLimitExceeded, TeeOptions, TeePrimary, TeeSecondary, TeeStream, tee_reader,
-    tee_reader_with_options,
-};
 mod checksum;
 pub use checksum::*;
 

crates/scanner/src/scanner/cycle_state.rs

@@ -2027,7 +2027,7 @@ fn legacy_incomplete_usage_fence(data: &[u8], usage: &DataUsageInfo) -> Option<L
     legacy_empty_usage_fence(data, usage).or_else(|| legacy_non_empty_usage_fence(data, usage))
 }
 
-// RUSTFS_COMPAT_TODO(backlog-2122): accept rc.1-rc.3 usage floors that were fenced before a scanner cycle completed. Remove after those releases are no longer supported direct-upgrade sources.
+// RUSTFS_COMPAT_TODO(backlog-2181): accept rc.1-rc.3 usage floors that were fenced before a scanner cycle completed. Remove after those releases are no longer supported direct-upgrade sources.
 fn legacy_non_empty_usage_fence(data: &[u8], usage: &DataUsageInfo) -> Option<LegacyIncompleteUsageFence> {
     if usage.last_update.is_none()
         || usage.scanner_cycle.is_some()

crates/utils/src/http/metadata_compat.rs

@@ -88,21 +88,6 @@ pub const SUFFIX_TIER_SKIP_FV_ID: &str = "tier-skip-fvid";
 /// Per-target delete-marker version ids are stored one key per target ARN.
 pub const SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX: &str = "replication-delete-marker-version-";
 
-// On-demand migration provenance. Written by the migration write-back onto
-// every pulled object so operators and later tooling can tell a migrated
-// object from a client write and audit where it came from. Internal keys, so
-// the existing internal-key filter keeps them out of client-visible metadata.
-/// Source of a pulled object as `<provider>:<bucket>`.
-pub const SUFFIX_ODM_SOURCE: &str = "odm-source";
-/// ETag the source reported for the pulled object, verbatim.
-pub const SUFFIX_ODM_SOURCE_ETAG: &str = "odm-source-etag";
-/// Source `Last-Modified` of the pulled object, RFC 3339.
-pub const SUFFIX_ODM_SOURCE_LAST_MODIFIED: &str = "odm-source-last-modified";
-/// Source version id of the pulled object; empty for an unversioned source.
-pub const SUFFIX_ODM_SOURCE_VERSION_ID: &str = "odm-source-version-id";
-/// When the object was pulled from the source, RFC 3339.
-pub const SUFFIX_ODM_PULLED_AT: &str = "odm-pulled-at";
-
 /// Case-insensitive (ASCII) check that `s` begins with `prefix`. Equivalent to
 /// `s.to_lowercase().starts_with(prefix)` when `prefix` is ASCII (as both internal prefixes are),
 /// but without allocating.
@@ -606,80 +591,6 @@ mod tests {
         assert!(!contains_key_bytes(&meta_sys, &long_suffix));
     }
 
-    const ODM_PROVENANCE_SUFFIXES: [&str; 5] = [
-        SUFFIX_ODM_SOURCE,
-        SUFFIX_ODM_SOURCE_ETAG,
-        SUFFIX_ODM_SOURCE_LAST_MODIFIED,
-        SUFFIX_ODM_SOURCE_VERSION_ID,
-        SUFFIX_ODM_PULLED_AT,
-    ];
-
-    #[test]
-    fn odm_provenance_suffixes_are_reserved_internal_keys_under_both_prefixes() {
-        for suffix in ODM_PROVENANCE_SUFFIXES {
-            for prefix in [RUSTFS_INTERNAL_PREFIX, MINIO_INTERNAL_PREFIX] {
-                let key = format!("{prefix}{suffix}");
-                assert!(is_internal_key(&key), "{key} must be an internal key");
-                assert!(has_internal_suffix(&key, suffix), "{key} must match its own suffix");
-                assert!(has_internal_suffix(&key.to_uppercase(), suffix), "{key} must match case-insensitively");
-                assert_eq!(strip_internal_prefix(&key).as_deref(), Some(suffix));
-            }
-            assert!(
-                !is_internal_key(&format!("x-amz-meta-{suffix}")),
-                "{suffix} must not leak as user metadata"
-            );
-            assert!(suffix.starts_with("odm-"), "{suffix} must stay in the odm- namespace");
-            assert!(
-                suffix.bytes().all(|b| b.is_ascii_lowercase() || b == b'-'),
-                "{suffix} must be lowercase-hyphenated"
-            );
-        }
-        let distinct: std::collections::HashSet<&str> = ODM_PROVENANCE_SUFFIXES.into_iter().collect();
-        assert_eq!(distinct.len(), ODM_PROVENANCE_SUFFIXES.len(), "provenance suffixes must be distinct");
-    }
-
-    #[test]
-    fn odm_provenance_values_round_trip_under_both_prefixes() {
-        let mut metadata = HashMap::new();
-        insert_str(&mut metadata, SUFFIX_ODM_SOURCE, "s3:legacy-bucket".to_string());
-        insert_str(&mut metadata, SUFFIX_ODM_SOURCE_ETAG, "0123456789abcdef0123456789abcdef-3".to_string());
-        insert_str(&mut metadata, SUFFIX_ODM_SOURCE_LAST_MODIFIED, "2026-01-02T03:04:05Z".to_string());
-        insert_str(&mut metadata, SUFFIX_ODM_SOURCE_VERSION_ID, String::new());
-        insert_str(&mut metadata, SUFFIX_ODM_PULLED_AT, "2026-09-02T00:00:00Z".to_string());
-
-        assert_eq!(
-            metadata.len(),
-            2 * ODM_PROVENANCE_SUFFIXES.len(),
-            "every marker is written under both prefixes"
-        );
-        for suffix in ODM_PROVENANCE_SUFFIXES {
-            assert!(metadata.contains_key(&internal_key_rustfs(suffix)), "missing RustFS key for {suffix}");
-            assert!(
-                metadata.contains_key(&format!("{MINIO_INTERNAL_PREFIX}{suffix}")),
-                "missing MinIO key for {suffix}"
-            );
-            assert!(contains_key_str(&metadata, suffix));
-        }
-        assert_eq!(get_str(&metadata, SUFFIX_ODM_SOURCE).as_deref(), Some("s3:legacy-bucket"));
-        assert_eq!(get_str(&metadata, SUFFIX_ODM_SOURCE_VERSION_ID).as_deref(), Some(""));
-        assert_eq!(
-            get_consistent_str(&metadata, SUFFIX_ODM_SOURCE_ETAG),
-            Some("0123456789abcdef0123456789abcdef-3")
-        );
-
-        // A MinIO-only reader must still find the marker.
-        let minio_only: HashMap<String, String> = metadata
-            .iter()
-            .filter(|(key, _)| starts_with_ignore_ascii_case(key, MINIO_INTERNAL_PREFIX))
-            .map(|(key, value)| (key.clone(), value.clone()))
-            .collect();
-        assert_eq!(get_str(&minio_only, SUFFIX_ODM_PULLED_AT).as_deref(), Some("2026-09-02T00:00:00Z"));
-
-        remove_str(&mut metadata, SUFFIX_ODM_SOURCE);
-        assert!(!contains_key_str(&metadata, SUFFIX_ODM_SOURCE));
-        assert!(contains_key_str(&metadata, SUFFIX_ODM_SOURCE_ETAG));
-    }
-
     #[test]
     fn target_delete_marker_versions_preserve_arn_case_and_report_conflicts() {
         let arn = "arn:rustfs:replication::Target:Bucket";

docs/architecture/compat-cleanup-register.md

@@ -13,7 +13,7 @@
 
 - `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on per-entry and cumulative GNU long-name, GNU long-link, and PAX extension limits; physical-entry, GNU sparse-map, and sparse-continuation limits; cancellation-safe sparse parsing; and fused entry streams after parser errors. The released tokio-tar API does not provide this complete boundary. Keep the reviewed fork pin until astral-sh/tokio-tar#118 is merged and one published tokio-tar release contains every listed capability with the Snowball regression fixtures passing against that release.
 - `backlog-2102` rc.2/rc.3 empty scanner usage floor recovery: old DeleteBucket cleanup could synthesize an empty incomplete v2 usage primary/backup before leadership added an epoch, while newer scanners require a durable authoritative baseline identity. New scanners recognize only that exact serialized empty-fence shape, preserve its epoch through a CAS-protected recovery marker, and rebuild namespace coverage without treating zero usage as authoritative. Remove this recovery path and marker after rc.2 and rc.3 are no longer supported direct-upgrade sources.
-- `backlog-2122` rc.1-rc.3 non-empty scanner usage floor recovery: leadership fencing in those releases can stamp scanner_epoch onto a real bucket-usage snapshot before any scanner cycle completed, leaving a non-empty floor with no scanner_cycle and no authoritative baseline identity. New scanners recognize only this consistent incomplete fenced shape, preserve the epoch through the CAS-protected recovery marker, and rebuild namespace coverage without treating the old usage data as authoritative. Remove this recovery path after rc.1, rc.2, and rc.3 are no longer supported direct-upgrade sources.
+- `backlog-2181` rc.1-rc.3 non-empty scanner usage floor recovery: leadership fencing in those releases can stamp scanner_epoch onto a real bucket-usage snapshot before any scanner cycle completed, leaving a non-empty floor with no scanner_cycle and no authoritative baseline identity. New scanners recognize only this consistent incomplete fenced shape, preserve the epoch through the CAS-protected recovery marker, and rebuild namespace coverage without treating the old usage data as authoritative. Remove this recovery path after rc.1, rc.2, and rc.3 are no longer supported direct-upgrade sources.
 - `s3gate-metadata-xml` persisted bucket XML migration: mixed-version site-replication peers, retained `.metadata.bin` objects, and backup archives can all carry XML written by the s3s codec, so the gateway migration must keep the legacy codec available until every stored form has crossed a verified rewrite boundary. Remove the legacy s3s parser and serializer only after the minimum supported direct-upgrade release reads and writes every persisted XML configuration family through the gateway codec, every supported mixed-version site-replication topology has completed its writer upgrade, and migration tooling has verified or rewritten every retained bucket metadata object and restorable backup archive.
 - `rustfs-6339` legacy bucket policy ID casing: earlier RustFS releases persisted the top-level policy identifier as "ID", while current writes use the S3-compatible "Id" spelling. Readers accept both spellings so retained bucket metadata remains usable after upgrade. Remove the legacy alias after migration tooling has rewritten every retained bucket policy using "ID".
 - `table-publication-fence-v1` table publication fencing: nodes that predate table and table-bucket publication fences can mutate live files while a new node is publishing a catalog pointer. New nodes retain exact object guards until the operator confirms that every serving node uses the new fences. Fleet confirmation also requires non-overlapping active warehouse prefixes and lifecycle workers that exclude table buckets. Remove the exact live-file fallback and the fleet-confirmation gate after the minimum supported RustFS release acquires table fences for registered-table mutations and table-bucket fences for unresolved-prefix mutations.

docs/operations/outbound-connection-policy.md

@@ -18,7 +18,7 @@ RustFS validates every operator-configured outbound destination to close a serve
 | Target configuration validation (startup and admin API) | Full policy | `crates/targets/src/config/common.rs` `validate_outbound_http_url`; `rustfs/src/admin/handlers/target_descriptor.rs` |
 | OIDC discovery, JWKS, and token requests | Full policy | A blocked provider logs `OIDC provider discovery blocked by outbound policy` naming the origin to allowlist (`crates/iam/src/oidc.rs`) |
 | Object Lambda targets | Full policy | `rustfs/src/admin/router.rs` `outbound_policy` |
-| Bucket replication targets | Literal check, relaxed | Private addresses are always allowed; loopback only with `RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET=true` (`crates/ecstore/src/bucket/remote_s3_client.rs` `validate_remote_endpoint`, shared with on-demand migration sources) |
+| Bucket replication targets | Literal check, relaxed | Private addresses are always allowed; loopback only with `RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET=true` (`crates/ecstore/src/bucket/bucket_target_sys.rs` `validate_replication_target_endpoint`) |
 | Site replication peers | Literal check | `rustfs/src/site_replication/mod.rs` |
 | Tiering warm backends (S3, MinIO, RustFS, Azure, GCS, Aliyun, Tencent, Huawei, R2) | Literal check | `crates/ecstore/src/services/tier/warm_backend.rs` `validate_endpoint`; the RustFS provider adds a debug-only, env-gated loopback exception for e2e tests |
 | Keystone `auth_url` | Literal check | `crates/keystone/src/config.rs` |

docs/operations/snowball-auto-extract.md

@@ -0,0 +1,36 @@
+# Snowball Auto-Extract Limits
+
+RustFS accepts MinIO-compatible Snowball auto-extract uploads. Archive
+members are streamed into objects while RustFS enforces entry-count, path,
+PAX metadata, per-object, cumulative unpacked-size, and decoded-stream
+limits.
+
+## Size limits
+
+The defaults remain compatible with the existing safety policy:
+
+| Environment variable | Default | Hard maximum | Meaning |
+| --- | ---: | ---: | --- |
+| `RUSTFS_SNOWBALL_MAX_ENTRY_BYTES` | 1 GiB | 1 TiB | Maximum unpacked size of one archive member |
+| `RUSTFS_SNOWBALL_MAX_UNPACKED_BYTES` | 10 GiB | 10 TiB | Maximum cumulative unpacked object bytes in one request |
+
+Invalid values use the default. Zero is treated as one byte, values above the
+hard maximum are clamped, and the per-entry limit is never allowed to exceed
+the cumulative request limit. RustFS derives a separate decoded-stream limit
+with bounded room for tar headers and PAX metadata; it cannot be disabled.
+
+Increasing either limit raises the maximum work performed by one admitted
+request. Snowball archive decoder admission remains globally bounded, so a
+larger archive cannot create an unbounded number of concurrent decoders.
+Restart RustFS after changing these environment variables.
+
+## Small-member concurrency
+
+For requests that set Snowball ignore-errors and do not use bucket quota
+accounting, RustFS stages members up to 128 KiB and commits at most 16 at a
+time. Requests that must stop on the first write error and quota-enabled
+requests remain serial so their observable error and accounting behavior does
+not change.
+
+Set `RUSTFS_SNOWBALL_EXTRACT_MAX_INFLIGHT=1` to restore fully serial member
+commits. Values are clamped to the range 1 through 16.

rustfs/src/app/object/extract.rs

@@ -373,10 +373,10 @@ const EXTRACT_MAX_EFFECTIVE_PAX_HEADER_BYTES: usize = 8 * 1024;
 const EXTRACT_MAX_EFFECTIVE_PAX_USER_METADATA_BYTES: usize = 2 * 1024;
 const EXTRACT_MAX_EFFECTIVE_PAX_FIELDS: usize = 4096;
 const EXTRACT_MAX_EXPANDED_PAX_METADATA_BYTES: u64 = 128 * 1024 * 1024;
-const EXTRACT_SMALL_MEMBER_MAX_BYTES: usize = 64 * 1024;
-const EXTRACT_DEFAULT_MAX_INFLIGHT: usize = 1;
+const EXTRACT_SMALL_MEMBER_MAX_BYTES: usize = 128 * 1024;
+const EXTRACT_DEFAULT_MAX_INFLIGHT: usize = 16;
 const EXTRACT_BATCH_MAX_MEMBERS: usize = 16;
-const EXTRACT_BATCH_MAX_STAGING_BYTES: usize = 2 * 1024 * 1024;
+const EXTRACT_BATCH_MAX_STAGING_BYTES: usize = 3 * 1024 * 1024;
 const EXTRACT_MEMBER_CONTEXT_OVERHEAD_BYTES: usize = 512;
 const EXTRACT_METADATA_ENTRY_OVERHEAD_BYTES: usize = 64;
 const ENV_RUSTFS_SNOWBALL_EXTRACT_MAX_INFLIGHT: &str = "RUSTFS_SNOWBALL_EXTRACT_MAX_INFLIGHT";
@@ -1864,7 +1864,34 @@ fn resolve_put_object_extract_options(headers: &HeaderMap) -> S3Result<PutObject
 }
 
 fn put_object_extract_limits() -> ArchiveLimits {
-    ArchiveLimits::default()
+    static LIMITS: OnceLock<ArchiveLimits> = OnceLock::new();
+    *LIMITS.get_or_init(|| {
+        normalize_put_object_extract_limits(
+            rustfs_utils::get_env_u64(
+                rustfs_config::ENV_SNOWBALL_MAX_ENTRY_BYTES,
+                rustfs_config::DEFAULT_SNOWBALL_MAX_ENTRY_BYTES,
+            ),
+            rustfs_utils::get_env_u64(
+                rustfs_config::ENV_SNOWBALL_MAX_UNPACKED_BYTES,
+                rustfs_config::DEFAULT_SNOWBALL_MAX_UNPACKED_BYTES,
+            ),
+        )
+    })
+}
+
+fn normalize_put_object_extract_limits(max_entry_bytes: u64, max_unpacked_bytes: u64) -> ArchiveLimits {
+    let defaults = ArchiveLimits::default();
+    let max_total_unpacked_size = max_unpacked_bytes.clamp(1, rustfs_config::MAX_SNOWBALL_UNPACKED_BYTES);
+    let max_entry_size = max_entry_bytes
+        .clamp(1, rustfs_config::MAX_SNOWBALL_ENTRY_BYTES)
+        .min(max_total_unpacked_size);
+
+    ArchiveLimits {
+        max_entry_size,
+        max_total_unpacked_size,
+        max_decoded_size: max_total_unpacked_size.saturating_add(max_entry_size),
+        ..defaults
+    }
 }
 
 fn build_put_object_extract_archive<R>(decoder: R, limits: ArchiveLimits) -> Archive<R>
@@ -2175,12 +2202,13 @@ impl DefaultObjectUsecase {
             .is_some_and(|result| result.uses_durable_reservations);
         // Without ignore-errors, the legacy contract stops before attempting a
         // later member after the first storage failure. Parallel commits cannot
-        // preserve that boundary, so concurrency requires both ignore-errors
-        // and an explicit max-inflight value above the serial default. Quota
-        // accounting can fail after storage commit, so quota-enabled imports
-        // also remain serial. An opted-in micro-batch is always drained; a
-        // fatal outcome stops later batches but cannot roll back peers that
-        // already committed in the current batch.
+        // preserve that boundary, so only ignore-errors requests use the
+        // configured micro-batch. Quota accounting can fail after storage
+        // commit, so quota-enabled imports also remain serial. Setting
+        // RUSTFS_SNOWBALL_EXTRACT_MAX_INFLIGHT=1 restores serial behavior. A
+        // micro-batch is always drained; a fatal outcome stops later batches
+        // but cannot roll back peers that already committed in the current
+        // batch.
         let max_inflight = select_put_object_extract_max_inflight(
             put_object_extract_max_inflight(),
             extract_options.ignore_errors,
@@ -2856,7 +2884,7 @@ mod tests {
 
     #[test]
     fn snowball_max_inflight_has_a_serial_compatibility_floor_and_bounded_ceiling() {
-        assert_eq!(EXTRACT_DEFAULT_MAX_INFLIGHT, 1);
+        assert_eq!(EXTRACT_DEFAULT_MAX_INFLIGHT, EXTRACT_BATCH_MAX_MEMBERS);
         assert_eq!(normalize_put_object_extract_max_inflight(0), 1);
         assert_eq!(normalize_put_object_extract_max_inflight(1), 1);
         assert_eq!(normalize_put_object_extract_max_inflight(usize::MAX), EXTRACT_BATCH_MAX_MEMBERS);
@@ -2877,6 +2905,35 @@ mod tests {
         );
     }
 
+    #[test]
+    fn snowball_archive_limits_preserve_defaults_and_clamp_operator_overrides() {
+        let defaults = ArchiveLimits::default();
+        assert_eq!(
+            normalize_put_object_extract_limits(
+                rustfs_config::DEFAULT_SNOWBALL_MAX_ENTRY_BYTES,
+                rustfs_config::DEFAULT_SNOWBALL_MAX_UNPACKED_BYTES,
+            ),
+            defaults
+        );
+
+        let minimum = normalize_put_object_extract_limits(0, 0);
+        assert_eq!(minimum.max_entry_size, 1);
+        assert_eq!(minimum.max_total_unpacked_size, 1);
+        assert_eq!(minimum.max_decoded_size, 2);
+
+        let bounded = normalize_put_object_extract_limits(u64::MAX, u64::MAX);
+        assert_eq!(bounded.max_entry_size, rustfs_config::MAX_SNOWBALL_ENTRY_BYTES);
+        assert_eq!(bounded.max_total_unpacked_size, rustfs_config::MAX_SNOWBALL_UNPACKED_BYTES);
+        assert_eq!(
+            bounded.max_decoded_size,
+            rustfs_config::MAX_SNOWBALL_UNPACKED_BYTES + rustfs_config::MAX_SNOWBALL_ENTRY_BYTES
+        );
+
+        let entry_is_bounded_by_the_request_total = normalize_put_object_extract_limits(1024, 512);
+        assert_eq!(entry_is_bounded_by_the_request_total.max_entry_size, 512);
+        assert_eq!(entry_is_bounded_by_the_request_total.max_total_unpacked_size, 512);
+    }
+
     #[test]
     fn snowball_batch_state_flushes_on_duplicates_limits_and_serial_barriers() {
         let mut state = ExtractBatchState::default();

rustfs/src/app/object/internal_put.rs

@@ -1,968 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Internal object write entry points for trusted in-process callers.
-//!
-//! A system write (on-demand migration write-back, future replays) must look
-//! like an ordinary client write: bucket default SSE, quota, versioning,
-//! Object Lock defaults, replication scheduling and creation events all apply.
-//! The single-object entry point runs the same [`DefaultObjectUsecase::put_object_core`]
-//! as the S3 PutObject handler; the multipart entry points mirror the S3
-//! multipart handlers' policy steps against the same storage contract.
-
-use super::*;
-
-use crate::app::object_data_cache::invalidate_object_data_cache_after_complete_multipart_success;
-use crate::app::storage_api::multipart_usecase::contract::multipart::{CompletePart, MultipartOperations as _};
-use crate::app::storage_api::object_usecase::compression::is_multipart_disk_compression_enabled;
-use crate::app::storage_api::object_usecase::io::WriteEncryption;
-use crate::app::storage_api::object_usecase::options::{
-    extract_metadata_from_mime, get_complete_multipart_upload_opts_with_replication_authorization,
-};
-use crate::app::storage_api::object_usecase::sse::{
-    EncryptionKeyKind, PrepareEncryptionRequest, mark_encrypted_multipart_metadata, sse_decryption, sse_prepare_encryption,
-};
-use crate::capacity::record_capacity_write;
-use crate::runtime_sources::NotifyInterface;
-use http::HeaderName;
-
-/// Inputs of an internal object write. Content and user metadata follow the
-/// S3 request shape so the shared write path treats them exactly like a
-/// client PUT: `content_headers` are the standard object headers
-/// (`Content-Type`, `Cache-Control`, `Content-Encoding`, `Content-Disposition`,
-/// `Content-Language`, `Expires`), `user_metadata` carries `x-amz-meta-*`
-/// entries with the prefix stripped, `tags` is the `x-amz-tagging` query
-/// string and `internal_metadata` holds `x-rustfs-internal-*` /
-/// `x-minio-internal-*` keys written verbatim.
-pub(crate) struct InternalPutContext {
-    pub(crate) bucket: String,
-    pub(crate) key: String,
-    /// Plaintext object length. The single-object path requires it, exactly
-    /// like S3 PutObject rejects an unknown `Content-Length`.
-    pub(crate) size: Option<u64>,
-    /// Lowercase hex MD5 the body must hash to; the write fails with
-    /// `BadDigest` otherwise and nothing is committed.
-    pub(crate) expected_md5_hex: Option<String>,
-    /// ETag to store instead of the computed one.
-    pub(crate) preserve_etag: Option<String>,
-    pub(crate) content_headers: HashMap<String, String>,
-    pub(crate) user_metadata: HashMap<String, String>,
-    pub(crate) tags: Option<String>,
-    pub(crate) internal_metadata: HashMap<String, String>,
-    /// Publish the `s3:ObjectCreated:*` event for the write.
-    pub(crate) emit_events: bool,
-    /// `userIdentity.principalId` of the creation event.
-    pub(crate) principal_id: &'static str,
-}
-
-const INTERNAL_PUT_METHOD_NAME: &str = "PUT";
-
-fn api_error_from_s3(err: S3Error) -> ApiError {
-    ApiError {
-        code: err.code().clone(),
-        message: err.message().unwrap_or_default().to_string(),
-        source: None,
-    }
-}
-
-fn not_initialized() -> ApiError {
-    ApiError {
-        code: S3ErrorCode::InternalError,
-        message: "Not init".to_string(),
-        source: None,
-    }
-}
-
-/// Build the header view of an internal write so header-driven policy
-/// (content-type detection, compressibility, standard metadata capture) runs
-/// unchanged.
-fn internal_put_headers(content_headers: &HashMap<String, String>) -> Result<HeaderMap, ApiError> {
-    let mut headers = HeaderMap::with_capacity(content_headers.len());
-    for (name, value) in content_headers {
-        let name = HeaderName::from_bytes(name.as_bytes())
-            .map_err(|err| ApiError::invalid_request(format!("invalid content header name {name:?}: {err}")))?;
-        let value = HeaderValue::from_str(value)
-            .map_err(|err| ApiError::invalid_request(format!("invalid content header value for {name}: {err}")))?;
-        headers.insert(name, value);
-    }
-    Ok(headers)
-}
-
-fn header_string(headers: &HeaderMap, name: &str) -> Option<String> {
-    headers.get(name).and_then(|value| value.to_str().ok()).map(ToOwned::to_owned)
-}
-
-fn internal_put_content_input(headers: &HeaderMap, tags: Option<String>) -> PutObjectContentInput {
-    PutObjectContentInput {
-        cache_control: header_string(headers, "cache-control"),
-        content_disposition: header_string(headers, "content-disposition"),
-        content_encoding: header_string(headers, "content-encoding"),
-        content_language: header_string(headers, "content-language"),
-        content_type: header_string(headers, "content-type"),
-        expires: header_string(headers, "expires"),
-        website_redirect_location: None,
-        tagging: tags,
-        storage_class: None,
-    }
-}
-
-async fn validate_internal_write_target(key: &str, bucket: &str, headers: &HeaderMap) -> Result<(), ApiError> {
-    validate_object_key(key, INTERNAL_PUT_METHOD_NAME).map_err(api_error_from_s3)?;
-    validate_table_catalog_object_mutation(bucket, key)
-        .await
-        .map_err(api_error_from_s3)?;
-    validate_archive_content_encoding(
-        key,
-        headers.get("content-type").and_then(|value| value.to_str().ok()),
-        headers.get("content-encoding").and_then(|value| value.to_str().ok()),
-    )
-    .map_err(api_error_from_s3)
-}
-
-fn internal_events_wanted() -> bool {
-    crate::module_switches::is_notify_module_enabled()
-        || rustfs_notify::notification_system().is_some_and(|system| system.has_live_listeners())
-}
-
-/// Creation event of an internal write, published on successful completion
-/// the way [`OperationHelper`] publishes it for an S3 request.
-pub(super) struct InternalPutObjectEvent {
-    builder: EventArgsBuilder,
-    notify: Arc<dyn NotifyInterface>,
-    request_context: request_context::RequestContext,
-}
-
-impl InternalPutObjectEvent {
-    /// `None` when neither the notify module nor a live listener wants events.
-    pub(super) fn new(
-        notify: Arc<dyn NotifyInterface>,
-        request_context: request_context::RequestContext,
-        event_name: EventName,
-        bucket: &str,
-        key: &str,
-        principal_id: &'static str,
-    ) -> Option<Self> {
-        if !internal_events_wanted() {
-            return None;
-        }
-        Some(Self {
-            builder: Self::builder(event_name, bucket, key, principal_id),
-            notify,
-            request_context,
-        })
-    }
-
-    fn builder(event_name: EventName, bucket: &str, key: &str, principal_id: &'static str) -> EventArgsBuilder {
-        // The object is a placeholder until `object()` supplies the committed
-        // ObjectInfo, matching the S3 helper.
-        let placeholder = ObjectInfo {
-            bucket: bucket.to_string(),
-            name: key.to_string(),
-            ..Default::default()
-        };
-        EventArgsBuilder::new(event_name, bucket.to_string(), convert_ecstore_object_info(placeholder))
-            .req_param("principalId", principal_id)
-    }
-
-    pub(super) fn object(self, obj_info: ObjectInfo) -> Self {
-        let Self {
-            builder,
-            notify,
-            request_context,
-        } = self;
-        Self {
-            builder: builder.object(convert_ecstore_object_info(obj_info)),
-            notify,
-            request_context,
-        }
-    }
-
-    pub(super) fn version_id(self, version_id: String) -> Self {
-        let Self {
-            builder,
-            notify,
-            request_context,
-        } = self;
-        Self {
-            builder: builder.version_id(version_id),
-            notify,
-            request_context,
-        }
-    }
-
-    /// Publish the event when `result` is a success; failures publish nothing.
-    pub(super) fn complete<T>(self, result: &S3Result<S3Response<T>>) {
-        let Ok(response) = result else {
-            return;
-        };
-        let Self {
-            builder,
-            notify,
-            request_context,
-        } = self;
-        let event_args = builder
-            .resp_elements(build_event_resp_elements(response, &request_context.request_id))
-            .build();
-        spawn_background_with_context(Some(request_context), async move {
-            notify.notify(event_args).await;
-        });
-    }
-}
-
-impl DefaultObjectUsecase {
-    /// Write one object through the ordinary PutObject path on behalf of a
-    /// trusted internal caller.
-    ///
-    /// The body is consumed exactly like a client request body: hashed
-    /// against `expected_md5_hex`, compressed and encrypted per bucket policy,
-    /// and committed under quota admission. On any failure nothing is left
-    /// behind.
-    pub(crate) async fn internal_put_object<B>(&self, ctx: InternalPutContext, body: B) -> Result<ObjectInfo, ApiError>
-    where
-        B: Stream<Item = io::Result<Bytes>> + Send + Sync + 'static,
-    {
-        let start_time = Instant::now();
-        let InternalPutContext {
-            bucket,
-            key,
-            size,
-            expected_md5_hex,
-            preserve_etag,
-            content_headers,
-            user_metadata,
-            tags,
-            internal_metadata,
-            emit_events,
-            principal_id,
-        } = ctx;
-        let Some(size) = size else {
-            return Err(ApiError::invalid_request("internal put requires a known object size"));
-        };
-        let size = i64::try_from(size).map_err(|_| ApiError::invalid_request("internal put size exceeds the supported range"))?;
-
-        let headers = internal_put_headers(&content_headers)?;
-        validate_internal_write_target(&key, &bucket, &headers).await?;
-
-        let write = PutObjectWriteRequest {
-            bucket,
-            key,
-            size,
-            quota_operation: QuotaOperation::PutObject,
-            ciphertext_passthrough: false,
-            inbound_replication_put: false,
-            headers: &headers,
-            query: None,
-            trailing_headers: None,
-            version_id: None,
-            sse: PutObjectSseInput {
-                server_side_encryption: None,
-                ssekms_key_id: None,
-                sse_customer_algorithm: None,
-                sse_customer_key: None,
-                sse_customer_key_md5: None,
-            },
-            user_metadata,
-            internal_metadata,
-            content: internal_put_content_input(&headers, tags),
-            object_lock: PutObjectLockInput {
-                legal_hold_status: None,
-                mode: None,
-                retain_until_date: None,
-            },
-            content_md5: expected_md5_hex.map(PutObjectContentMd5::Hex),
-            preserve_etag,
-            origin: PutObjectOrigin::Internal {
-                principal_id,
-                emit_events,
-            },
-        };
-        let committed = self
-            .put_object_core(write, StreamingBlob::wrap(body), start_time)
-            .await
-            .map_err(api_error_from_s3)?;
-
-        let obj_info = committed.obj_info.clone();
-        let result: S3Result<S3Response<()>> = Ok(S3Response::new(()));
-        committed.finish(&result);
-        Ok(obj_info)
-    }
-
-    /// Start a multipart upload for an internal write. The session carries the
-    /// same metadata, bucket default SSE session material, Object Lock
-    /// defaults and replication decision a client-initiated session would.
-    pub(crate) async fn internal_create_multipart_upload(&self, ctx: &InternalPutContext) -> Result<String, ApiError> {
-        let headers = internal_put_headers(&ctx.content_headers)?;
-        validate_internal_write_target(&ctx.key, &ctx.bucket, &headers).await?;
-        let store = self.object_store().ok_or_else(not_initialized)?;
-
-        let mut metadata = ctx.user_metadata.clone();
-        namespace_reserved_user_metadata(&mut metadata);
-        extract_metadata_from_mime(&headers, &mut metadata);
-        if let Some(tags) = ctx.tags.clone() {
-            metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags);
-        }
-
-        let object_lock_config_state = load_bucket_object_lock_config_state(&ctx.bucket)
-            .await
-            .map_err(api_error_from_s3)?;
-        apply_bucket_default_lock_retention(&ctx.bucket, &object_lock_config_state, &mut metadata, false)
-            .map_err(api_error_from_s3)?;
-
-        // Internal callers carry no credential; a bucket default of SSE-KMS is
-        // authorized as an internal write, like every other system write.
-        let prepared_material = sse_prepare_encryption(PrepareEncryptionRequest {
-            bucket: &ctx.bucket,
-            key: &ctx.key,
-            server_side_encryption: None,
-            ssekms_key_id: None,
-            ssekms_context: None,
-            sse_customer_algorithm: None,
-            sse_customer_key: None,
-            sse_customer_key_md5: None,
-            principal: None,
-        })
-        .await?;
-        if let Some(material) = prepared_material {
-            let mut encryption_metadata = encryption_material_to_metadata(&material)?;
-            if material.key_kind == EncryptionKeyKind::Object {
-                mark_encrypted_multipart_metadata(&mut encryption_metadata);
-            }
-            metadata.extend(encryption_metadata);
-        }
-
-        if is_multipart_disk_compression_enabled() && is_disk_compressible(&headers, &ctx.key) {
-            insert_str(
-                &mut metadata,
-                SUFFIX_COMPRESSION,
-                compression_metadata_value(CompressionAlgorithm::default()),
-            );
-        }
-        metadata.extend(ctx.internal_metadata.clone());
-
-        let mt2 = metadata.clone();
-        let mut opts = put_opts_with_replication_authorization(&ctx.bucket, &ctx.key, None, &headers, metadata, false)
-            .await
-            .map_err(ApiError::from)?;
-
-        let dsc = must_replicate_object(
-            &ctx.bucket,
-            &ctx.key,
-            &mt2,
-            "".to_string(),
-            opts.delete_marker_replication_status(),
-            opts.clone(),
-        )
-        .await;
-        if dsc.replicate_any() {
-            insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
-            insert_str(
-                &mut opts.user_defined,
-                SUFFIX_REPLICATION_STATUS,
-                dsc.pending_status().unwrap_or_default(),
-            );
-        }
-
-        let current_opts = get_opts(&ctx.bucket, &ctx.key, opts.version_id.clone(), None, &headers)
-            .await
-            .map_err(ApiError::from)?;
-        match store.get_object_info(&ctx.bucket, &ctx.key, &current_opts).await {
-            Ok(existing_obj_info) => {
-                validate_existing_object_lock_for_write(&object_lock_config_state, &existing_obj_info, &opts)
-                    .map_err(api_error_from_s3)?;
-            }
-            Err(err) => {
-                if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
-                    return Err(ApiError::from(err));
-                }
-            }
-        }
-
-        let upload = store
-            .new_multipart_upload(&ctx.bucket, &ctx.key, &opts)
-            .await
-            .map_err(ApiError::from)?;
-        Ok(upload.upload_id)
-    }
-
-    /// Stage one part of an internal multipart upload. Compression and managed
-    /// SSE follow the session metadata recorded at creation; the staged part
-    /// is verified against `expected_md5_hex` when given.
-    pub(crate) async fn internal_upload_part<B>(
-        &self,
-        ctx: &InternalPutContext,
-        upload_id: &str,
-        part_number: usize,
-        size: u64,
-        expected_md5_hex: Option<String>,
-        body: B,
-    ) -> Result<CompletePart, ApiError>
-    where
-        B: Stream<Item = io::Result<Bytes>> + Send + Sync + Unpin + 'static,
-    {
-        let size =
-            i64::try_from(size).map_err(|_| ApiError::invalid_request("internal part size exceeds the supported range"))?;
-        let bucket = ctx.bucket.as_str();
-        let key = ctx.key.as_str();
-        let store = self.object_store().ok_or_else(not_initialized)?;
-        let mut opts = ObjectOptions::default();
-        let session = store
-            .get_multipart_info(bucket, key, upload_id, &opts)
-            .await
-            .map_err(ApiError::from)?;
-
-        let upload_part_admission = match get_concurrency_manager()
-            .admit_multipart_part(size)
-            .await
-            .map_err(|_| ApiError::other(io::Error::other("foreground write admission closed")))?
-        {
-            ForegroundWriteAdmission::Disabled => None,
-            ForegroundWriteAdmission::Admitted(permit) => {
-                counter!("rustfs.upload_part.foreground_admission.total", "result" => "admitted").increment(1);
-                Some(permit)
-            }
-            ForegroundWriteAdmission::Rejected => {
-                counter!("rustfs.upload_part.foreground_admission.total", "result" => "rejected").increment(1);
-                return Err(ApiError {
-                    code: S3ErrorCode::SlowDown,
-                    message: "foreground write concurrency limit reached, please reduce your request rate".to_string(),
-                    source: None,
-                });
-            }
-        };
-
-        let buffer_size = get_buffer_size_opt_in(size);
-        let body = tokio::io::BufReader::with_capacity(buffer_size, StreamReader::new(body));
-        let actual_size = size;
-        let mut write_plan = WritePlan::new();
-        let mut reader = if rustfs_utils::http::contains_key_str(&session.user_defined, SUFFIX_COMPRESSION) {
-            let hrd = HashReader::from_stream(body, size, actual_size, expected_md5_hex, None, false).map_err(ApiError::from)?;
-            write_plan = write_plan.with_compression(CompressionAlgorithm::default());
-            hrd
-        } else {
-            HashReader::from_stream(body, size, actual_size, expected_md5_hex, None, false).map_err(ApiError::from)?
-        };
-        opts.want_checksum = reader.checksum();
-
-        if session
-            .user_defined
-            .contains_key("x-amz-server-side-encryption-customer-algorithm")
-        {
-            return Err(ApiError::invalid_request("internal multipart writes cannot continue an SSE-C session"));
-        }
-        if session.user_defined.contains_key("x-amz-server-side-encryption") {
-            // Reuses the envelope prepared at creation; the session pins the key.
-            let managed_material = sse_decryption(DecryptionRequest {
-                bucket,
-                key,
-                metadata: &session.user_defined,
-                sse_customer_key: None,
-                sse_customer_key_md5: None,
-                principal: None,
-            })
-            .await?
-            .ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
-            let managed_write = match managed_material.key_kind {
-                EncryptionKeyKind::Object => {
-                    WriteEncryption::multipart_object_key(managed_material.key_bytes, part_number as u32)
-                }
-                EncryptionKeyKind::Direct => {
-                    WriteEncryption::multipart(managed_material.key_bytes, managed_material.base_nonce, part_number)
-                }
-            };
-            write_plan = write_plan.with_encryption(managed_write);
-        }
-
-        reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?;
-        let mut reader = PutObjReader::new(reader);
-
-        let _upload_part_admission = upload_part_admission;
-        let info = store
-            .put_object_part(bucket, key, upload_id, part_number, &mut reader, &opts)
-            .await
-            .map_err(ApiError::from)?;
-        drop(_upload_part_admission);
-
-        Ok(CompletePart {
-            part_num: info.part_num,
-            etag: info.etag,
-            ..Default::default()
-        })
-    }
-
-    /// Complete an internal multipart upload: versioning, quota admission,
-    /// Object Lock validation of the overwritten version, usage accounting,
-    /// immediate ILM transition, replication scheduling and the creation
-    /// event, as the S3 CompleteMultipartUpload handler performs them.
-    pub(crate) async fn internal_complete_multipart_upload(
-        &self,
-        ctx: &InternalPutContext,
-        upload_id: &str,
-        parts: Vec<CompletePart>,
-    ) -> Result<ObjectInfo, ApiError> {
-        let bucket = ctx.bucket.clone();
-        let key = ctx.key.clone();
-        if parts.is_empty() {
-            return Err(ApiError::invalid_request("You must specify at least one part"));
-        }
-        if parts.windows(2).any(|pair| pair[0].part_num >= pair[1].part_num) {
-            return Err(ApiError::invalid_request("multipart parts must be listed in ascending part number order"));
-        }
-        validate_table_catalog_object_mutation(&bucket, &key)
-            .await
-            .map_err(api_error_from_s3)?;
-        let store = self.object_store().ok_or_else(not_initialized)?;
-
-        let headers = HeaderMap::new();
-        let mut opts =
-            get_complete_multipart_upload_opts_with_replication_authorization(&headers, false).map_err(ApiError::from)?;
-        opts.preserve_etag = ctx.preserve_etag.clone();
-        let versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
-        opts.versioned = versioned;
-        opts.version_suspended = BucketVersioningSys::prefix_suspended(&bucket, &key).await;
-        let capacity_scope_token = Uuid::new_v4();
-        opts.capacity_scope_token = Some(capacity_scope_token);
-
-        let current_opts =
-            internal_object_info_lookup_opts(get_opts(&bucket, &key, None, None, &headers).await.map_err(ApiError::from)?);
-        let object_lock_config_state = load_bucket_object_lock_config_state(&bucket)
-            .await
-            .map_err(api_error_from_s3)?;
-        let previous_current_sizes = match store.get_object_info(&bucket, &key, &current_opts).await {
-            Ok(existing_obj_info) => {
-                validate_existing_object_lock_for_write(&object_lock_config_state, &existing_obj_info, &current_opts)
-                    .map_err(api_error_from_s3)?;
-                let physical_size = existing_obj_info.size.max(0) as u64;
-                let logical_size = quota_object_size(&existing_obj_info);
-                Some((physical_size, logical_size))
-            }
-            Err(err) => {
-                if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
-                    return Err(ApiError::from(err));
-                }
-                None
-            }
-        };
-
-        let cache_adapter = self.object_data_cache();
-        let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await;
-
-        let quota_metadata_sys = self.bucket_metadata_sys();
-        let quota_tracking = quota_metadata_sys.is_some();
-        let mut quota_enabled = false;
-        if let Some(metadata_sys) = quota_metadata_sys {
-            let quota_checker = QuotaChecker::new(metadata_sys);
-            let check_result =
-                map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)
-                    .map_err(api_error_from_s3)?;
-            quota_enabled = check_result.quota_limit.is_some();
-            apply_quota_admission(&mut opts, &check_result).map_err(api_error_from_s3)?;
-        }
-
-        let previous_current_size = match previous_current_sizes {
-            Some((_, Ok(logical_size))) if quota_enabled => Some(logical_size),
-            Some((_, Err(err))) if quota_enabled => return Err(ApiError::from(err)),
-            Some((physical_size, _)) => Some(physical_size),
-            None => None,
-        };
-
-        let event = ctx.emit_events.then(|| {
-            InternalPutObjectEvent::new(
-                current_notify_interface_for_context(self.context.as_deref()),
-                request_context::RequestContext::fallback(),
-                EventName::ObjectCreatedCompleteMultipartUpload,
-                &bucket,
-                &key,
-                ctx.principal_id,
-            )
-        });
-
-        // The spawned task owns the commit so a cancelled caller cannot leave
-        // the bookkeeping half done.
-        let complete_commit = spawn_traced_join({
-            let store = Arc::clone(&store);
-            let bucket = bucket.clone();
-            let key = key.clone();
-            let upload_id = upload_id.to_string();
-            let opts = opts.clone();
-            async move {
-                let obj_info = store
-                    .clone()
-                    .complete_multipart_upload(&bucket, &key, &upload_id, parts, &opts)
-                    .await
-                    .map_err(ApiError::from)?;
-                let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await;
-                record_capacity_write(Some(capacity_scope_token)).await;
-
-                if quota_tracking {
-                    let committed_size = quota_accounting_object_size(&obj_info, quota_enabled).map_err(api_error_from_s3)?;
-                    if versioned {
-                        record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await;
-                    } else {
-                        record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await;
-                    }
-                }
-
-                enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await;
-
-                let mt2 = obj_info.user_defined.clone();
-                let dsc = must_replicate_object(
-                    &bucket,
-                    &key,
-                    &mt2,
-                    "".to_string(),
-                    opts.delete_marker_replication_status(),
-                    opts.clone(),
-                )
-                .await;
-                if dsc.replicate_any() {
-                    schedule_object_replication(obj_info.clone(), store, dsc).await;
-                }
-
-                rustfs_scanner::record_dirty_usage_bucket(&bucket);
-                Ok::<_, ApiError>(obj_info)
-            }
-        });
-        let obj_info = complete_commit.await.map_err(|err| {
-            ApiError::other(io::Error::other(format!("complete multipart upload commit owner task failed: {err}")))
-        })??;
-
-        if let Some(event) = event.flatten() {
-            let mut event = event.object(obj_info.clone());
-            if versioned && let Some(version_id) = obj_info.version_id {
-                event = event.version_id(version_id.to_string());
-            }
-            let result: S3Result<S3Response<()>> = Ok(S3Response::new(()));
-            event.complete(&result);
-        }
-        Ok(obj_info)
-    }
-
-    /// Discard an internal multipart upload and its staged parts.
-    pub(crate) async fn internal_abort_multipart_upload(&self, bucket: &str, key: &str, upload_id: &str) -> Result<(), ApiError> {
-        let store = self.object_store().ok_or_else(not_initialized)?;
-        store
-            .abort_multipart_upload(bucket, key, upload_id, &ObjectOptions::default())
-            .await
-            .map_err(ApiError::from)?;
-        rustfs_scanner::record_dirty_usage_bucket(bucket);
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
-    use rustfs_utils::http::{
-        MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, SUFFIX_ODM_PULLED_AT, SUFFIX_ODM_SOURCE, SUFFIX_ODM_SOURCE_ETAG,
-        SUFFIX_ODM_SOURCE_LAST_MODIFIED, SUFFIX_ODM_SOURCE_VERSION_ID, contains_key_str, get_str,
-    };
-
-    const TEST_PRINCIPAL: &str = "rustfs-internal-put-test";
-
-    fn md5_hex(body: &[u8]) -> String {
-        hex_simd::encode_to_string(Md5::digest(body), hex_simd::AsciiCase::Lower)
-    }
-
-    fn body_stream(chunks: Vec<Bytes>) -> impl Stream<Item = io::Result<Bytes>> + Send + Sync + Unpin + 'static {
-        futures::stream::iter(chunks.into_iter().map(Ok))
-    }
-
-    fn provenance_metadata() -> HashMap<String, String> {
-        let mut internal_metadata = HashMap::new();
-        insert_str(&mut internal_metadata, SUFFIX_ODM_SOURCE, "s3:source-bucket".to_string());
-        insert_str(
-            &mut internal_metadata,
-            SUFFIX_ODM_SOURCE_ETAG,
-            "\"0123456789abcdef0123456789abcdef-3\"".to_string(),
-        );
-        insert_str(
-            &mut internal_metadata,
-            SUFFIX_ODM_SOURCE_LAST_MODIFIED,
-            "2026-01-02T03:04:05Z".to_string(),
-        );
-        insert_str(&mut internal_metadata, SUFFIX_ODM_SOURCE_VERSION_ID, String::new());
-        insert_str(&mut internal_metadata, SUFFIX_ODM_PULLED_AT, "2026-09-02T00:00:00Z".to_string());
-        internal_metadata
-    }
-
-    fn internal_context(bucket: &str, key: &str, body: &[u8]) -> InternalPutContext {
-        InternalPutContext {
-            bucket: bucket.to_string(),
-            key: key.to_string(),
-            size: Some(body.len() as u64),
-            expected_md5_hex: Some(md5_hex(body)),
-            preserve_etag: None,
-            content_headers: HashMap::from([
-                ("Content-Type".to_string(), "text/plain".to_string()),
-                ("Cache-Control".to_string(), "max-age=60".to_string()),
-            ]),
-            user_metadata: HashMap::from([("origin".to_string(), "unit-test".to_string())]),
-            tags: Some("team=storage".to_string()),
-            internal_metadata: provenance_metadata(),
-            emit_events: false,
-            principal_id: TEST_PRINCIPAL,
-        }
-    }
-
-    async fn internal_put_test_bucket(prefix: &str) -> (Arc<ECStore>, String) {
-        let store = crate::app::gating_test_env::shared_gating_ecstore().await;
-        crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
-        let bucket = format!("{prefix}-{}", Uuid::new_v4().simple());
-        store
-            .make_bucket(&bucket, &MakeBucketOptions::default())
-            .await
-            .expect("create internal put test bucket");
-        (store, bucket)
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn internal_put_object_writes_through_the_shared_put_path() {
-        let (store, bucket) = internal_put_test_bucket("internal-put").await;
-        let body = b"internal write-back body".to_vec();
-        let ctx = internal_context(&bucket, "dir/object.txt", &body);
-
-        let obj_info = DefaultObjectUsecase::from_global()
-            .internal_put_object(ctx, body_stream(vec![Bytes::from(body.clone())]))
-            .await
-            .expect("internal put must succeed");
-
-        assert_eq!(obj_info.etag.as_deref(), Some(md5_hex(&body).as_str()));
-        assert_eq!(obj_info.size, body.len() as i64);
-
-        let stored = store
-            .get_object_info(&bucket, "dir/object.txt", &ObjectOptions::default())
-            .await
-            .expect("internal put must leave a readable object");
-        let metadata = &stored.user_defined;
-        assert_eq!(metadata.get("content-type").map(String::as_str), Some("text/plain"));
-        assert_eq!(metadata.get("cache-control").map(String::as_str), Some("max-age=60"));
-        assert_eq!(metadata.get("origin").map(String::as_str), Some("unit-test"));
-        assert_eq!(stored.user_tags.as_str(), "team=storage", "tags must be committed as object tags");
-        for suffix in [
-            SUFFIX_ODM_SOURCE,
-            SUFFIX_ODM_SOURCE_ETAG,
-            SUFFIX_ODM_SOURCE_LAST_MODIFIED,
-            SUFFIX_ODM_SOURCE_VERSION_ID,
-            SUFFIX_ODM_PULLED_AT,
-        ] {
-            assert!(
-                metadata.contains_key(&format!("{RUSTFS_INTERNAL_PREFIX}{suffix}")),
-                "missing RustFS provenance key {suffix}"
-            );
-            assert!(
-                metadata.contains_key(&format!("{MINIO_INTERNAL_PREFIX}{suffix}")),
-                "missing MinIO provenance key {suffix}"
-            );
-        }
-        assert_eq!(get_str(metadata, SUFFIX_ODM_SOURCE).as_deref(), Some("s3:source-bucket"));
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn internal_put_object_preserves_the_caller_etag() {
-        let (store, bucket) = internal_put_test_bucket("internal-put-etag").await;
-        let body = b"etag is preserved verbatim".to_vec();
-        let mut ctx = internal_context(&bucket, "preserved.bin", &body);
-        ctx.preserve_etag = Some("0123456789abcdef0123456789abcdef-3".to_string());
-
-        let obj_info = DefaultObjectUsecase::from_global()
-            .internal_put_object(ctx, body_stream(vec![Bytes::from(body.clone())]))
-            .await
-            .expect("internal put with a preserved ETag must succeed");
-        assert_eq!(obj_info.etag.as_deref(), Some("0123456789abcdef0123456789abcdef-3"));
-
-        let stored = store
-            .get_object_info(&bucket, "preserved.bin", &ObjectOptions::default())
-            .await
-            .expect("preserved-ETag object must be readable");
-        assert_eq!(stored.etag.as_deref(), Some("0123456789abcdef0123456789abcdef-3"));
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn internal_put_object_rejects_a_digest_mismatch_without_committing() {
-        let (store, bucket) = internal_put_test_bucket("internal-put-digest").await;
-        let body = b"body whose digest will not match".to_vec();
-        let mut ctx = internal_context(&bucket, "mismatch.bin", &body);
-        ctx.expected_md5_hex = Some(md5_hex(b"a different body"));
-
-        let err = DefaultObjectUsecase::from_global()
-            .internal_put_object(ctx, body_stream(vec![Bytes::from(body)]))
-            .await
-            .expect_err("digest mismatch must fail the internal put");
-        assert_eq!(err.code, S3ErrorCode::BadDigest, "unexpected error: {err}");
-
-        let lookup = store
-            .get_object_info(&bucket, "mismatch.bin", &ObjectOptions::default())
-            .await;
-        assert!(
-            lookup.as_ref().is_err_and(is_err_object_not_found),
-            "a rejected internal put must not leave an object behind"
-        );
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn internal_put_object_requires_a_known_size() {
-        let (_store, bucket) = internal_put_test_bucket("internal-put-size").await;
-        let body = b"unknown length".to_vec();
-        let mut ctx = internal_context(&bucket, "unknown.bin", &body);
-        ctx.size = None;
-
-        let err = DefaultObjectUsecase::from_global()
-            .internal_put_object(ctx, body_stream(vec![Bytes::from(body)]))
-            .await
-            .expect_err("an unknown size must be rejected before the body is read");
-        assert_eq!(err.code, S3ErrorCode::InvalidRequest);
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn internal_multipart_roundtrip_completes_and_abort_leaves_nothing() {
-        const FIRST_PART_SIZE: usize = 5 * 1024 * 1024;
-        let (store, bucket) = internal_put_test_bucket("internal-mpu").await;
-        let usecase = DefaultObjectUsecase::from_global();
-        let first_part = vec![0x41u8; FIRST_PART_SIZE];
-        let last_part = b"tail of the multipart object".to_vec();
-        let mut ctx = internal_context(&bucket, "multipart/object.bin", &[]);
-        ctx.size = None;
-        ctx.expected_md5_hex = None;
-        ctx.preserve_etag = Some("0123456789abcdef0123456789abcdef-2".to_string());
-
-        let upload_id = usecase
-            .internal_create_multipart_upload(&ctx)
-            .await
-            .expect("internal multipart create must succeed");
-        let part_one = usecase
-            .internal_upload_part(
-                &ctx,
-                &upload_id,
-                1,
-                first_part.len() as u64,
-                Some(md5_hex(&first_part)),
-                body_stream(vec![Bytes::from(first_part.clone())]),
-            )
-            .await
-            .expect("first internal part must stage");
-        let part_two = usecase
-            .internal_upload_part(
-                &ctx,
-                &upload_id,
-                2,
-                last_part.len() as u64,
-                Some(md5_hex(&last_part)),
-                body_stream(vec![Bytes::from(last_part.clone())]),
-            )
-            .await
-            .expect("last internal part must stage");
-        assert_eq!(part_one.part_num, 1);
-        assert_eq!(part_two.part_num, 2);
-
-        let obj_info = usecase
-            .internal_complete_multipart_upload(&ctx, &upload_id, vec![part_one, part_two])
-            .await
-            .expect("internal multipart complete must succeed");
-        assert_eq!(obj_info.size, (first_part.len() + last_part.len()) as i64);
-        assert_eq!(obj_info.parts.len(), 2);
-        assert_eq!(obj_info.etag.as_deref(), Some("0123456789abcdef0123456789abcdef-2"));
-        let stored = store
-            .get_object_info(&bucket, &ctx.key, &ObjectOptions::default())
-            .await
-            .expect("completed multipart object must be readable");
-        assert_eq!(stored.user_defined.get("content-type").map(String::as_str), Some("text/plain"));
-        assert_eq!(stored.user_defined.get("origin").map(String::as_str), Some("unit-test"));
-        assert!(contains_key_str(&stored.user_defined, SUFFIX_ODM_SOURCE));
-
-        let aborted_upload_id = usecase
-            .internal_create_multipart_upload(&ctx)
-            .await
-            .expect("second internal multipart create must succeed");
-        usecase
-            .internal_upload_part(
-                &ctx,
-                &aborted_upload_id,
-                1,
-                last_part.len() as u64,
-                None,
-                body_stream(vec![Bytes::from(last_part.clone())]),
-            )
-            .await
-            .expect("part of the aborted upload must stage");
-        usecase
-            .internal_abort_multipart_upload(&bucket, &ctx.key, &aborted_upload_id)
-            .await
-            .expect("internal abort must succeed");
-        let uploads = store
-            .list_multipart_uploads(&bucket, &ctx.key, None, None, None, 100)
-            .await
-            .expect("list multipart uploads after abort");
-        assert!(
-            uploads.uploads.iter().all(|upload| upload.upload_id != aborted_upload_id),
-            "aborted internal upload must not linger: {:?}",
-            uploads.uploads
-        );
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn internal_complete_multipart_upload_rejects_unordered_parts() {
-        let (_store, bucket) = internal_put_test_bucket("internal-mpu-order").await;
-        let ctx = internal_context(&bucket, "unordered.bin", &[]);
-        let parts = vec![
-            CompletePart {
-                part_num: 2,
-                ..Default::default()
-            },
-            CompletePart {
-                part_num: 1,
-                ..Default::default()
-            },
-        ];
-        let err = DefaultObjectUsecase::from_global()
-            .internal_complete_multipart_upload(&ctx, "upload", parts)
-            .await
-            .expect_err("unordered parts must be rejected before touching the store");
-        assert_eq!(err.code, S3ErrorCode::InvalidRequest);
-    }
-
-    #[test]
-    fn internal_put_event_names_the_principal_and_creation_event() {
-        let event_args = InternalPutObjectEvent::builder(EventName::ObjectCreatedPut, "bucket", "key", TEST_PRINCIPAL).build();
-        assert_eq!(event_args.event_name, EventName::ObjectCreatedPut);
-        assert_eq!(event_args.bucket_name, "bucket");
-        assert_eq!(event_args.req_params.get("principalId").map(String::as_str), Some(TEST_PRINCIPAL));
-        assert!(!event_args.is_replication_request());
-    }
-
-    #[test]
-    fn internal_put_headers_normalize_names_and_reject_invalid_values() {
-        let headers = internal_put_headers(&HashMap::from([
-            ("Content-Type".to_string(), "application/json".to_string()),
-            ("Cache-Control".to_string(), "no-cache".to_string()),
-        ]))
-        .expect("valid content headers must build");
-        assert_eq!(headers.get("content-type").and_then(|v| v.to_str().ok()), Some("application/json"));
-        let content = internal_put_content_input(&headers, Some("a=b".to_string()));
-        assert_eq!(content.content_type.as_deref(), Some("application/json"));
-        assert_eq!(content.cache_control.as_deref(), Some("no-cache"));
-        assert_eq!(content.tagging.as_deref(), Some("a=b"));
-        assert!(content.storage_class.is_none());
-
-        let err = internal_put_headers(&HashMap::from([("Content-Type".to_string(), "bad\nvalue".to_string())]))
-            .expect_err("a header value with a control character must be rejected");
-        assert_eq!(err.code, S3ErrorCode::InvalidRequest);
-    }
-}

rustfs/src/app/object/mod.rs

@@ -191,10 +191,6 @@ mod delete;
 mod extract;
 mod get;
 mod head;
-// Consumed by the on-demand migration write-back (rustfs/backlog#2153); until
-// that lands only tests construct the internal entry points.
-#[cfg_attr(not(test), expect(dead_code, reason = "wired by the on-demand migration write-back"))]
-mod internal_put;
 mod put;
 mod restore;
 mod shared;
@@ -206,7 +202,6 @@ pub(crate) use self::copy::*;
 pub(crate) use self::delete::*;
 pub(crate) use self::extract::*;
 pub(crate) use self::get::*;
-pub(crate) use self::internal_put::*;
 use self::put::*;
 pub(crate) use self::put::{guard_put_object_body_read_timeout, put_object_body_read_timeout};
 pub(crate) use self::shared::*;

rustfs/src/app/object/put.rs

@@ -895,226 +895,6 @@ fn is_post_object_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap)
     is_sse_kms_requested(input, headers)
 }
 
-/// Standard content headers and the tagging / storage-class values of a PUT
-/// that become object metadata through [`apply_put_request_metadata`].
-pub(super) struct PutObjectContentInput {
-    pub(super) cache_control: Option<CacheControl>,
-    pub(super) content_disposition: Option<ContentDisposition>,
-    pub(super) content_encoding: Option<ContentEncoding>,
-    pub(super) content_language: Option<ContentLanguage>,
-    pub(super) content_type: Option<ContentType>,
-    pub(super) expires: Option<String>,
-    pub(super) website_redirect_location: Option<WebsiteRedirectLocation>,
-    pub(super) tagging: Option<TaggingHeader>,
-    pub(super) storage_class: Option<StorageClass>,
-}
-
-/// Encryption inputs of a PUT after the S3 input/header merge.
-pub(super) struct PutObjectSseInput {
-    pub(super) server_side_encryption: Option<ServerSideEncryption>,
-    pub(super) ssekms_key_id: Option<SSEKMSKeyId>,
-    pub(super) sse_customer_algorithm: Option<SSECustomerAlgorithm>,
-    pub(super) sse_customer_key: Option<s3s::dto::SSECustomerKey>,
-    pub(super) sse_customer_key_md5: Option<SSECustomerKeyMD5>,
-}
-
-/// Explicit Object Lock values of a PUT; all `None` means the bucket default
-/// retention decides.
-pub(super) struct PutObjectLockInput {
-    pub(super) legal_hold_status: Option<ObjectLockLegalHoldStatus>,
-    pub(super) mode: Option<ObjectLockMode>,
-    pub(super) retain_until_date: Option<Timestamp>,
-}
-
-/// Expected body MD5 as the caller carries it. It is decoded at the same
-/// point of the write path for every origin so an invalid digest keeps its
-/// precedence relative to quota, admission and Object Lock errors.
-pub(super) enum PutObjectContentMd5 {
-    /// `Content-MD5` request header value.
-    Base64(String),
-    /// Lowercase hex digest, as an internal caller already holds it.
-    #[cfg_attr(not(test), expect(dead_code, reason = "constructed by the internal put entry point"))]
-    Hex(String),
-}
-
-/// Where a single-object write originates.
-pub(super) enum PutObjectOrigin<'a> {
-    /// The S3 PutObject/PostObject handler: request-bound identity, the
-    /// audit/notification chain and the bucket-generation guard installed by
-    /// the access layer all come from the request.
-    S3 {
-        req: &'a S3Request<PutObjectInput>,
-        event_name: EventName,
-    },
-    /// A trusted in-process caller writing on the server's behalf. There is no
-    /// request and no credential: managed-SSE authorization treats the write
-    /// as internal, and the creation event, when requested, names
-    /// `principal_id` instead of an access key.
-    #[cfg_attr(not(test), expect(dead_code, reason = "constructed by the internal put entry point"))]
-    Internal { principal_id: &'static str, emit_events: bool },
-}
-
-impl PutObjectOrigin<'_> {
-    fn replication_request_authorized(&self) -> bool {
-        match self {
-            Self::S3 { req, .. } => replication_request_authorized(req),
-            Self::Internal { .. } => false,
-        }
-    }
-
-    fn apply_bucket_generation_guard(&self, bucket: &str, opts: &mut ObjectOptions) -> S3Result<()> {
-        match self {
-            Self::S3 { req, .. } => apply_bucket_generation_guard(req, bucket, opts),
-            Self::Internal { .. } => Ok(()),
-        }
-    }
-
-    fn sse_principal(&self) -> Option<SseKmsPrincipal> {
-        match self {
-            Self::S3 { req, .. } => SseKmsPrincipal::from_request(req),
-            Self::Internal { .. } => None,
-        }
-    }
-}
-
-/// Every input the shared single-object write path needs, independent of
-/// whether an S3 request or an internal caller produced it.
-pub(super) struct PutObjectWriteRequest<'a> {
-    pub(super) bucket: String,
-    pub(super) key: String,
-    /// Authoritative plaintext length; never negative.
-    pub(super) size: i64,
-    pub(super) quota_operation: QuotaOperation,
-    /// Authorized SSE-C replication body that is already ciphertext.
-    pub(super) ciphertext_passthrough: bool,
-    pub(super) inbound_replication_put: bool,
-    /// Request headers, or the object's content headers for an internal write.
-    pub(super) headers: &'a HeaderMap,
-    pub(super) query: Option<&'a str>,
-    pub(super) trailing_headers: Option<s3s::TrailingHeaders>,
-    pub(super) version_id: Option<String>,
-    pub(super) sse: PutObjectSseInput,
-    /// User metadata keyed the way s3s delivers it (`x-amz-meta-` stripped).
-    pub(super) user_metadata: HashMap<String, String>,
-    /// Internal `x-rustfs-internal-*` / `x-minio-internal-*` keys written
-    /// verbatim onto the object; empty for S3 requests.
-    pub(super) internal_metadata: HashMap<String, String>,
-    pub(super) content: PutObjectContentInput,
-    pub(super) object_lock: PutObjectLockInput,
-    pub(super) content_md5: Option<PutObjectContentMd5>,
-    /// ETag to store instead of the computed one; `None` keeps the computed
-    /// (or replication-header-derived) value.
-    pub(super) preserve_etag: Option<String>,
-    pub(super) origin: PutObjectOrigin<'a>,
-}
-
-/// Audit/notification completion of a write, per origin.
-pub(super) enum PutObjectCompletion {
-    S3(OperationHelper),
-    Internal(Option<Box<InternalPutObjectEvent>>),
-}
-
-impl PutObjectCompletion {
-    fn object(self, obj_info: ObjectInfo) -> Self {
-        match self {
-            Self::S3(helper) => Self::S3(helper.object(obj_info)),
-            Self::Internal(event) => Self::Internal(event.map(|event| Box::new(event.object(obj_info)))),
-        }
-    }
-
-    fn version_id(self, version_id: String) -> Self {
-        match self {
-            Self::S3(helper) => Self::S3(helper.version_id(version_id)),
-            Self::Internal(event) => Self::Internal(event.map(|event| Box::new(event.version_id(version_id)))),
-        }
-    }
-
-    fn complete<T>(self, result: &S3Result<S3Response<T>>) -> Self {
-        match self {
-            Self::S3(helper) => Self::S3(helper.complete(result)),
-            Self::Internal(event) => {
-                if let Some(event) = event {
-                    event.complete(result);
-                }
-                Self::Internal(None)
-            }
-        }
-    }
-}
-
-/// Record the failed completion and hand the error back to the caller.
-fn fail_put_object(completion: PutObjectCompletion, err: S3Error) -> S3Error {
-    let result: S3Result<S3Response<()>> = Err(err);
-    let _ = completion.complete(&result);
-    match result {
-        Err(err) => err,
-        Ok(_) => unreachable!("failed PutObject completion carries an error"),
-    }
-}
-
-/// A committed single-object write awaiting its response-side completion.
-pub(super) struct PutObjectCommitted {
-    pub(super) obj_info: ObjectInfo,
-    pub(super) put_versioned: bool,
-    pub(super) effective_sse: Option<ServerSideEncryption>,
-    pub(super) effective_kms_key_id: Option<SSEKMSKeyId>,
-    pub(super) sse_customer_algorithm: Option<SSECustomerAlgorithm>,
-    pub(super) sse_customer_key_md5: Option<SSECustomerKeyMD5>,
-    pub(super) put_extra_checksum_headers: Vec<(&'static str, String)>,
-    completion: PutObjectCompletion,
-    put_request_guard: PutObjectGuard,
-    bucket: String,
-    key: String,
-    start_time: Instant,
-    size: i64,
-    use_zero_copy_eager_put_path: bool,
-    concurrent_put_requests: usize,
-    buffer_size: usize,
-}
-
-impl PutObjectCommitted {
-    /// Publish the audit entry / creation event for `result`, record the
-    /// request-level PutObject metrics and release the request guard.
-    pub(super) fn finish<T>(self, result: &S3Result<S3Response<T>>) {
-        let Self {
-            completion,
-            mut put_request_guard,
-            bucket,
-            key,
-            start_time,
-            size,
-            use_zero_copy_eager_put_path,
-            concurrent_put_requests,
-            buffer_size,
-            ..
-        } = self;
-        let _ = completion.complete(result);
-
-        // Record PutObject metrics via zero-copy-metrics
-        {
-            let duration_ms = start_time.elapsed().as_millis() as f64;
-            rustfs_io_metrics::record_put_object(
-                duration_ms,
-                size,
-                use_zero_copy_eager_put_path, // Track if zero-copy was enabled
-            );
-        }
-
-        debug!(
-            target: "rustfs::app::object_usecase",
-            component = "app",
-            subsystem = "object",
-            bucket = %bucket,
-            key = %key,
-            concurrent_put_requests,
-            buffer_size,
-            "PutObject request completed"
-        );
-
-        put_request_guard.finish_ok();
-    }
-}
-
 impl DefaultObjectUsecase {
     fn should_use_large_put_concurrency_tuning(size: i64) -> bool {
         size >= DEFAULT_PUT_LARGE_CONCURRENCY_TUNING_MIN_SIZE_BYTES
@@ -1275,7 +1055,7 @@ impl DefaultObjectUsecase {
         };
 
         // Resolve the authoritative decoded/plain object length (rejecting negative/unknown) before anything else consumes it.
-        let size = resolve_put_object_authoritative_size(&req.headers, content_length)?;
+        let mut size = resolve_put_object_authoritative_size(&req.headers, content_length)?;
 
         if let Some(limit) = max_content_length
             && u64::try_from(size).is_ok_and(|size| size > limit)
@@ -1283,140 +1063,6 @@ impl DefaultObjectUsecase {
             return Err(S3Error::new(S3ErrorCode::EntityTooLarge));
         }
 
-        let write = PutObjectWriteRequest {
-            bucket: bucket.clone(),
-            key,
-            size,
-            quota_operation,
-            ciphertext_passthrough,
-            inbound_replication_put,
-            headers: &req.headers,
-            query: req.uri.query(),
-            trailing_headers: req.trailing_headers.clone(),
-            version_id,
-            sse: PutObjectSseInput {
-                server_side_encryption,
-                ssekms_key_id,
-                sse_customer_algorithm,
-                sse_customer_key,
-                sse_customer_key_md5,
-            },
-            user_metadata: metadata.unwrap_or_default(),
-            internal_metadata: HashMap::new(),
-            content: PutObjectContentInput {
-                cache_control,
-                content_disposition,
-                content_encoding,
-                content_language,
-                content_type,
-                expires,
-                website_redirect_location,
-                tagging,
-                storage_class,
-            },
-            object_lock: PutObjectLockInput {
-                legal_hold_status: object_lock_legal_hold_status,
-                mode: object_lock_mode,
-                retain_until_date: object_lock_retain_until_date,
-            },
-            content_md5: content_md5.map(PutObjectContentMd5::Base64),
-            preserve_etag: None,
-            origin: PutObjectOrigin::S3 { req: &req, event_name },
-        };
-        let committed = self.put_object_core(write, body, start_time).await?;
-
-        let raw_version = committed.obj_info.version_id.map(|v| v.to_string());
-        let put_version = if committed.put_versioned { raw_version } else { None };
-
-        let e_tag = committed.obj_info.etag.clone().map(|etag| to_s3s_etag(&etag));
-
-        let expiration = resolve_put_object_expiration(&bucket, &committed.obj_info).await;
-
-        let mut checksums = PutObjectChecksums {
-            crc32: input.checksum_crc32,
-            crc32c: input.checksum_crc32c,
-            sha1: input.checksum_sha1,
-            sha256: input.checksum_sha256,
-            crc64nvme: input.checksum_crc64nvme,
-        };
-        apply_trailing_checksums(
-            input.checksum_algorithm.as_ref().map(|a| a.as_str()),
-            &req.trailing_headers,
-            &mut checksums,
-        );
-
-        let output = PutObjectOutput {
-            e_tag,
-            server_side_encryption: committed.effective_sse.clone(),
-            sse_customer_algorithm: committed.sse_customer_algorithm.clone(),
-            sse_customer_key_md5: committed.sse_customer_key_md5.clone(),
-            ssekms_key_id: committed.effective_kms_key_id.clone(),
-            expiration,
-            checksum_crc32: checksums.crc32,
-            checksum_crc32c: checksums.crc32c,
-            checksum_sha1: checksums.sha1,
-            checksum_sha256: checksums.sha256,
-            checksum_crc64nvme: checksums.crc64nvme,
-            version_id: put_version,
-            ..Default::default()
-        };
-
-        // For browser-based POST uploads (multipart/form-data), response status/body handling
-        // is decided by s3s PostObject serializer (success_action_status / redirect semantics).
-
-        let response_build_stage_start = put_stage_metrics_enabled.then(Instant::now);
-        let mut response = S3Response::new(output);
-        // Echo XXHash3/64/128 / SHA-512 checksums that s3s PutObjectOutput has no typed
-        // field for (#1256).
-        inject_additional_checksum_headers(&mut response.headers, &committed.put_extra_checksum_headers);
-        rustfs_io_metrics::record_put_object_stage_duration_from("app_response_build", response_build_stage_start);
-        let result = Ok(response);
-        committed.finish(&result);
-
-        result
-    }
-
-    /// The single-object write path shared by the S3 handler and internal
-    /// callers: quota admission, foreground write admission, bucket default
-    /// SSE, Object Lock defaults, put options, the hashing/compressing/
-    /// encrypting reader, the owned store commit, usage accounting,
-    /// replication scheduling and the creation-event setup. The caller shapes
-    /// the request before and builds its response after.
-    pub(super) async fn put_object_core(
-        &self,
-        write: PutObjectWriteRequest<'_>,
-        body: StreamingBlob,
-        start_time: Instant,
-    ) -> S3Result<PutObjectCommitted> {
-        let put_stage_metrics_enabled = rustfs_io_metrics::put_stage_metrics_enabled();
-        let PutObjectWriteRequest {
-            bucket,
-            key,
-            mut size,
-            quota_operation,
-            ciphertext_passthrough,
-            inbound_replication_put,
-            headers,
-            query,
-            trailing_headers,
-            version_id,
-            sse,
-            user_metadata,
-            internal_metadata,
-            content,
-            object_lock,
-            content_md5,
-            preserve_etag,
-            origin,
-        } = write;
-        let PutObjectSseInput {
-            server_side_encryption,
-            ssekms_key_id,
-            sse_customer_algorithm,
-            sse_customer_key,
-            sse_customer_key_md5,
-        } = sse;
-
         // The app check preserves the existing S3 error contract; the storage
         // commit path reserves the exact net logical growth under its locks.
         let quota_stage_start = put_stage_metrics_enabled.then(Instant::now);
@@ -1438,7 +1084,7 @@ impl DefaultObjectUsecase {
 
         let ingress_stage_start = put_stage_metrics_enabled.then(Instant::now);
         let should_compress =
-            is_disk_compressible(headers, &key) && size > MIN_DISK_COMPRESSIBLE_SIZE as i64 && !ciphertext_passthrough;
+            is_disk_compressible(&req.headers, &key) && size > MIN_DISK_COMPRESSIBLE_SIZE as i64 && !ciphertext_passthrough;
 
         // Resolve the store through the request-bound server context
         // (backlog#1052 S6), not the process-global handle, so an embedded
@@ -1529,7 +1175,7 @@ impl DefaultObjectUsecase {
         let (put_path, zero_copy_eager_put_path_status, use_zero_copy_eager_put_path, use_empty_or_small_eager_put_path) =
             select_put_path_with_concurrency(
                 size,
-                headers,
+                &req.headers,
                 server_side_encryption_requested,
                 should_compress,
                 false,
@@ -1554,33 +1200,33 @@ impl DefaultObjectUsecase {
         validate_sse_headers_for_write(
             effective_sse.as_ref(),
             effective_kms_key_id.as_ref(),
-            extract_ssekms_context_from_headers(headers)?.as_ref(),
+            extract_ssekms_context_from_headers(&req.headers)?.as_ref(),
             sse_customer_algorithm.as_ref(),
             sse_customer_key.as_ref(),
             sse_customer_key_md5.as_ref(),
             true, // PutObject requires all three: algorithm, key, key_md5
         )?;
 
-        let mut metadata = user_metadata;
-        let has_explicit_object_lock_retention = object_lock.mode.is_some()
-            || object_lock.retain_until_date.is_some()
-            || has_replication_retention_update(headers, inbound_replication_put);
+        let mut metadata = metadata.unwrap_or_default();
+        let has_explicit_object_lock_retention = object_lock_mode.is_some()
+            || object_lock_retain_until_date.is_some()
+            || has_replication_retention_update(&req.headers, inbound_replication_put);
         let object_lock_config_stage_start = put_stage_metrics_enabled.then(Instant::now);
         let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?;
         rustfs_io_metrics::record_put_object_stage_duration_from("app_object_lock_config_lookup", object_lock_config_stage_start);
         apply_put_request_metadata(
             &mut metadata,
-            headers,
+            &req.headers,
             &key,
-            content.cache_control,
-            content.content_disposition,
-            content.content_encoding,
-            content.content_language,
-            content.content_type,
-            content.expires,
-            content.website_redirect_location,
-            content.tagging,
-            content.storage_class,
+            cache_control,
+            content_disposition,
+            content_encoding,
+            content_language,
+            content_type,
+            expires,
+            website_redirect_location,
+            tagging,
+            storage_class.clone(),
         )?;
         apply_bucket_default_lock_retention(
             &bucket,
@@ -1588,33 +1234,29 @@ impl DefaultObjectUsecase {
             &mut metadata,
             has_explicit_object_lock_retention,
         )?;
-        metadata.extend(internal_metadata);
 
         let put_opts_stage_start = put_stage_metrics_enabled.then(Instant::now);
         let mut opts: ObjectOptions = put_opts_with_replication_authorization(
             &bucket,
             &key,
             version_id.clone(),
-            headers,
+            &req.headers,
             metadata.clone(),
-            origin.replication_request_authorized(),
+            replication_request_authorized(&req),
         )
         .await
         .map_err(ApiError::from)?;
-        if let Some(etag) = preserve_etag {
-            opts.preserve_etag = Some(etag);
-        }
         if let Some(quota_check) = quota_check.as_ref() {
             apply_quota_admission(&mut opts, quota_check)?;
         }
         rustfs_io_metrics::record_put_object_stage_duration_from("app_put_opts_build", put_opts_stage_start);
-        origin.apply_bucket_generation_guard(&bucket, &mut opts)?;
+        apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
         apply_put_request_object_lock_opts(
             &bucket,
             &object_lock_config_state,
-            object_lock.legal_hold_status,
-            object_lock.mode,
-            object_lock.retain_until_date,
+            object_lock_legal_hold_status,
+            object_lock_mode,
+            object_lock_retain_until_date,
             &mut opts,
         )?;
         let eager_put_commit_cancellation =
@@ -1636,7 +1278,7 @@ impl DefaultObjectUsecase {
         let prelookup_stage_start = (prelookup_required && put_stage_metrics_enabled).then(Instant::now);
         let prelookup_previous_current_size: Option<Option<u64>> = if prelookup_required {
             let current_opts: ObjectOptions = internal_object_info_lookup_opts(
-                get_opts(&bucket, &key, version_id.clone(), None, headers)
+                get_opts(&bucket, &key, version_id.clone(), None, &req.headers)
                     .await
                     .map_err(ApiError::from)?,
             );
@@ -1673,18 +1315,16 @@ impl DefaultObjectUsecase {
             )?;
         }
 
-        let mut md5hex = match content_md5 {
-            Some(PutObjectContentMd5::Base64(base64_md5)) => {
-                let md5 = base64_simd::STANDARD
-                    .decode_to_vec(base64_md5.as_bytes())
-                    .map_err(|e| ApiError::from(StorageError::other(format!("Invalid content MD5: {e}"))))?;
-                Some(hex_simd::encode_to_string(&md5, hex_simd::AsciiCase::Lower))
-            }
-            Some(PutObjectContentMd5::Hex(md5hex)) => Some(md5hex),
-            None => None,
+        let mut md5hex = if let Some(base64_md5) = content_md5 {
+            let md5 = base64_simd::STANDARD
+                .decode_to_vec(base64_md5.as_bytes())
+                .map_err(|e| ApiError::from(StorageError::other(format!("Invalid content MD5: {e}"))))?;
+            Some(hex_simd::encode_to_string(&md5, hex_simd::AsciiCase::Lower))
+        } else {
+            None
         };
 
-        let mut sha256hex = get_content_sha256_with_query(headers, query);
+        let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query());
 
         let mut write_plan = WritePlan::new();
         // Additional-checksum (XXHash3/64/128, SHA-512) values to echo on the PutObject
@@ -1702,7 +1342,7 @@ impl DefaultObjectUsecase {
             let mut hrd =
                 HashReader::from_stream(body, size, size, md5hex.take(), sha256hex.take(), false).map_err(ApiError::from)?;
 
-            if let Err(err) = hrd.add_checksum_from_s3s(headers, trailing_headers.clone(), false) {
+            if let Err(err) = hrd.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), false) {
                 return Err(ApiError::from(err).into());
             }
 
@@ -1752,7 +1392,7 @@ impl DefaultObjectUsecase {
         };
 
         if size >= 0 {
-            if let Err(err) = reader.add_checksum_from_s3s(headers, trailing_headers.clone(), false) {
+            if let Err(err) = reader.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), false) {
                 return Err(ApiError::from(err).into());
             }
 
@@ -1762,35 +1402,12 @@ impl DefaultObjectUsecase {
         rustfs_io_metrics::record_put_object_path(put_path);
         rustfs_io_metrics::record_put_object_stage_duration_from("ingress_prepare", ingress_stage_start);
 
-        let (mut completion, request_context) = match &origin {
-            PutObjectOrigin::S3 { req, event_name } => (
-                PutObjectCompletion::S3(OperationHelper::new(req, *event_name, S3Operation::PutObject)),
-                req.extensions.get::<request_context::RequestContext>().cloned(),
-            ),
-            PutObjectOrigin::Internal {
-                principal_id,
-                emit_events,
-            } => {
-                let principal_id = *principal_id;
-                let request_context = request_context::RequestContext::fallback();
-                let event = emit_events.then(|| {
-                    InternalPutObjectEvent::new(
-                        current_notify_interface_for_context(self.context.as_deref()),
-                        request_context.clone(),
-                        EventName::ObjectCreatedPut,
-                        &bucket,
-                        &key,
-                        principal_id,
-                    )
-                });
-                (PutObjectCompletion::Internal(event.flatten().map(Box::new)), Some(request_context))
-            }
-        };
-        let ssekms_context = extract_ssekms_context_from_headers(headers)?;
+        let mut helper = OperationHelper::new(&req, event_name, S3Operation::PutObject);
+        let ssekms_context = extract_ssekms_context_from_headers(&req.headers)?;
 
         // Apply encryption using unified SSE API.
         let encryption_stage_start = put_stage_metrics_enabled.then(Instant::now);
-        let write_principal = origin.sse_principal();
+        let write_principal = SseKmsPrincipal::from_request(&req);
         let encryption_request = EncryptionRequest {
             bucket: &bucket,
             key: &key,
@@ -1813,7 +1430,11 @@ impl DefaultObjectUsecase {
         } else {
             match sse_encryption(encryption_request).await {
                 Ok(material) => material,
-                Err(err) => return Err(fail_put_object(completion, err.into())),
+                Err(err) => {
+                    let result = Err(err.into());
+                    let _ = helper.complete(&result);
+                    return result;
+                }
             }
         };
 
@@ -1839,6 +1460,7 @@ impl DefaultObjectUsecase {
 
         let mt2 = metadata.clone();
         opts.user_defined.extend(metadata);
+        let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
         let request_id = request_context
             .as_ref()
             .map(|ctx| ctx.request_id.clone())
@@ -2038,41 +1660,100 @@ impl DefaultObjectUsecase {
         let PutObjectCommitResult { obj_info, put_versioned } = match put_commit_result {
             Ok(Ok(result)) => result,
             Ok(Err(err)) => {
+                let result: S3Result<S3Response<PutObjectOutput>> = Err(err);
                 put_request_guard.finish_err();
-                return Err(fail_put_object(completion, err));
+                let _ = helper.complete(&result);
+                return result;
             }
             Err(err) => {
-                put_request_guard.finish_err();
-                return Err(fail_put_object(
-                    completion,
-                    S3Error::with_message(S3ErrorCode::InternalError, format!("put object commit owner task failed: {err}")),
+                let result: S3Result<S3Response<PutObjectOutput>> = Err(S3Error::with_message(
+                    S3ErrorCode::InternalError,
+                    format!("put object commit owner task failed: {err}"),
                 ));
+                put_request_guard.finish_err();
+                let _ = helper.complete(&result);
+                return result;
             }
         };
 
-        completion = completion.object(obj_info.clone());
-        if let Some(version_id) = obj_info.version_id {
-            completion = completion.version_id(version_id.to_string());
+        let raw_version = obj_info.version_id.map(|v| v.to_string());
+
+        helper = helper.object(obj_info.clone());
+        if let Some(version_id) = &raw_version {
+            helper = helper.version_id(version_id.clone());
         }
 
-        Ok(PutObjectCommitted {
-            obj_info,
-            put_versioned,
-            effective_sse,
-            effective_kms_key_id,
-            sse_customer_algorithm,
-            sse_customer_key_md5,
-            put_extra_checksum_headers,
-            completion,
-            put_request_guard,
-            bucket,
-            key,
-            start_time,
-            size,
-            use_zero_copy_eager_put_path,
+        let put_version = if put_versioned { raw_version } else { None };
+
+        let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag));
+
+        let expiration = resolve_put_object_expiration(&bucket, &obj_info).await;
+
+        let mut checksums = PutObjectChecksums {
+            crc32: input.checksum_crc32,
+            crc32c: input.checksum_crc32c,
+            sha1: input.checksum_sha1,
+            sha256: input.checksum_sha256,
+            crc64nvme: input.checksum_crc64nvme,
+        };
+        apply_trailing_checksums(
+            input.checksum_algorithm.as_ref().map(|a| a.as_str()),
+            &req.trailing_headers,
+            &mut checksums,
+        );
+
+        let output = PutObjectOutput {
+            e_tag,
+            server_side_encryption: effective_sse,
+            sse_customer_algorithm: sse_customer_algorithm.clone(),
+            sse_customer_key_md5: sse_customer_key_md5.clone(),
+            ssekms_key_id: effective_kms_key_id,
+            expiration,
+            checksum_crc32: checksums.crc32,
+            checksum_crc32c: checksums.crc32c,
+            checksum_sha1: checksums.sha1,
+            checksum_sha256: checksums.sha256,
+            checksum_crc64nvme: checksums.crc64nvme,
+            version_id: put_version,
+            ..Default::default()
+        };
+
+        // For browser-based POST uploads (multipart/form-data), response status/body handling
+        // is decided by s3s PostObject serializer (success_action_status / redirect semantics).
+
+        let response_build_stage_start = put_stage_metrics_enabled.then(Instant::now);
+        let mut response = S3Response::new(output);
+        // Echo XXHash3/64/128 / SHA-512 checksums that s3s PutObjectOutput has no typed
+        // field for (#1256).
+        inject_additional_checksum_headers(&mut response.headers, &put_extra_checksum_headers);
+        rustfs_io_metrics::record_put_object_stage_duration_from("app_response_build", response_build_stage_start);
+        let result = Ok(response);
+        let _ = helper.complete(&result);
+
+        // Record PutObject metrics via zero-copy-metrics
+        {
+            let duration_ms = start_time.elapsed().as_millis() as f64;
+            rustfs_io_metrics::record_put_object(
+                duration_ms,
+                size,
+                use_zero_copy_eager_put_path, // Track if zero-copy was enabled
+            );
+        }
+
+        debug!(
+            target: "rustfs::app::object_usecase",
+            component = "app",
+            subsystem = "object",
+            bucket = %bucket,
+            key = %key,
             concurrent_put_requests,
             buffer_size,
-        })
+            "PutObject request completed"
+        );
+
+        put_request_guard.finish_ok();
+
+        result
     }
 }
 

rustfs/src/protocols/client.rs

@@ -163,7 +163,8 @@ fn build_object_uri(bucket: &str, key: &str, query: &[(&str, Option<&str>)]) ->
 struct RequestParams<'a> {
     bucket: Option<String>,
     object: Option<String>,
-    credentials: &'a rustfs_credentials::Credentials,
+    access_key: &'a str,
+    secret_key: &'a str,
 }
 
 /// Protocol storage client that implements the StorageBackend trait
@@ -180,22 +181,39 @@ impl ProtocolStorageClient {
     }
 
     /// Create a proper S3Request with ReqInfo extension for authorization
-    fn create_request<T>(input: T, method: Method, uri: http::Uri, params: RequestParams<'_>) -> S3Result<S3Request<T>> {
+    async fn create_request<T>(
+        &self,
+        input: T,
+        method: Method,
+        uri: http::Uri,
+        params: RequestParams<'_>,
+    ) -> S3Result<S3Request<T>> {
         let mut extensions = http::Extensions::default();
 
         let is_owner = if let Some(global_cred) = current_action_credentials() {
-            params.credentials.access_key == global_cred.access_key
+            params.access_key == global_cred.access_key
         } else {
             false
         };
 
         let credentials = Some(s3s::auth::Credentials {
-            access_key: params.credentials.access_key.clone(),
-            secret_key: params.credentials.secret_key.clone().into(),
+            access_key: params.access_key.to_string(),
+            secret_key: params.secret_key.to_string().into(),
         });
 
         extensions.insert(ReqInfo {
-            cred: Some(params.credentials.clone()),
+            cred: Some(rustfs_credentials::Credentials {
+                access_key: params.access_key.to_string(),
+                secret_key: params.secret_key.to_string(),
+                session_token: String::new(),
+                expiration: None,
+                status: String::new(),
+                parent_user: String::new(),
+                groups: None,
+                claims: None,
+                name: None,
+                description: None,
+            }),
             is_owner,
             bucket: params.bucket,
             object: params.object,
@@ -229,7 +247,8 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         &self,
         bucket: &str,
         key: &str,
-        credentials: &rustfs_credentials::Credentials,
+        access_key: &str,
+        secret_key: &str,
         start_pos: Option<u64>,
     ) -> Result<GetObjectOutput, Self::Error> {
         trace_protocol_request("get_object", Some(bucket), Some(key));
@@ -260,16 +279,19 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         })?;
 
         let uri = build_object_uri(bucket, key, &[])?;
-        let req = Self::create_request(
-            input,
-            Method::GET,
-            uri,
-            RequestParams {
-                bucket: Some(bucket.to_string()),
-                object: Some(key.to_string()),
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::GET,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket.to_string()),
+                    object: Some(key.to_string()),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
 
         match self.fs.get_object(req).await {
             Ok(response) => Ok(response.output),
@@ -280,7 +302,8 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn put_object(
         &self,
         input: PutObjectInput,
-        credentials: &rustfs_credentials::Credentials,
+        access_key: &str,
+        secret_key: &str,
     ) -> Result<PutObjectOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
@@ -307,16 +330,19 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
             }
         }
 
-        let req = Self::create_request(
-            input,
-            Method::PUT,
-            uri,
-            RequestParams {
-                bucket: Some(bucket),
-                object: Some(key),
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::PUT,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket),
+                    object: Some(key),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
         let req = S3Request { headers, ..req };
 
         match self.fs.put_object(req).await {
@@ -329,7 +355,8 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         &self,
         bucket: &str,
         key: &str,
-        credentials: &rustfs_credentials::Credentials,
+        access_key: &str,
+        secret_key: &str,
     ) -> Result<DeleteObjectOutput, Self::Error> {
         trace_protocol_request("delete_object", Some(bucket), Some(key));
 
@@ -342,16 +369,19 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
             })?;
 
         let uri = build_object_uri(bucket, key, &[])?;
-        let req = Self::create_request(
-            input,
-            Method::DELETE,
-            uri,
-            RequestParams {
-                bucket: Some(bucket.to_string()),
-                object: Some(key.to_string()),
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::DELETE,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket.to_string()),
+                    object: Some(key.to_string()),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
 
         match self.fs.delete_object(req).await {
             Ok(response) => Ok(response.output),
@@ -363,7 +393,8 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         &self,
         bucket: &str,
         key: &str,
-        credentials: &rustfs_credentials::Credentials,
+        access_key: &str,
+        secret_key: &str,
     ) -> Result<HeadObjectOutput, Self::Error> {
         trace_protocol_request("head_object", Some(bucket), Some(key));
 
@@ -376,16 +407,19 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
             })?;
 
         let uri = build_object_uri(bucket, key, &[])?;
-        let req = Self::create_request(
-            input,
-            Method::HEAD,
-            uri,
-            RequestParams {
-                bucket: Some(bucket.to_string()),
-                object: Some(key.to_string()),
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::HEAD,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket.to_string()),
+                    object: Some(key.to_string()),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
 
         match self.fs.head_object(req).await {
             Ok(response) => Ok(response.output),
@@ -393,11 +427,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         }
     }
 
-    async fn head_bucket(
-        &self,
-        bucket: &str,
-        credentials: &rustfs_credentials::Credentials,
-    ) -> Result<HeadBucketOutput, Self::Error> {
+    async fn head_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<HeadBucketOutput, Self::Error> {
         trace_protocol_request("head_bucket", Some(bucket), None);
 
         let input = HeadBucketInput::builder().bucket(bucket.to_string()).build().map_err(|e| {
@@ -405,16 +435,19 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         })?;
 
         let uri = build_bucket_uri(bucket, &[])?;
-        let req = Self::create_request(
-            input,
-            Method::HEAD,
-            uri,
-            RequestParams {
-                bucket: Some(bucket.to_string()),
-                object: None,
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::HEAD,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket.to_string()),
+                    object: None,
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
 
         match self.fs.head_bucket(req).await {
             Ok(response) => Ok(response.output),
@@ -425,22 +458,26 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn list_objects_v2(
         &self,
         input: ListObjectsV2Input,
-        credentials: &rustfs_credentials::Credentials,
+        access_key: &str,
+        secret_key: &str,
     ) -> Result<ListObjectsV2Output, Self::Error> {
         trace_protocol_request("list_objects_v2", Some(&input.bucket), None);
 
         let bucket = input.bucket.clone();
         let uri = build_bucket_uri(&bucket, &[("list-type", Some("2"))])?;
-        let req = Self::create_request(
-            input,
-            Method::GET,
-            uri,
-            RequestParams {
-                bucket: Some(bucket),
-                object: None,
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::GET,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket),
+                    object: None,
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
 
         match self.fs.list_objects_v2(req).await {
             Ok(response) => Ok(response.output),
@@ -448,13 +485,13 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         }
     }
 
-    async fn list_buckets(&self, credentials: &rustfs_credentials::Credentials) -> Result<ListBucketsOutput, Self::Error> {
+    async fn list_buckets(&self, access_key: &str, secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
             component = LOG_COMPONENT_PROTOCOLS,
             subsystem = LOG_SUBSYSTEM_STORAGE_CLIENT,
             operation = "list_buckets",
-            access_key = %MaskedAccessKey(&credentials.access_key),
+            access_key = %MaskedAccessKey(access_key),
             "Protocol storage client request"
         );
 
@@ -462,16 +499,19 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
             s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build ListBucketsInput: {}", e))
         })?;
 
-        let req = Self::create_request(
-            input,
-            Method::GET,
-            http::Uri::from_static("/"),
-            RequestParams {
-                bucket: None,
-                object: None,
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::GET,
+                http::Uri::from_static("/"),
+                RequestParams {
+                    bucket: None,
+                    object: None,
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
 
         match self.fs.list_buckets(req).await {
             Ok(response) => Ok(response.output),
@@ -502,11 +542,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         self.fs.list_buckets(request).await.map(|response| response.output)
     }
 
-    async fn create_bucket(
-        &self,
-        bucket: &str,
-        credentials: &rustfs_credentials::Credentials,
-    ) -> Result<CreateBucketOutput, Self::Error> {
+    async fn create_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<CreateBucketOutput, Self::Error> {
         trace_protocol_request("create_bucket", Some(bucket), None);
 
         let input = CreateBucketInput::builder().bucket(bucket.to_string()).build().map_err(|e| {
@@ -514,16 +550,19 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         })?;
 
         let uri = build_bucket_uri(bucket, &[])?;
-        let req = Self::create_request(
-            input,
-            Method::PUT,
-            uri,
-            RequestParams {
-                bucket: Some(bucket.to_string()),
-                object: None,
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::PUT,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket.to_string()),
+                    object: None,
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
 
         match self.fs.create_bucket(req).await {
             Ok(response) => Ok(response.output),
@@ -535,7 +574,8 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         &self,
         bucket: &str,
         key: &str,
-        credentials: &rustfs_credentials::Credentials,
+        access_key: &str,
+        secret_key: &str,
         start_pos: u64,
         length: u64,
     ) -> Result<GetObjectOutput, Self::Error> {
@@ -567,16 +607,19 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
             })?;
 
         let uri = build_object_uri(bucket, key, &[])?;
-        let req = Self::create_request(
-            input,
-            Method::GET,
-            uri,
-            RequestParams {
-                bucket: Some(bucket.to_string()),
-                object: Some(key.to_string()),
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::GET,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket.to_string()),
+                    object: Some(key.to_string()),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
 
         match self.fs.get_object(req).await {
             Ok(response) => Ok(response.output),
@@ -587,7 +630,8 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn copy_object(
         &self,
         input: CopyObjectInput,
-        credentials: &rustfs_credentials::Credentials,
+        access_key: &str,
+        secret_key: &str,
     ) -> Result<CopyObjectOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
@@ -603,16 +647,19 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         let key = input.key.clone();
         let uri = build_object_uri(&bucket, &key, &[])?;
 
-        let req = Self::create_request(
-            input,
-            Method::PUT,
-            uri,
-            RequestParams {
-                bucket: Some(bucket),
-                object: Some(key),
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::PUT,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket),
+                    object: Some(key),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
 
         match self.fs.copy_object(req).await {
             Ok(response) => Ok(response.output),
@@ -620,11 +667,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         }
     }
 
-    async fn delete_bucket(
-        &self,
-        bucket: &str,
-        credentials: &rustfs_credentials::Credentials,
-    ) -> Result<DeleteBucketOutput, Self::Error> {
+    async fn delete_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<DeleteBucketOutput, Self::Error> {
         trace_protocol_request("delete_bucket", Some(bucket), None);
 
         let input = DeleteBucketInput::builder().bucket(bucket.to_string()).build().map_err(|e| {
@@ -632,16 +675,19 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         })?;
 
         let uri = build_bucket_uri(bucket, &[])?;
-        let req = Self::create_request(
-            input,
-            Method::DELETE,
-            uri,
-            RequestParams {
-                bucket: Some(bucket.to_string()),
-                object: None,
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::DELETE,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket.to_string()),
+                    object: None,
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
 
         match self.fs.delete_bucket(req).await {
             Ok(response) => Ok(response.output),
@@ -652,7 +698,8 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn create_multipart_upload(
         &self,
         input: CreateMultipartUploadInput,
-        credentials: &rustfs_credentials::Credentials,
+        access_key: &str,
+        secret_key: &str,
     ) -> Result<CreateMultipartUploadOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
@@ -668,16 +715,19 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         let key = input.key.clone();
         let uri = build_object_uri(&bucket, &key, &[("uploads", None)])?;
 
-        let req = Self::create_request(
-            input,
-            Method::POST,
-            uri,
-            RequestParams {
-                bucket: Some(bucket),
-                object: Some(key),
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::POST,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket),
+                    object: Some(key),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
 
         match self.fs.create_multipart_upload(req).await {
             Ok(response) => Ok(response.output),
@@ -688,7 +738,8 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn upload_part(
         &self,
         input: UploadPartInput,
-        credentials: &rustfs_credentials::Credentials,
+        access_key: &str,
+        secret_key: &str,
     ) -> Result<UploadPartOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
@@ -735,16 +786,19 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
             }
         }
 
-        let req = Self::create_request(
-            input,
-            Method::PUT,
-            uri,
-            RequestParams {
-                bucket: Some(bucket),
-                object: Some(key),
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::PUT,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket),
+                    object: Some(key),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
         let req = S3Request { headers, ..req };
 
         match self.fs.upload_part(req).await {
@@ -756,7 +810,8 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn complete_multipart_upload(
         &self,
         input: CompleteMultipartUploadInput,
-        credentials: &rustfs_credentials::Credentials,
+        access_key: &str,
+        secret_key: &str,
     ) -> Result<CompleteMultipartUploadOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
@@ -773,16 +828,19 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         let upload_id = input.upload_id.clone();
         let uri = build_object_uri(&bucket, &key, &[("uploadId", Some(upload_id.as_str()))])?;
 
-        let req = Self::create_request(
-            input,
-            Method::POST,
-            uri,
-            RequestParams {
-                bucket: Some(bucket),
-                object: Some(key),
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::POST,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket),
+                    object: Some(key),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
 
         match self.fs.complete_multipart_upload(req).await {
             Ok(response) => Ok(response.output),
@@ -793,7 +851,8 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn abort_multipart_upload(
         &self,
         input: AbortMultipartUploadInput,
-        credentials: &rustfs_credentials::Credentials,
+        access_key: &str,
+        secret_key: &str,
     ) -> Result<AbortMultipartUploadOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
@@ -811,16 +870,19 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         let upload_id = input.upload_id.clone();
         let uri = build_object_uri(&bucket, &key, &[("uploadId", Some(upload_id.as_str()))])?;
 
-        let req = Self::create_request(
-            input,
-            Method::DELETE,
-            uri,
-            RequestParams {
-                bucket: Some(bucket),
-                object: Some(key),
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::DELETE,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket),
+                    object: Some(key),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
 
         match self.fs.abort_multipart_upload(req).await {
             Ok(response) => Ok(response.output),
@@ -831,7 +893,8 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn upload_part_copy(
         &self,
         input: UploadPartCopyInput,
-        credentials: &rustfs_credentials::Credentials,
+        access_key: &str,
+        secret_key: &str,
     ) -> Result<UploadPartCopyOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
@@ -858,16 +921,19 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
             ],
         )?;
 
-        let req = Self::create_request(
-            input,
-            Method::PUT,
-            uri,
-            RequestParams {
-                bucket: Some(bucket),
-                object: Some(key),
-                credentials,
-            },
-        )?;
+        let req = self
+            .create_request(
+                input,
+                Method::PUT,
+                uri,
+                RequestParams {
+                    bucket: Some(bucket),
+                    object: Some(key),
+                    access_key,
+                    secret_key,
+                },
+            )
+            .await?;
 
         match self.fs.upload_part_copy(req).await {
             Ok(response) => Ok(response.output),
@@ -879,47 +945,6 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
 #[cfg(test)]
 mod tests {
     use super::*;
-    use rustfs_credentials::{IAM_POLICY_CLAIM_NAME_SA, INHERITED_POLICY_TYPE};
-
-    #[test]
-    fn create_request_preserves_authenticated_service_account_identity() {
-        let claims = std::collections::HashMap::from([
-            ("parent".to_string(), serde_json::json!("alice")),
-            (IAM_POLICY_CLAIM_NAME_SA.to_string(), serde_json::json!(INHERITED_POLICY_TYPE)),
-        ]);
-        let credentials = rustfs_credentials::Credentials {
-            access_key: "service-account".to_string(),
-            secret_key: "secret".to_string(),
-            session_token: "signed-service-account-token".to_string(),
-            parent_user: "alice".to_string(),
-            groups: Some(vec!["developers".to_string()]),
-            claims: Some(claims.clone()),
-            ..Default::default()
-        };
-
-        let request = ProtocolStorageClient::create_request(
-            ListObjectsV2Input::default(),
-            Method::GET,
-            http::Uri::from_static("/bucket?list-type=2"),
-            RequestParams {
-                bucket: Some("bucket".to_string()),
-                object: None,
-                credentials: &credentials,
-            },
-        )
-        .expect("request should build");
-        let request_info = request.extensions.get::<ReqInfo>().expect("request info should be present");
-        let copied = request_info.cred.as_ref().expect("credentials should be present");
-
-        assert_eq!(copied.access_key, credentials.access_key);
-        assert_eq!(copied.secret_key, credentials.secret_key);
-        assert_eq!(copied.session_token, credentials.session_token);
-        assert_eq!(copied.parent_user, credentials.parent_user);
-        assert_eq!(copied.groups, credentials.groups);
-        assert_eq!(copied.claims, Some(claims));
-        assert!(copied.is_service_account());
-    }
-
     #[cfg(feature = "webdav")]
     #[test]
     fn request_extensions_preserve_authenticated_identity_and_source_ip() {

rustfs/src/storage/access.rs

@@ -4361,15 +4361,10 @@ mod tests {
         apply_bucket_generation_guard(&req, &bucket, &mut opts).expect("apply the RestoreObject authorization guard");
         assert_eq!(opts.expected_bucket_incarnation_id, Some(authorized_incarnation_id));
 
-        // The RestoreObject usecase future is large enough that, inlined into
-        // this test body, the test thread's 2 MiB stack sits within a few KiB
-        // of overflowing on Linux; heap-pin it so unrelated growth in bucket
-        // metadata futures cannot tip the test over.
-        let err = Box::pin(
-            crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(app_context)).execute_restore_object(req),
-        )
-        .await
-        .expect_err("the old RestoreObject authorization must not reach the recreated bucket");
+        let err = crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(app_context))
+            .execute_restore_object(req)
+            .await
+            .expect_err("the old RestoreObject authorization must not reach the recreated bucket");
         assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
         store
             .delete_bucket(&bucket, &DeleteBucketOptions::default())

rustfs/src/storage/rpc/node_service.rs

@@ -181,10 +181,11 @@ fn remove_heal_control_replay(
 
 static HEAL_CONTROL_REPLAY_CACHE: OnceLock<tokio::sync::Mutex<HashMap<String, Arc<HealControlReplayEntry>>>> = OnceLock::new();
 static NODE_CAPABILITY_SERVER_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
-// v3 additionally promises the v6 tier-delete dispatch-manifest policy. The
+// v3 additionally promises the v6 tier-delete dispatch-manifest policy; v4
+// promises the sticky per-target decommission capacity fence. The
 // existing periodic topology probe carries both capabilities so normal object
 // operations do not add another peer RPC.
-const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 3;
+const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 4;
 
 fn admit_heal_control_replay(
     replay_cache: &mut HashMap<String, Arc<HealControlReplayEntry>>,
@@ -3770,7 +3771,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn cross_pool_fence_probe_authenticates_supported_v3_state() {
+    async fn cross_pool_fence_probe_authenticates_supported_v4_state() {
         let _ = rustfs_credentials::set_global_rpc_secret("cross-pool-fence-node-service-test-secret".to_string());
         let endpoints = heal_control_test_endpoints_with_coordinator("node-0", true);
         assert!(

scripts/error-other-format-baseline.txt

@@ -5,6 +5,7 @@
 # scripts/check_error_other_format_ratchet.sh --update-baseline. A PR that
 # raises a count or adds a file is introducing a new quorum-bucketing hazard
 # and must carry an explicit exemption rationale in its description.
+2|crates/ecstore/src/bucket/bucket_target_sys.rs
 4|crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
 3|crates/ecstore/src/bucket/lifecycle/durable_namespace.rs
 2|crates/ecstore/src/bucket/lifecycle/metadata_boundary.rs
@@ -26,7 +27,7 @@
 6|crates/ecstore/src/config/com.rs
 14|crates/ecstore/src/config/storageclass.rs
 182|crates/ecstore/src/core/pools.rs
-8|crates/ecstore/src/data_movement/mod.rs
+7|crates/ecstore/src/data_movement/mod.rs
 2|crates/ecstore/src/data_usage/local_snapshot.rs
 12|crates/ecstore/src/data_usage/mod.rs
 5|crates/ecstore/src/disk/local.rs
@@ -69,5 +70,5 @@
 12|crates/ecstore/src/store/init.rs
 2|crates/ecstore/src/store/init_format.rs
 3|crates/ecstore/src/store/multipart.rs
-7|crates/ecstore/src/store/object.rs
+6|crates/ecstore/src/store/object.rs
 5|crates/ecstore/src/store/rebalance/support.rs