refactor(object): extract internal put entry and ODM provenance keys (#7071)

* feat(utils): add on-demand migration provenance metadata suffixes

* refactor(object): extract internal put entry from the S3 PutObject path

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=ef914ec0b8daa9c2c5e52f501d339914662f42d6f6ed9d33877d56b97adf16f9
+sha256-darwin=9dccb0cd537cf79ae70c1c20e8281d36d03f2f09f81142a5341e26e3dc18709d
 sha256-linux=a8a816d7bb0e7cb5632b1863b33794bcb9fc7e765f150aa5e1bf16518e28dfb4

Cargo.lock

@@ -10449,6 +10449,7 @@ dependencies = [
  "base64-simd",
  "bytes",
  "crc-fast",
+ "criterion",
  "faster-hex",
  "futures",
  "hex-simd",
@@ -10460,6 +10461,7 @@ dependencies = [
  "md-5 0.11.0",
  "minlz",
  "pin-project-lite",
+ "proptest",
  "rand 0.10.2",
  "reqwest",
  "rustfs-config",

crates/e2e_test/src/fake_s3_target/README.md

@@ -1,11 +1,15 @@
 # Programmable fake S3 target
 
-This module is the shared failure-injection boundary for replication end-to-end tests. It runs an in-process, path-style S3 endpoint backed by `s3s`; no production crate depends on it.
+This module is the shared failure-injection boundary for replication end-to-end tests and the programmable external source for on-demand-migration (ODM) tests. It runs an in-process, path-style S3 endpoint backed by `s3s`; no production crate depends on it.
 
 `FakeS3Target::start()` creates the listener. Add target buckets with `create_bucket`, point a RustFS remote target at `address()`, use `FAKE_ACCESS_KEY` / `FAKE_SECRET_KEY`, then enqueue per-operation faults with `inject`. Faults for one operation are consumed in FIFO order and do not consume faults queued for another operation. A fault is consumed only after `s3s` verifies the full request signature, so anonymous, other-access-key, and bad-signature traffic cannot disturb a script.
 
-Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, Get/Put/Delete ObjectTagging (tags live per version; Put replaces the whole set, Delete clears it), and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
+Supported data operations are HeadBucket, GetBucketVersioning, ListObjectsV2, PUT/GET/HEAD/DELETE Object, Get/Put/Delete ObjectTagging (tags live per version; Put replaces the whole set, Delete clears it), and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets created with `create_bucket` are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
 
-Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions. Each record also journals a `ProxyHeaderSnapshot` — the read-proxy anti-loop marker (`x-{rustfs,minio}-source-proxy-request`), the replication-check exemption header, and the client SSE-C header family (algorithm and key-MD5 values; for the key itself only its presence) — so proxy tests can pin the exact wire contract.
+`create_bucket_with_mode(name, BucketMode::Unversioned)` models a plain migration source: PUT overwrites in place, DELETE removes the key without a delete marker, GetBucketVersioning reports no status, and no `x-amz-version-id` is returned by PUT, GET, HEAD, tagging, or multipart completion. The only `versionId` such a bucket accepts is `null`; any other value is rejected with `InvalidArgument`. The mode is fixed at creation.
 
-The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type at 1 KiB. A PUT or uploaded part is capped at 64 MiB; a completed multipart object and all stored object/part data are capped at 128 MiB. Body drain, body-permit waits, delay, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound.
+ListObjectsV2 lists current versions only (a key whose newest version is a delete marker is hidden) in byte order and supports `prefix`, `delimiter`, `max-keys` (clamped to 1000), `start-after`, and `continuation-token`; common prefixes count toward `max-keys`, `IsTruncated` / `NextContinuationToken` / `KeyCount` follow S3, and continuation tokens are opaque. `encoding-type` and `fetch-owner` are accepted but ignored, and ListObjects (v1) is not implemented. GET and HEAD honor `Range` in the `bytes=first-last`, `bytes=first-`, and `bytes=-suffix` forms with a 206 status, exact `Content-Range`, and `Accept-Ranges: bytes`; unsatisfiable ranges answer 416 `InvalidRange` with `Content-Range: bytes */<length>`. PUT and CreateMultipartUpload accept `Content-Type`, `Content-Encoding`, `Content-Disposition`, `Content-Language`, `Cache-Control`, `Expires`, and `x-amz-meta-*` (names stored lowercased), and HEAD/GET replay them verbatim together with `Last-Modified` and the ETag (hex MD5 for single PUTs, `<md5-of-part-md5s>-<parts>` for multipart objects). `put_seed_object` stores an object directly, bypassing the wire, the fault script, and the journal, so a source can be seeded without polluting the assertions a scenario later makes.
+
+Fault actions cover HTTP 401/403/503 responses (`Status`), any 4xx/5xx status paired with the matching S3 error code (`ResponseStatus`), pre-dispatch delay, holding a fully computed successful response before its first byte (`Stall`), connection abort when a logical request-body threshold is reached, GetObject bodies cut off after N bytes while `Content-Length` announces the full size (`TruncateBodyAt`), streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions and `count_requests(operation, key)` counts entries for one exact key. Each record journals the `Range` and `User-Agent` request headers, the ListObjectsV2 `prefix` and `continuation-token` query values, and a `ProxyHeaderSnapshot` — the read-proxy anti-loop marker (`x-{rustfs,minio}-source-proxy-request`), the replication-check exemption header, and the client SSE-C header family (algorithm and key-MD5 values; for the key itself only its presence) — so proxy tests can pin the exact wire contract.
+
+The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type and each standard object header at 1 KiB. By default a PUT or uploaded part is capped at 64 MiB and a completed multipart object and all stored object/part data are capped at 128 MiB; `FakeS3Target::start_with_options(FakeS3TargetOptions { max_object_bytes })` raises the object cap up to 256 MiB, and the total budget then becomes twice the object cap (never below 128 MiB). Body drain, body-permit waits, delay, stall, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound.

crates/e2e_test/src/lib.rs

@@ -23,10 +23,17 @@ pub mod common;
 #[cfg(test)]
 pub mod chaos;
 
-// Programmable S3 target for replication failure-path tests (backlog#1147 repl-8).
+// Programmable S3 target for replication failure-path tests (backlog#1147 repl-8)
+// and on-demand-migration source scenarios (backlog#2151).
 #[cfg(test)]
 pub mod fake_s3_target;
 
+// On-demand migration (backlog#2147): shared two-server environment, admin
+// wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
+// added by later ODM tasks.
+#[cfg(test)]
+pub mod on_demand_migration;
+
 // Socket-level network fault-injection proxy for black-box cluster tests
 // (backlog#1325 network fault-injection block): latency / blackhole / one-way
 // partition on the wire between nodes. Serves #1312/#1319 (lock-plane one-way

crates/e2e_test/src/on_demand_migration/common.rs

@@ -0,0 +1,452 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Shared environment for on-demand migration (ODM) end-to-end tests.
+//!
+//! [`OdmTestEnv`] pairs one RustFS server under test with one in-process
+//! programmable S3 source ([`FakeS3Target`]). Admin calls target the route
+//! convention fixed by the tracking plan
+//! (`/rustfs/admin/v3/on-demand-migration/{bucket}`, JSON bodies); the
+//! server side lands with ODM-07, so until then the wrappers compile but are
+//! not exercised by the harness self-test.
+
+use crate::common::{RustFSTestEnvironment, signed_request};
+use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FakeS3TargetOptions, SeedMetadata};
+use aws_config::retry::RetryConfig;
+use aws_sdk_s3::Client;
+use aws_sdk_s3::config::{Credentials, Region};
+use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
+use bytes::Bytes;
+use serde::Serialize;
+use std::fmt;
+
+pub type BoxError = Box<dyn std::error::Error + Send + Sync>;
+
+/// Module switch the server reads at startup (`false` before GA). The harness
+/// turns it on so scenario tests exercise the feature without repeating it.
+pub const ODM_MODULE_SWITCH_ENV: &str = "RUSTFS_ON_DEMAND_MIGRATION_ENABLED";
+/// Admin route prefix; the bucket name is appended as one path segment.
+pub const ODM_ADMIN_ROUTE: &str = "/rustfs/admin/v3/on-demand-migration";
+/// Region the fake source is addressed with (it accepts any SigV4 region).
+pub const FAKE_SOURCE_REGION: &str = "us-east-1";
+
+/// Wire form of the bucket-level ODM configuration (ODM-01 model). Every
+/// field is public so a scenario can tweak one knob and serialize the rest
+/// with the documented defaults.
+#[derive(Debug, Clone, Serialize)]
+pub struct OdmSourceSpec {
+    pub version: u32,
+    pub enabled: bool,
+    pub source: OdmSource,
+    pub filter: OdmFilter,
+    pub policy: OdmPolicy,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct OdmSource {
+    pub provider: String,
+    pub endpoint: String,
+    pub region: String,
+    pub bucket: String,
+    pub path_style: String,
+    pub credentials: Option<OdmCredentials>,
+    pub tls: OdmTls,
+}
+
+#[derive(Clone, Serialize)]
+pub struct OdmCredentials {
+    pub access_key: String,
+    pub secret_key: String,
+    pub session_token: Option<String>,
+}
+
+impl fmt::Debug for OdmCredentials {
+    /// Test logs are captured into CI artifacts; keep the secret out of them.
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("OdmCredentials")
+            .field("access_key", &self.access_key)
+            .field("secret_key", &"REDACTED")
+            .field("session_token", &self.session_token.as_ref().map(|_| "REDACTED"))
+            .finish()
+    }
+}
+
+#[derive(Debug, Clone, Default, Serialize)]
+pub struct OdmTls {
+    pub skip_verify: bool,
+    pub ca_cert_pem: Option<String>,
+}
+
+#[derive(Debug, Clone, Default, Serialize)]
+pub struct OdmFilter {
+    pub prefix: Option<String>,
+    pub source_prefix: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct OdmPolicy {
+    pub head: String,
+    pub range_get: String,
+    pub source_error: String,
+    pub respect_local_delete_marker: bool,
+    pub preserve_etag: bool,
+    pub copy_tags: bool,
+    pub emit_events: bool,
+    pub negative_cache_ttl_secs: u64,
+    pub inline_max_bytes: u64,
+    pub multipart_part_size_bytes: u64,
+    pub max_concurrent_pulls: u32,
+    pub pull_queue_capacity: u32,
+    pub source_timeout: OdmSourceTimeout,
+    pub bandwidth_limit_bytes_per_sec: Option<u64>,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct OdmSourceTimeout {
+    pub connect_ms: u64,
+    pub first_byte_ms: u64,
+    pub idle_ms: u64,
+}
+
+impl Default for OdmPolicy {
+    /// The ODM-01 defaults verbatim.
+    fn default() -> Self {
+        Self {
+            head: "proxy".to_string(),
+            range_get: "serve_and_backfill".to_string(),
+            source_error: "propagate".to_string(),
+            respect_local_delete_marker: true,
+            preserve_etag: true,
+            copy_tags: false,
+            emit_events: true,
+            negative_cache_ttl_secs: 30,
+            inline_max_bytes: 16 * 1024 * 1024,
+            multipart_part_size_bytes: 64 * 1024 * 1024,
+            max_concurrent_pulls: 8,
+            pull_queue_capacity: 1024,
+            source_timeout: OdmSourceTimeout {
+                connect_ms: 5_000,
+                first_byte_ms: 15_000,
+                idle_ms: 30_000,
+            },
+            bandwidth_limit_bytes_per_sec: None,
+        }
+    }
+}
+
+impl OdmSourceSpec {
+    /// Enabled configuration pointing at a bucket on the fake source with the
+    /// fixture credentials, path-style addressing, and default policy.
+    pub fn for_fake_source(source: &FakeS3Target, source_bucket: impl Into<String>) -> Self {
+        Self::new(
+            "s3",
+            source.endpoint(),
+            FAKE_SOURCE_REGION,
+            source_bucket,
+            FAKE_ACCESS_KEY,
+            FAKE_SECRET_KEY,
+        )
+    }
+
+    /// Enabled configuration pointing at a bucket on a second RustFS server
+    /// (see [`start_source_rustfs`]).
+    pub fn for_rustfs_source(source: &RustFSTestEnvironment, source_bucket: impl Into<String>) -> Self {
+        Self::new(
+            "rustfs",
+            &source.url,
+            FAKE_SOURCE_REGION,
+            source_bucket,
+            &source.access_key,
+            &source.secret_key,
+        )
+    }
+
+    fn new(
+        provider: &str,
+        endpoint: &str,
+        region: &str,
+        source_bucket: impl Into<String>,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Self {
+        Self {
+            version: 1,
+            enabled: true,
+            source: OdmSource {
+                provider: provider.to_string(),
+                endpoint: endpoint.to_string(),
+                region: region.to_string(),
+                bucket: source_bucket.into(),
+                path_style: "path".to_string(),
+                credentials: Some(OdmCredentials {
+                    access_key: access_key.to_string(),
+                    secret_key: secret_key.to_string(),
+                    session_token: None,
+                }),
+                tls: OdmTls::default(),
+            },
+            filter: OdmFilter::default(),
+            policy: OdmPolicy::default(),
+        }
+    }
+
+    pub fn to_json(&self) -> serde_json::Value {
+        serde_json::to_value(self).expect("ODM source spec serializes")
+    }
+}
+
+/// Backfill job control (ODM-12 route shape).
+#[derive(Debug, Clone)]
+pub enum BackfillOp {
+    Start(BackfillRequest),
+    Cancel,
+    Status,
+}
+
+#[derive(Debug, Clone, Default, Serialize)]
+pub struct BackfillRequest {
+    pub prefix: Option<String>,
+    pub skip_existing: Option<String>,
+    pub dry_run: bool,
+}
+
+/// Status plus raw body of an admin call, so a scenario can assert on the
+/// HTTP status first and only then parse the JSON.
+#[derive(Debug, Clone)]
+pub struct AdminResponse {
+    pub status: u16,
+    pub body: String,
+}
+
+impl AdminResponse {
+    pub fn json(&self) -> Result<serde_json::Value, BoxError> {
+        Ok(serde_json::from_str(&self.body)?)
+    }
+}
+
+/// One object to seed into the source.
+#[derive(Clone)]
+pub struct SeedObject {
+    pub key: String,
+    pub body: Bytes,
+    pub metadata: SeedMetadata,
+}
+
+impl SeedObject {
+    pub fn new(key: impl Into<String>, body: impl Into<Bytes>) -> Self {
+        Self {
+            key: key.into(),
+            body: body.into(),
+            metadata: SeedMetadata::new(),
+        }
+    }
+
+    pub fn with_metadata(mut self, metadata: SeedMetadata) -> Self {
+        self.metadata = metadata;
+        self
+    }
+}
+
+/// RustFS under test plus its fake S3 source.
+pub struct OdmTestEnv {
+    pub rustfs: RustFSTestEnvironment,
+    pub source: FakeS3Target,
+    /// S3 client for the RustFS under test.
+    pub client: Client,
+}
+
+impl OdmTestEnv {
+    /// Start a fake source with default limits and a RustFS server with the
+    /// ODM module switch enabled.
+    pub async fn start() -> Result<Self, BoxError> {
+        Self::start_with_options(FakeS3TargetOptions::default()).await
+    }
+
+    pub async fn start_with_options(options: FakeS3TargetOptions) -> Result<Self, BoxError> {
+        let source = FakeS3Target::start_with_options(options).await?;
+        let mut rustfs = RustFSTestEnvironment::new().await?;
+        rustfs
+            .start_rustfs_server_with_env(vec![], &[(ODM_MODULE_SWITCH_ENV, "true")])
+            .await?;
+        let client = rustfs.create_s3_client();
+        Ok(Self { rustfs, source, client })
+    }
+
+    /// S3 client addressing the fake source directly, for assertions on the
+    /// source's own state. Retries are off so a scripted fault is consumed by
+    /// exactly the request the test issued.
+    pub fn source_client(&self) -> Client {
+        fake_source_client(&self.source)
+    }
+
+    /// Enabled ODM configuration for `source_bucket` on the fake source.
+    pub fn fake_source_spec(&self, source_bucket: impl Into<String>) -> OdmSourceSpec {
+        OdmSourceSpec::for_fake_source(&self.source, source_bucket)
+    }
+
+    /// `PUT /rustfs/admin/v3/on-demand-migration/{bucket}` with the JSON spec.
+    pub async fn configure_source(&self, bucket: &str, spec: &OdmSourceSpec) -> Result<AdminResponse, BoxError> {
+        self.admin(http::Method::PUT, &format!("/{bucket}"), Some(spec.to_json()))
+            .await
+    }
+
+    /// Same as [`Self::configure_source`] with `dry-run=true`: validate and
+    /// probe without persisting.
+    pub async fn validate_source(&self, bucket: &str, spec: &OdmSourceSpec) -> Result<AdminResponse, BoxError> {
+        self.admin(http::Method::PUT, &format!("/{bucket}?dry-run=true"), Some(spec.to_json()))
+            .await
+    }
+
+    /// `GET .../{bucket}`: redacted configuration, 404 when unconfigured.
+    pub async fn get_config(&self, bucket: &str) -> Result<AdminResponse, BoxError> {
+        self.admin(http::Method::GET, &format!("/{bucket}"), None).await
+    }
+
+    /// `DELETE .../{bucket}`: remove the configuration (idempotent).
+    pub async fn disable(&self, bucket: &str) -> Result<AdminResponse, BoxError> {
+        self.admin(http::Method::DELETE, &format!("/{bucket}"), None).await
+    }
+
+    /// `GET .../{bucket}/status`: runtime snapshot.
+    pub async fn status(&self, bucket: &str) -> Result<AdminResponse, BoxError> {
+        self.admin(http::Method::GET, &format!("/{bucket}/status"), None).await
+    }
+
+    /// Backfill control: `POST .../{bucket}/backfill?op=start|cancel` or
+    /// `GET .../{bucket}/backfill` for the checkpoint.
+    pub async fn backfill(&self, bucket: &str, op: BackfillOp) -> Result<AdminResponse, BoxError> {
+        match op {
+            BackfillOp::Start(request) => {
+                self.admin(
+                    http::Method::POST,
+                    &format!("/{bucket}/backfill?op=start"),
+                    Some(serde_json::to_value(request)?),
+                )
+                .await
+            }
+            BackfillOp::Cancel => {
+                self.admin(http::Method::POST, &format!("/{bucket}/backfill?op=cancel"), None)
+                    .await
+            }
+            BackfillOp::Status => self.admin(http::Method::GET, &format!("/{bucket}/backfill"), None).await,
+        }
+    }
+
+    async fn admin(
+        &self,
+        method: http::Method,
+        path_and_query: &str,
+        body: Option<serde_json::Value>,
+    ) -> Result<AdminResponse, BoxError> {
+        let url = format!("{}{ODM_ADMIN_ROUTE}{path_and_query}", self.rustfs.url);
+        let body = body.map(|value| serde_json::to_vec(&value)).transpose()?;
+        let content_type = body.is_some().then_some("application/json");
+        let response = signed_request(method, &url, &self.rustfs.access_key, &self.rustfs.secret_key, body, content_type).await?;
+        Ok(AdminResponse {
+            status: response.status().as_u16(),
+            body: response.text().await?,
+        })
+    }
+
+    /// Store objects directly in the fake source (no wire traffic, no journal
+    /// entries). Returns the ETags in input order.
+    pub fn seed_source(&self, source_bucket: &str, objects: &[SeedObject]) -> Vec<String> {
+        objects
+            .iter()
+            .map(|object| {
+                self.source
+                    .put_seed_object(source_bucket, object.key.clone(), object.body.clone(), &object.metadata)
+            })
+            .collect()
+    }
+
+    /// Whether `key` is listed by the RustFS under test. Listing is served from
+    /// local state only, so this does not trigger a migration the way GET or
+    /// HEAD would.
+    pub async fn local_key_listed(&self, bucket: &str, key: &str) -> Result<bool, BoxError> {
+        let listed = self
+            .client
+            .list_objects_v2()
+            .bucket(bucket)
+            .prefix(key)
+            .max_keys(1)
+            .send()
+            .await?;
+        Ok(listed.contents().iter().any(|object| object.key() == Some(key)))
+    }
+
+    /// Panics unless `key` is stored locally with exactly `expected` bytes.
+    /// Presence is checked through listing first so a missing object fails
+    /// here instead of being pulled from the source by the GET.
+    pub async fn assert_local_present(&self, bucket: &str, key: &str, expected: &[u8]) {
+        assert!(
+            self.local_key_listed(bucket, key)
+                .await
+                .unwrap_or_else(|error| panic!("listing {bucket}/{key} failed: {error}")),
+            "{bucket}/{key} must be present locally"
+        );
+        let body = self
+            .client
+            .get_object()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .unwrap_or_else(|error| panic!("GET {bucket}/{key} failed: {error}"))
+            .body
+            .collect()
+            .await
+            .unwrap_or_else(|error| panic!("reading {bucket}/{key} failed: {error}"))
+            .into_bytes();
+        assert_eq!(body.as_ref(), expected, "{bucket}/{key} local content mismatch");
+    }
+
+    /// Panics if `key` is listed locally.
+    pub async fn assert_local_absent(&self, bucket: &str, key: &str) {
+        assert!(
+            !self
+                .local_key_listed(bucket, key)
+                .await
+                .unwrap_or_else(|error| panic!("listing {bucket}/{key} failed: {error}")),
+            "{bucket}/{key} must be absent locally"
+        );
+    }
+}
+
+/// S3 client for the fake source with retries disabled (see
+/// [`OdmTestEnv::source_client`]).
+pub fn fake_source_client(source: &FakeS3Target) -> Client {
+    let credentials = Credentials::new(FAKE_ACCESS_KEY, FAKE_SECRET_KEY, None, None, "odm-fake-source");
+    Client::from_conf(
+        aws_sdk_s3::Config::builder()
+            .credentials_provider(credentials)
+            .region(Region::new(FAKE_SOURCE_REGION))
+            .endpoint_url(source.endpoint())
+            .force_path_style(true)
+            .behavior_version_latest()
+            .retry_config(RetryConfig::standard().with_max_attempts(1))
+            .http_client(SmithyHttpClientBuilder::new().build_http())
+            .build(),
+    )
+}
+
+/// Start a second, fully independent RustFS process (own port, data
+/// directory, and default credentials) to act as a real S3 source. It is
+/// spawned the same way `reliant::tiering` starts its cold tier; the process
+/// is stopped and its directory removed when the returned environment drops.
+pub async fn start_source_rustfs() -> Result<RustFSTestEnvironment, BoxError> {
+    let mut source = RustFSTestEnvironment::new().await?;
+    source.start_rustfs_server_without_cleanup(vec![]).await?;
+    Ok(source)
+}

crates/e2e_test/src/on_demand_migration/harness_self_test.rs

@@ -0,0 +1,606 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Self-test of the ODM harness (rustfs/backlog#2151): the fake source's
+//! migration-facing surface (ListObjectsV2 paging, `Range`, unversioned
+//! buckets, metadata replay, fault actions) and the two-server environment.
+//! No ODM behavior is exercised here.
+
+use super::common::{OdmTestEnv, SeedObject, fake_source_client, start_source_rustfs};
+use crate::fake_s3_target::{BucketMode, FakeS3Target, FakeS3TargetOptions, FaultAction, Operation, SeedMetadata};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::primitives::{ByteStream, DateTime};
+use bytes::Bytes;
+use std::collections::BTreeSet;
+use std::time::{Duration, Instant};
+
+type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+
+const SOURCE_BUCKET: &str = "odm-source";
+
+/// Position-dependent payload so a misaligned range read is caught.
+fn payload(len: usize) -> Bytes {
+    (0..len).map(|index| (index % 251) as u8).collect::<Vec<u8>>().into()
+}
+
+async fn fake_source() -> Result<(FakeS3Target, Client), Box<dyn std::error::Error + Send + Sync>> {
+    let source = FakeS3Target::start().await?;
+    source.create_bucket(SOURCE_BUCKET);
+    let client = fake_source_client(&source);
+    Ok((source, client))
+}
+
+/// Full ListObjectsV2 traversal. Returns `(keys, common prefixes, pages)` and
+/// checks the page shape on the way: every page except the last is full and
+/// truncated, the last carries no continuation token.
+async fn list_all(
+    client: &Client,
+    prefix: Option<&str>,
+    delimiter: Option<&str>,
+    start_after: Option<&str>,
+    max_keys: i32,
+) -> Result<(Vec<String>, Vec<String>, usize), Box<dyn std::error::Error + Send + Sync>> {
+    let mut keys = Vec::new();
+    let mut prefixes = Vec::new();
+    let mut pages = 0usize;
+    let mut token: Option<String> = None;
+    loop {
+        let page = client
+            .list_objects_v2()
+            .bucket(SOURCE_BUCKET)
+            .set_prefix(prefix.map(str::to_string))
+            .set_delimiter(delimiter.map(str::to_string))
+            .set_start_after(start_after.map(str::to_string))
+            .max_keys(max_keys)
+            .set_continuation_token(token.clone())
+            .send()
+            .await?;
+        pages += 1;
+        let page_keys: Vec<String> = page
+            .contents()
+            .iter()
+            .filter_map(|object| object.key().map(str::to_string))
+            .collect();
+        let page_prefixes: Vec<String> = page
+            .common_prefixes()
+            .iter()
+            .filter_map(|common| common.prefix().map(str::to_string))
+            .collect();
+        let entries = page_keys.len() + page_prefixes.len();
+        assert_eq!(page.key_count(), Some(entries as i32), "KeyCount must count keys and prefixes");
+        assert_eq!(page.continuation_token(), token.as_deref(), "the request token must be echoed");
+        keys.extend(page_keys);
+        prefixes.extend(page_prefixes);
+        if page.is_truncated() == Some(true) {
+            assert_eq!(entries as i32, max_keys, "every truncated page must be full");
+            token = Some(
+                page.next_continuation_token()
+                    .expect("truncated page must carry a continuation token")
+                    .to_string(),
+            );
+        } else {
+            assert!(page.next_continuation_token().is_none(), "final page must not carry a token");
+            return Ok((keys, prefixes, pages));
+        }
+    }
+}
+
+#[tokio::test]
+async fn fake_source_list_objects_v2_paginates_with_delimiter() -> TestResult {
+    let (source, client) = fake_source().await?;
+    let mut expected_keys = BTreeSet::new();
+    for directory in 0..30 {
+        for file in 0..30 {
+            expected_keys.insert(format!("d{directory:02}/k{file:03}"));
+        }
+    }
+    for index in 0..100 {
+        expected_keys.insert(format!("top-{index:03}"));
+    }
+    assert_eq!(expected_keys.len(), 1000);
+    for key in &expected_keys {
+        source.put_seed_object(SOURCE_BUCKET, key.clone(), Bytes::from(key.clone()), &SeedMetadata::new());
+    }
+    // A key whose current version is a delete marker must stay hidden.
+    client
+        .put_object()
+        .bucket(SOURCE_BUCKET)
+        .key("hidden/marker")
+        .body(ByteStream::from_static(b"gone"))
+        .send()
+        .await?;
+    client
+        .delete_object()
+        .bucket(SOURCE_BUCKET)
+        .key("hidden/marker")
+        .send()
+        .await?;
+    let expected_sorted: Vec<String> = expected_keys.iter().cloned().collect();
+    let expected_prefixes: Vec<String> = (0..30).map(|directory| format!("d{directory:02}/")).collect();
+    let expected_top: Vec<String> = (0..100).map(|index| format!("top-{index:03}")).collect();
+
+    // Flat traversal in byte order, 1000 keys in pages of 7.
+    let (keys, prefixes, pages) = list_all(&client, None, None, None, 7).await?;
+    assert_eq!(keys, expected_sorted);
+    assert!(prefixes.is_empty());
+    assert_eq!(pages, 143);
+
+    // Delimiter folding: 30 common prefixes then 100 top-level keys, pages of 7.
+    let (keys, prefixes, pages) = list_all(&client, None, Some("/"), None, 7).await?;
+    assert_eq!(prefixes, expected_prefixes);
+    assert_eq!(keys, expected_top);
+    assert_eq!(pages, 19);
+
+    // Empty prefix equals no prefix.
+    let (keys, _, _) = list_all(&client, Some(""), None, None, 1000).await?;
+    assert_eq!(keys, expected_sorted);
+
+    // No match: empty, not truncated, no token.
+    let (keys, prefixes, pages) = list_all(&client, Some("zzz/"), Some("/"), None, 7).await?;
+    assert!(keys.is_empty() && prefixes.is_empty());
+    assert_eq!(pages, 1);
+    let (keys, _, _) = list_all(&client, Some("hidden/"), None, None, 7).await?;
+    assert!(keys.is_empty(), "a current delete marker must hide its key");
+
+    // Exact page boundary: 30 keys under one directory, max-keys=30 -> one
+    // untruncated page.
+    let (keys, prefixes, pages) = list_all(&client, Some("d05/"), Some("/"), None, 30).await?;
+    assert_eq!(keys.len(), 30);
+    assert!(prefixes.is_empty());
+    assert_eq!(pages, 1);
+
+    // start-after skips keys at or before the marker.
+    let (keys, _, _) = list_all(&client, None, None, Some("top-097"), 1000).await?;
+    assert_eq!(keys, ["top-098", "top-099"]);
+
+    // max-keys is clamped to 1000; exactly 1000 keys fit in one page.
+    let (keys, _, pages) = list_all(&client, None, None, None, 5000).await?;
+    assert_eq!(keys.len(), 1000);
+    assert_eq!(pages, 1);
+
+    let listings: Vec<_> = source
+        .requests()
+        .into_iter()
+        .filter(|record| record.operation == Operation::ListObjectsV2)
+        .collect();
+    assert!(listings.len() >= 143 + 19);
+    assert!(listings.iter().any(|record| record.prefix.as_deref() == Some("d05/")));
+    assert!(
+        listings.iter().any(|record| record.continuation_token.is_some()),
+        "resumed pages must journal their continuation token"
+    );
+    assert!(listings.iter().all(|record| record.user_agent.is_some()));
+    source.shutdown().await;
+    Ok(())
+}
+
+#[tokio::test]
+async fn fake_source_range_get_variants_and_416() -> TestResult {
+    let (source, client) = fake_source().await?;
+    let body = payload(1000);
+    source.put_seed_object(SOURCE_BUCKET, "ranged", body.clone(), &SeedMetadata::new());
+
+    for (range, expected_range, expected_slice) in [
+        ("bytes=10-19", "bytes 10-19/1000", &body[10..20]),
+        ("bytes=990-", "bytes 990-999/1000", &body[990..]),
+        ("bytes=-5", "bytes 995-999/1000", &body[995..]),
+        ("bytes=0-5000", "bytes 0-999/1000", &body[..]),
+    ] {
+        let output = client
+            .get_object()
+            .bucket(SOURCE_BUCKET)
+            .key("ranged")
+            .range(range)
+            .send()
+            .await?;
+        assert_eq!(output.content_range(), Some(expected_range), "{range}");
+        assert_eq!(output.accept_ranges(), Some("bytes"), "{range}");
+        assert_eq!(output.content_length(), Some(expected_slice.len() as i64), "{range}");
+        let collected = output.body.collect().await?.into_bytes();
+        assert_eq!(collected.as_ref(), expected_slice, "{range}");
+    }
+    let head = client
+        .head_object()
+        .bucket(SOURCE_BUCKET)
+        .key("ranged")
+        .range("bytes=10-19")
+        .send()
+        .await?;
+    assert_eq!(head.content_range(), Some("bytes 10-19/1000"));
+    assert_eq!(head.content_length(), Some(10));
+
+    for range in ["bytes=1000-", "bytes=-0"] {
+        let error = client
+            .get_object()
+            .bucket(SOURCE_BUCKET)
+            .key("ranged")
+            .range(range)
+            .send()
+            .await
+            .expect_err("unsatisfiable range must fail");
+        let response = error.raw_response().expect("416 must retain the raw response");
+        assert_eq!(response.status().as_u16(), 416, "{range}");
+        assert_eq!(response.headers().get("content-range"), Some("bytes */1000"), "{range}");
+        assert_eq!(error.code(), Some("InvalidRange"), "{range}");
+    }
+
+    let ranged = source
+        .requests()
+        .into_iter()
+        .find(|record| record.operation == Operation::GetObject && record.range.as_deref() == Some("bytes=10-19"))
+        .expect("the Range header must be journaled verbatim");
+    assert_eq!(ranged.key.as_deref(), Some("ranged"));
+    assert!(source.count_requests(Operation::GetObject, "ranged") >= 6);
+    source.shutdown().await;
+    Ok(())
+}
+
+#[tokio::test]
+async fn fake_source_unversioned_bucket_overwrites_and_deletes() -> TestResult {
+    let (source, client) = fake_source().await?;
+    source.create_bucket_with_mode("plain-source", BucketMode::Unversioned);
+    let versioning = client.get_bucket_versioning().bucket("plain-source").send().await?;
+    assert!(versioning.status().is_none(), "unversioned bucket must report no versioning status");
+
+    let first = client
+        .put_object()
+        .bucket("plain-source")
+        .key("doc")
+        .body(ByteStream::from_static(b"first"))
+        .send()
+        .await?;
+    assert!(first.version_id().is_none());
+    let second = client
+        .put_object()
+        .bucket("plain-source")
+        .key("doc")
+        .body(ByteStream::from_static(b"second"))
+        .send()
+        .await?;
+    assert!(second.version_id().is_none());
+    let get = client.get_object().bucket("plain-source").key("doc").send().await?;
+    assert!(get.version_id().is_none(), "GET must not return x-amz-version-id");
+    assert_eq!(get.body.collect().await?.into_bytes().as_ref(), b"second");
+    let head = client.head_object().bucket("plain-source").key("doc").send().await?;
+    assert!(head.version_id().is_none(), "HEAD must not return x-amz-version-id");
+    assert_eq!(source.stored_versions("plain-source", "doc").len(), 1, "overwrite must replace in place");
+
+    let deleted = client.delete_object().bucket("plain-source").key("doc").send().await?;
+    assert!(deleted.delete_marker().is_none() && deleted.version_id().is_none());
+    let missing = client
+        .get_object()
+        .bucket("plain-source")
+        .key("doc")
+        .send()
+        .await
+        .expect_err("deleted object must be gone");
+    assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
+    assert_eq!(missing.code(), Some("NoSuchKey"));
+    let missing_head = client
+        .head_object()
+        .bucket("plain-source")
+        .key("doc")
+        .send()
+        .await
+        .expect_err("deleted object must fail HEAD");
+    assert_eq!(missing_head.raw_response().map(|response| response.status().as_u16()), Some(404));
+    assert!(source.stored_versions("plain-source", "doc").is_empty(), "DELETE must not leave a marker");
+
+    // The versioned bucket on the same target keeps its version ids.
+    let versioned = client
+        .put_object()
+        .bucket(SOURCE_BUCKET)
+        .key("doc")
+        .body(ByteStream::from_static(b"versioned"))
+        .send()
+        .await?;
+    assert!(versioned.version_id().is_some());
+    source.shutdown().await;
+    Ok(())
+}
+
+#[tokio::test]
+async fn fake_source_replays_standard_and_user_metadata() -> TestResult {
+    let (source, client) = fake_source().await?;
+    let body = payload(4096);
+    let expected_etag = format!("\"{}\"", {
+        use md5::Digest as _;
+        hex_simd::encode_to_string(md5::Md5::digest(&body), hex_simd::AsciiCase::Lower)
+    });
+    // 2026-01-01T00:00:00Z rendered as an HTTP date by the SDK.
+    let expires = DateTime::from_secs(1_767_225_600);
+    client
+        .put_object()
+        .bucket(SOURCE_BUCKET)
+        .key("meta")
+        .body(ByteStream::from(body.clone()))
+        .content_type("application/x-odm")
+        .content_encoding("gzip")
+        .content_disposition("attachment; filename=\"meta.bin\"")
+        .content_language("en-US")
+        .cache_control("max-age=60")
+        .expires(expires)
+        .metadata("Foo-Bar", "mixed case name")
+        .metadata("UPPER", "upper name")
+        .metadata("already-lower", "lower name")
+        .send()
+        .await?;
+
+    let head = client.head_object().bucket(SOURCE_BUCKET).key("meta").send().await?;
+    let get = client.get_object().bucket(SOURCE_BUCKET).key("meta").send().await?;
+    for (label, content_type, content_encoding, content_disposition, content_language, cache_control, expires_string, e_tag) in [
+        (
+            "HEAD",
+            head.content_type(),
+            head.content_encoding(),
+            head.content_disposition(),
+            head.content_language(),
+            head.cache_control(),
+            head.expires_string(),
+            head.e_tag(),
+        ),
+        (
+            "GET",
+            get.content_type(),
+            get.content_encoding(),
+            get.content_disposition(),
+            get.content_language(),
+            get.cache_control(),
+            get.expires_string(),
+            get.e_tag(),
+        ),
+    ] {
+        assert_eq!(content_type, Some("application/x-odm"), "{label}");
+        assert_eq!(content_encoding, Some("gzip"), "{label}");
+        assert_eq!(content_disposition, Some("attachment; filename=\"meta.bin\""), "{label}");
+        assert_eq!(content_language, Some("en-US"), "{label}");
+        assert_eq!(cache_control, Some("max-age=60"), "{label}");
+        assert_eq!(expires_string, Some("Thu, 01 Jan 2026 00:00:00 GMT"), "{label}");
+        assert_eq!(e_tag, Some(expected_etag.as_str()), "{label}");
+    }
+    for metadata in [head.metadata(), get.metadata()] {
+        let metadata = metadata.expect("user metadata must be replayed");
+        assert_eq!(metadata.get("foo-bar").map(String::as_str), Some("mixed case name"));
+        assert_eq!(metadata.get("upper").map(String::as_str), Some("upper name"));
+        assert_eq!(metadata.get("already-lower").map(String::as_str), Some("lower name"));
+        assert!(!metadata.contains_key("Foo-Bar") && !metadata.contains_key("UPPER"));
+    }
+    assert!(head.last_modified().is_some());
+    assert_eq!(head.last_modified(), get.last_modified());
+    assert_eq!(head.content_length(), Some(4096));
+    assert_eq!(get.body.collect().await?.into_bytes(), body);
+
+    // Seeded objects replay the same way.
+    let seeded_etag = source.put_seed_object(
+        SOURCE_BUCKET,
+        "seeded",
+        Bytes::from_static(b"seeded"),
+        &SeedMetadata::new()
+            .content_type("text/plain")
+            .content_encoding("identity")
+            .cache_control("no-store")
+            .user_metadata("Origin", "seed"),
+    );
+    let seeded = client.head_object().bucket(SOURCE_BUCKET).key("seeded").send().await?;
+    assert_eq!(seeded.e_tag(), Some(format!("\"{seeded_etag}\"").as_str()));
+    assert_eq!(seeded.content_type(), Some("text/plain"));
+    assert_eq!(seeded.content_encoding(), Some("identity"));
+    assert_eq!(seeded.cache_control(), Some("no-store"));
+    assert_eq!(
+        seeded
+            .metadata()
+            .and_then(|metadata| metadata.get("origin"))
+            .map(String::as_str),
+        Some("seed")
+    );
+    source.shutdown().await;
+    Ok(())
+}
+
+#[tokio::test]
+async fn fake_source_fault_actions_truncate_stall_and_status() -> TestResult {
+    let (source, client) = fake_source().await?;
+    let body = payload(4096);
+    source.put_seed_object(SOURCE_BUCKET, "faulty", body.clone(), &SeedMetadata::new());
+
+    // TruncateBodyAt: headers promise 4096 bytes, the body ends after 100.
+    source.inject_for_key(Operation::GetObject, "faulty", FaultAction::TruncateBodyAt(100), 1);
+    let truncated = client.get_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
+    assert_eq!(truncated.content_length(), Some(4096));
+    let short_read = truncated
+        .body
+        .collect()
+        .await
+        .expect_err("a truncated body must fail to collect");
+    let short_read = short_read.to_string();
+    assert!(!short_read.is_empty());
+
+    // ResponseStatus: arbitrary status with the matching S3 error code.
+    for (code, expected_code) in [
+        (429u16, "SlowDown"),
+        (404, "NoSuchKey"),
+        (500, "InternalError"),
+        (503, "ServiceUnavailable"),
+    ] {
+        source.inject(Operation::GetObject, FaultAction::ResponseStatus(code), 1);
+        let error = client
+            .get_object()
+            .bucket(SOURCE_BUCKET)
+            .key("faulty")
+            .send()
+            .await
+            .expect_err("scripted status must fail");
+        assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(code));
+        assert_eq!(error.code(), Some(expected_code));
+    }
+
+    // Stall: the fully computed response is held before its first byte.
+    source.inject(Operation::HeadObject, FaultAction::Stall(Duration::from_millis(400)), 1);
+    let started = Instant::now();
+    let stalled = client.head_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
+    assert!(started.elapsed() >= Duration::from_millis(350), "stall must delay the first byte");
+    assert_eq!(stalled.content_length(), Some(4096));
+    let post_stall_started = Instant::now();
+    client.head_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
+    assert!(post_stall_started.elapsed() < Duration::from_millis(350), "stall is consumed once");
+
+    // The object is intact once the script is drained.
+    let intact = client.get_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
+    assert_eq!(intact.body.collect().await?.into_bytes(), body);
+
+    assert_eq!(source.count_requests(Operation::GetObject, "faulty"), 6);
+    assert_eq!(source.count_requests(Operation::HeadObject, "faulty"), 2);
+    assert_eq!(source.count_requests(Operation::GetObject, "other"), 0);
+    let records = source.requests();
+    assert!(
+        records.iter().all(|record| record
+            .user_agent
+            .as_deref()
+            .is_some_and(|agent| agent.contains("aws-sdk-rust"))),
+        "the SDK user agent must be journaled"
+    );
+    assert!(
+        records
+            .iter()
+            .any(|record| record.fault == Some(FaultAction::TruncateBodyAt(100)))
+    );
+    assert!(
+        records
+            .iter()
+            .any(|record| record.fault == Some(FaultAction::Stall(Duration::from_millis(400))))
+    );
+    source.shutdown().await;
+    Ok(())
+}
+
+#[tokio::test]
+async fn fake_source_raised_object_cap_accepts_large_put() -> TestResult {
+    let source = FakeS3Target::start_with_options(FakeS3TargetOptions {
+        max_object_bytes: 96 * 1024 * 1024,
+    })
+    .await?;
+    source.create_bucket(SOURCE_BUCKET);
+    let client = fake_source_client(&source);
+    let len = 64 * 1024 * 1024 + 1;
+    client
+        .put_object()
+        .bucket(SOURCE_BUCKET)
+        .key("large")
+        .body(ByteStream::from(vec![7u8; len]))
+        .send()
+        .await?;
+    let head = client.head_object().bucket(SOURCE_BUCKET).key("large").send().await?;
+    assert_eq!(head.content_length(), Some(len as i64));
+    let tail = client
+        .get_object()
+        .bucket(SOURCE_BUCKET)
+        .key("large")
+        .range("bytes=-1")
+        .send()
+        .await?;
+    assert_eq!(tail.content_range(), Some(format!("bytes {}-{}/{len}", len - 1, len - 1).as_str()));
+    source.shutdown().await;
+    Ok(())
+}
+
+#[tokio::test]
+async fn odm_env_starts_rustfs_and_fake_source() -> TestResult {
+    let env = OdmTestEnv::start().await?;
+    env.source.create_bucket(SOURCE_BUCKET);
+    let local_bucket = "odm-local";
+    env.rustfs.create_test_bucket(local_bucket).await?;
+
+    let etags = env.seed_source(
+        SOURCE_BUCKET,
+        &[
+            SeedObject::new("seed/a", Bytes::from_static(b"alpha")),
+            SeedObject::new("seed/b", Bytes::from_static(b"beta"))
+                .with_metadata(SeedMetadata::new().content_type("text/plain").user_metadata("Kind", "seed")),
+        ],
+    );
+    assert_eq!(etags.len(), 2);
+    assert!(env.source.requests().is_empty(), "seeding must not touch the journal");
+    let source_client = env.source_client();
+    let seeded = source_client.head_object().bucket(SOURCE_BUCKET).key("seed/b").send().await?;
+    assert_eq!(seeded.content_type(), Some("text/plain"));
+    assert_eq!(seeded.e_tag(), Some(format!("\"{}\"", etags[1]).as_str()));
+    assert_eq!(env.source.count_requests(Operation::HeadObject, "seed/b"), 1);
+
+    env.assert_local_absent(local_bucket, "seed/a").await;
+    env.client
+        .put_object()
+        .bucket(local_bucket)
+        .key("seed/a")
+        .body(ByteStream::from_static(b"alpha"))
+        .send()
+        .await?;
+    env.assert_local_present(local_bucket, "seed/a", b"alpha").await;
+    env.assert_local_absent(local_bucket, "seed/b").await;
+
+    let spec = env.fake_source_spec(SOURCE_BUCKET).to_json();
+    assert_eq!(spec["version"], 1);
+    assert_eq!(spec["enabled"], true);
+    assert_eq!(spec["source"]["provider"], "s3");
+    assert_eq!(spec["source"]["endpoint"], env.source.endpoint());
+    assert_eq!(spec["source"]["bucket"], SOURCE_BUCKET);
+    assert_eq!(spec["source"]["credentials"]["secret_key"], "fake-secret");
+    assert_eq!(spec["policy"]["source_timeout"]["first_byte_ms"], 15_000);
+    assert!(spec["policy"]["bandwidth_limit_bytes_per_sec"].is_null());
+    let debug = format!("{:?}", env.fake_source_spec(SOURCE_BUCKET));
+    assert!(!debug.contains("fake-secret"), "Debug output must redact the secret");
+    Ok(())
+}
+
+#[tokio::test]
+async fn start_source_rustfs_round_trips_put_get() -> TestResult {
+    let env = OdmTestEnv::start().await?;
+    let source = start_source_rustfs().await?;
+    assert_ne!(source.url, env.rustfs.url, "the source must be a separate instance");
+
+    source.create_test_bucket(SOURCE_BUCKET).await?;
+    let source_client = source.create_s3_client();
+    let body = payload(70_000);
+    let put = source_client
+        .put_object()
+        .bucket(SOURCE_BUCKET)
+        .key("real/object")
+        .body(ByteStream::from(body.clone()))
+        .content_type("application/octet-stream")
+        .send()
+        .await?;
+    assert!(put.e_tag().is_some());
+    let get = source_client
+        .get_object()
+        .bucket(SOURCE_BUCKET)
+        .key("real/object")
+        .send()
+        .await?;
+    assert_eq!(get.content_type(), Some("application/octet-stream"));
+    assert_eq!(get.body.collect().await?.into_bytes(), body);
+
+    let visible_to_primary = env
+        .client
+        .list_buckets()
+        .send()
+        .await?
+        .buckets()
+        .iter()
+        .any(|bucket| bucket.name() == Some(SOURCE_BUCKET));
+    assert!(!visible_to_primary, "the two servers must not share state");
+    let spec = super::common::OdmSourceSpec::for_rustfs_source(&source, SOURCE_BUCKET).to_json();
+    assert_eq!(spec["source"]["provider"], "rustfs");
+    assert_eq!(spec["source"]["endpoint"], source.url);
+    Ok(())
+}

crates/e2e_test/src/on_demand_migration/mod.rs

@@ -0,0 +1,24 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! On-demand migration (ODM) end-to-end suite (rustfs/backlog#2147).
+//!
+//! `common` is the shared environment: one RustFS under test, one programmable
+//! fake S3 source, admin-API wrappers, seeding and local-state assertions.
+//! `harness_self_test` proves the harness itself; ODM behavior scenarios are
+//! separate modules wired by later tasks.
+
+pub mod common;
+
+mod harness_self_test;

crates/ecstore/Cargo.toml

@@ -166,7 +166,7 @@ uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnos
 reed-solomon-erasure = { workspace = true, features = ["simd-accel"] }
 reed-solomon-simd = { workspace = true }
 lazy_static.workspace = true
-moka = { workspace = true, features = ["future", "sync"] }
+moka = { workspace = true, features = ["future"] }
 rustfs-lock.workspace = true
 rustfs-io-metrics.workspace = true
 regex = { workspace = true }
@@ -185,7 +185,7 @@ hyper-rustls = { workspace = true, default-features = false, features = ["native
 hostname.workspace = true
 rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
 rustls-pki-types.workspace = true
-tokio = { workspace = true, features = ["io-util", "sync", "signal", "fs", "rt-multi-thread", "time"] }
+tokio = { workspace = true, features = ["io-util", "sync", "signal", "fs", "rt-multi-thread"] }
 tonic = { workspace = true, features = ["gzip", "deflate"] }
 xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
 tower = { workspace = true, features = ["timeout"] }

crates/ecstore/src/api/mod.rs

@@ -146,14 +146,6 @@ pub mod bucket {
     }
 
     pub mod on_demand_migration {
-        pub use crate::bucket::on_demand_migration::{
-            ApplyOutcome, BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION,
-            Breaker, BreakerState, BreakerTransition, BreakerVerdict, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, GaugeGuard,
-            LastSourceError, LatencyBucketSnapshot, NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache, OdmBucketSnapshot, OdmLookup,
-            OdmOp, OdmOutcome, OdmStateError, OdmStats, OdmStatsSnapshot, OnDemandMigrationSys, PullError, PullFailureReason,
-            PullFollower, PullLeader, PullOutcome, PullPath, PullResult, PullSlot, SOURCE_LATENCY_BUCKET_BOUNDS_MS,
-            SourceLatencySnapshot, source_client_spec,
-        };
         pub use crate::bucket::on_demand_migration::{
             ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION,
             OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy,

crates/ecstore/src/bucket/on_demand_migration/breaker.rs

@@ -1,362 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Per-bucket three-state circuit breaker protecting an on-demand migration
-//! source (rustfs/backlog#2152).
-//!
-//! `Closed` lets every request through and counts consecutive failures
-//! inside a sliding window; reaching the threshold opens the breaker. `Open`
-//! rejects everything until the open duration elapses, then moves to
-//! `HalfOpen`, which admits a single probe: success closes the breaker,
-//! failure re-opens it. Timing uses `tokio::time::Instant` so tests can drive
-//! it with `tokio::time::pause`.
-//!
-//! Only transport-level failures count (`Throttled`, `Timeout`, `Connect`,
-//! `ServerError`). `NotFound` is a healthy answer and resets the failure
-//! streak; `AccessDenied`, `Unsupported` and `Other` are configuration or
-//! object problems that neither open nor close the breaker.
-
-use super::source_client::SourceError;
-use parking_lot::Mutex;
-use serde::{Deserialize, Serialize};
-use std::time::Duration;
-use tokio::time::Instant;
-
-/// Consecutive counted failures that open the breaker.
-pub const BREAKER_FAILURE_THRESHOLD: u32 = 5;
-/// Failures further apart than this do not accumulate.
-pub const BREAKER_FAILURE_WINDOW: Duration = Duration::from_secs(30);
-/// How long an open breaker rejects before admitting a probe.
-pub const BREAKER_OPEN_DURATION: Duration = Duration::from_secs(30);
-/// Probes admitted while half-open.
-pub const BREAKER_HALF_OPEN_MAX_PROBES: u32 = 1;
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub enum BreakerState {
-    Closed,
-    Open,
-    HalfOpen,
-}
-
-impl BreakerState {
-    pub fn as_str(self) -> &'static str {
-        match self {
-            BreakerState::Closed => "closed",
-            BreakerState::Open => "open",
-            BreakerState::HalfOpen => "half_open",
-        }
-    }
-}
-
-/// A state change the caller may want to log.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub struct BreakerTransition {
-    pub from: BreakerState,
-    pub to: BreakerState,
-}
-
-/// How a source result is scored by the breaker.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum BreakerVerdict {
-    /// Resets the failure streak; closes a half-open breaker.
-    Success,
-    /// Counts toward the threshold; re-opens a half-open breaker.
-    Failure,
-    /// Leaves the breaker untouched.
-    Neutral,
-}
-
-impl BreakerVerdict {
-    /// `None` is a successful source call.
-    pub fn for_result(error: Option<&SourceError>) -> Self {
-        match error {
-            None | Some(SourceError::NotFound) => BreakerVerdict::Success,
-            Some(SourceError::Throttled | SourceError::Timeout | SourceError::Connect(_) | SourceError::ServerError(_)) => {
-                BreakerVerdict::Failure
-            }
-            Some(SourceError::AccessDenied | SourceError::Unsupported(_) | SourceError::Other(_)) => BreakerVerdict::Neutral,
-        }
-    }
-}
-
-#[derive(Debug)]
-struct Inner {
-    state: BreakerState,
-    consecutive_failures: u32,
-    last_failure_at: Option<Instant>,
-    opened_at: Option<Instant>,
-    half_open_probes: u32,
-}
-
-#[derive(Debug)]
-pub struct Breaker {
-    inner: Mutex<Inner>,
-}
-
-impl Default for Breaker {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl Breaker {
-    pub fn new() -> Self {
-        Self {
-            inner: Mutex::new(Inner {
-                state: BreakerState::Closed,
-                consecutive_failures: 0,
-                last_failure_at: None,
-                opened_at: None,
-                half_open_probes: 0,
-            }),
-        }
-    }
-
-    /// Current state after applying the open-duration timeout.
-    pub fn state(&self) -> BreakerState {
-        let mut inner = self.inner.lock();
-        Self::advance(&mut inner, Instant::now());
-        inner.state
-    }
-
-    /// Whether a request may reach the source right now. Consumes the
-    /// half-open probe budget when it grants one.
-    pub fn allow_request(&self) -> bool {
-        let mut inner = self.inner.lock();
-        Self::advance(&mut inner, Instant::now());
-        match inner.state {
-            BreakerState::Closed => true,
-            BreakerState::Open => false,
-            BreakerState::HalfOpen => {
-                if inner.half_open_probes < BREAKER_HALF_OPEN_MAX_PROBES {
-                    inner.half_open_probes += 1;
-                    true
-                } else {
-                    false
-                }
-            }
-        }
-    }
-
-    /// Scores a source result; returns the transition it caused, if any.
-    pub fn record(&self, verdict: BreakerVerdict) -> Option<BreakerTransition> {
-        match verdict {
-            BreakerVerdict::Success => self.record_success(),
-            BreakerVerdict::Failure => self.record_failure(),
-            BreakerVerdict::Neutral => None,
-        }
-    }
-
-    pub fn record_success(&self) -> Option<BreakerTransition> {
-        let mut inner = self.inner.lock();
-        let now = Instant::now();
-        Self::advance(&mut inner, now);
-        inner.consecutive_failures = 0;
-        inner.last_failure_at = None;
-        match inner.state {
-            BreakerState::Closed => None,
-            // A success while open can only come from a request admitted
-            // before the breaker opened; it says nothing about recovery.
-            BreakerState::Open => None,
-            BreakerState::HalfOpen => Some(Self::transition(&mut inner, BreakerState::Closed, now)),
-        }
-    }
-
-    pub fn record_failure(&self) -> Option<BreakerTransition> {
-        let mut inner = self.inner.lock();
-        let now = Instant::now();
-        Self::advance(&mut inner, now);
-        match inner.state {
-            BreakerState::Closed => {
-                let within_window = inner
-                    .last_failure_at
-                    .is_some_and(|last| now.saturating_duration_since(last) <= BREAKER_FAILURE_WINDOW);
-                inner.consecutive_failures = if within_window { inner.consecutive_failures + 1 } else { 1 };
-                inner.last_failure_at = Some(now);
-                if inner.consecutive_failures >= BREAKER_FAILURE_THRESHOLD {
-                    Some(Self::transition(&mut inner, BreakerState::Open, now))
-                } else {
-                    None
-                }
-            }
-            BreakerState::Open => None,
-            BreakerState::HalfOpen => Some(Self::transition(&mut inner, BreakerState::Open, now)),
-        }
-    }
-
-    fn advance(inner: &mut Inner, now: Instant) {
-        if inner.state == BreakerState::Open
-            && inner
-                .opened_at
-                .is_some_and(|opened| now.saturating_duration_since(opened) >= BREAKER_OPEN_DURATION)
-        {
-            Self::transition(inner, BreakerState::HalfOpen, now);
-        }
-    }
-
-    fn transition(inner: &mut Inner, to: BreakerState, now: Instant) -> BreakerTransition {
-        let from = inner.state;
-        inner.state = to;
-        match to {
-            BreakerState::Open => {
-                inner.opened_at = Some(now);
-                inner.half_open_probes = 0;
-            }
-            BreakerState::HalfOpen => {
-                inner.half_open_probes = 0;
-            }
-            BreakerState::Closed => {
-                inner.opened_at = None;
-                inner.half_open_probes = 0;
-                inner.consecutive_failures = 0;
-                inner.last_failure_at = None;
-            }
-        }
-        BreakerTransition { from, to }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    fn server_error() -> SourceError {
-        SourceError::ServerError(503)
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn five_failures_open_then_half_open_after_timeout() {
-        let breaker = Breaker::new();
-        for i in 0..BREAKER_FAILURE_THRESHOLD - 1 {
-            assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&server_error()))), None, "failure {i}");
-            assert_eq!(breaker.state(), BreakerState::Closed);
-        }
-        assert_eq!(
-            breaker.record(BreakerVerdict::for_result(Some(&server_error()))),
-            Some(BreakerTransition {
-                from: BreakerState::Closed,
-                to: BreakerState::Open
-            })
-        );
-        assert_eq!(breaker.state(), BreakerState::Open);
-        assert!(!breaker.allow_request());
-
-        tokio::time::advance(BREAKER_OPEN_DURATION - Duration::from_secs(1)).await;
-        assert!(!breaker.allow_request());
-        assert_eq!(breaker.state(), BreakerState::Open);
-
-        tokio::time::advance(Duration::from_secs(1)).await;
-        assert_eq!(breaker.state(), BreakerState::HalfOpen);
-        assert!(breaker.allow_request(), "one probe is admitted");
-        assert!(!breaker.allow_request(), "second probe is rejected");
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn half_open_probe_success_closes_and_failure_reopens() {
-        let breaker = Breaker::new();
-        for _ in 0..BREAKER_FAILURE_THRESHOLD {
-            breaker.record_failure();
-        }
-        tokio::time::advance(BREAKER_OPEN_DURATION).await;
-        assert!(breaker.allow_request());
-        assert_eq!(
-            breaker.record_failure(),
-            Some(BreakerTransition {
-                from: BreakerState::HalfOpen,
-                to: BreakerState::Open
-            })
-        );
-        assert!(!breaker.allow_request());
-
-        tokio::time::advance(BREAKER_OPEN_DURATION).await;
-        assert!(breaker.allow_request());
-        assert_eq!(
-            breaker.record_success(),
-            Some(BreakerTransition {
-                from: BreakerState::HalfOpen,
-                to: BreakerState::Closed
-            })
-        );
-        assert_eq!(breaker.state(), BreakerState::Closed);
-        assert!(breaker.allow_request());
-        // The streak restarts from zero after closing.
-        for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
-            assert_eq!(breaker.record_failure(), None);
-        }
-        assert_eq!(breaker.state(), BreakerState::Closed);
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn failures_outside_window_do_not_accumulate() {
-        let breaker = Breaker::new();
-        for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
-            breaker.record_failure();
-        }
-        tokio::time::advance(BREAKER_FAILURE_WINDOW + Duration::from_secs(1)).await;
-        assert_eq!(breaker.record_failure(), None, "stale streak restarts at one");
-        assert_eq!(breaker.state(), BreakerState::Closed);
-    }
-
-    #[test]
-    fn not_found_and_access_denied_do_not_count() {
-        let breaker = Breaker::new();
-        for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
-            breaker.record(BreakerVerdict::for_result(Some(&server_error())));
-        }
-        assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::AccessDenied))), None);
-        assert_eq!(breaker.state(), BreakerState::Closed);
-        // AccessDenied is neutral: the streak is still one short of opening.
-        assert_eq!(
-            breaker.record(BreakerVerdict::for_result(Some(&SourceError::Unsupported("sse-c".into())))),
-            None
-        );
-        assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::Other("x".into())))), None);
-        // NotFound is a healthy answer and resets the streak entirely.
-        assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::NotFound))), None);
-        for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
-            assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::Timeout))), None);
-        }
-        assert_eq!(breaker.state(), BreakerState::Closed);
-    }
-
-    #[test]
-    fn verdicts_cover_every_source_error_class() {
-        assert_eq!(BreakerVerdict::for_result(None), BreakerVerdict::Success);
-        assert_eq!(BreakerVerdict::for_result(Some(&SourceError::NotFound)), BreakerVerdict::Success);
-        for failure in [
-            SourceError::Throttled,
-            SourceError::Timeout,
-            SourceError::Connect("refused".into()),
-            SourceError::ServerError(500),
-        ] {
-            assert_eq!(BreakerVerdict::for_result(Some(&failure)), BreakerVerdict::Failure, "{failure:?}");
-        }
-        for neutral in [
-            SourceError::AccessDenied,
-            SourceError::Unsupported("sse-c".into()),
-            SourceError::Other("x".into()),
-        ] {
-            assert_eq!(BreakerVerdict::for_result(Some(&neutral)), BreakerVerdict::Neutral, "{neutral:?}");
-        }
-    }
-
-    #[test]
-    fn state_labels_are_stable() {
-        assert_eq!(BreakerState::Closed.as_str(), "closed");
-        assert_eq!(BreakerState::Open.as_str(), "open");
-        assert_eq!(BreakerState::HalfOpen.as_str(), "half_open");
-        assert_eq!(serde_json::to_string(&BreakerState::HalfOpen).unwrap(), "\"half_open\"");
-    }
-}

crates/ecstore/src/bucket/on_demand_migration/mod.rs

@@ -15,33 +15,14 @@
 //! On-Demand Migration (ODM): a bucket can name an external S3-compatible
 //! source bucket; GET misses are served from that source and backfilled
 //! locally. This module owns the bucket-level configuration model
-//! (`on-demand-migration.json` in the bucket metadata file), the source
-//! client, and the per-node runtime (`sys`) that turns configs into live
-//! clients guarded by a breaker, a negative cache, singleflight and a pull
-//! concurrency limit (rustfs/backlog#2147).
+//! (`on-demand-migration.json` in the bucket metadata file); the runtime is
+//! layered on top of it by later tasks (rustfs/backlog#2147).
 
-pub mod breaker;
 pub mod config;
-pub mod negative_cache;
 pub mod source_client;
-pub mod stats;
-pub mod sys;
 
-pub use breaker::{
-    BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION, Breaker,
-    BreakerState, BreakerTransition, BreakerVerdict,
-};
 pub use config::{
     ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION,
     OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, SourceConfig,
     SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
 };
-pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache};
-pub use stats::{
-    GaugeGuard, LastSourceError, LatencyBucketSnapshot, OdmOp, OdmOutcome, OdmStats, OdmStatsSnapshot, PullFailureReason,
-    PullPath, SOURCE_LATENCY_BUCKET_BOUNDS_MS, SourceLatencySnapshot,
-};
-pub use sys::{
-    ApplyOutcome, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, OdmBucketSnapshot, OdmLookup, OdmStateError,
-    OnDemandMigrationSys, PullError, PullFollower, PullLeader, PullOutcome, PullResult, PullSlot, source_client_spec,
-};

crates/ecstore/src/bucket/on_demand_migration/negative_cache.rs

@@ -1,130 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Per-bucket cache of keys the source answered 404 for
-//! (rustfs/backlog#2152). A hit short-circuits the source lookup for
-//! `policy.negative_cache_ttl_secs`; a TTL of zero disables the cache.
-//!
-//! Entries are never invalidated on a local PUT: once the object exists
-//! locally the handler never consults ODM for it, so a stale negative entry
-//! is harmless.
-
-use std::time::Duration;
-
-/// Upper bound on remembered keys per bucket; LRU eviction beyond it.
-pub const NEGATIVE_CACHE_MAX_ENTRIES: u64 = 100_000;
-
-#[derive(Debug)]
-pub struct NegativeCache {
-    cache: Option<moka::sync::Cache<String, ()>>,
-    ttl: Duration,
-}
-
-impl NegativeCache {
-    /// `ttl == 0` builds a disabled cache that never records anything.
-    pub fn new(ttl: Duration) -> Self {
-        Self::with_capacity(ttl, NEGATIVE_CACHE_MAX_ENTRIES)
-    }
-
-    pub fn with_capacity(ttl: Duration, max_entries: u64) -> Self {
-        let cache = (!ttl.is_zero()).then(|| {
-            moka::sync::Cache::builder()
-                .max_capacity(max_entries)
-                .time_to_live(ttl)
-                .build()
-        });
-        Self { cache, ttl }
-    }
-
-    pub fn is_enabled(&self) -> bool {
-        self.cache.is_some()
-    }
-
-    pub fn ttl(&self) -> Duration {
-        self.ttl
-    }
-
-    /// Whether `key` is currently remembered as absent on the source.
-    pub fn contains(&self, key: &str) -> bool {
-        self.cache.as_ref().is_some_and(|cache| cache.get(key).is_some())
-    }
-
-    /// Remembers `key` as absent; no-op when disabled.
-    pub fn insert(&self, key: &str) {
-        if let Some(cache) = &self.cache {
-            cache.insert(key.to_string(), ());
-        }
-    }
-
-    /// Forgets `key` (e.g. after an admin-triggered backfill found it).
-    pub fn remove(&self, key: &str) {
-        if let Some(cache) = &self.cache {
-            cache.invalidate(key);
-        }
-    }
-
-    /// Approximate live entry count, for status snapshots only.
-    pub fn len(&self) -> u64 {
-        self.cache.as_ref().map_or(0, |cache| {
-            cache.run_pending_tasks();
-            cache.entry_count()
-        })
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn entry_expires_after_ttl() {
-        let cache = NegativeCache::new(Duration::from_millis(80));
-        assert!(cache.is_enabled());
-        cache.insert("a/x");
-        assert!(cache.contains("a/x"));
-        assert!(!cache.contains("a/y"));
-        std::thread::sleep(Duration::from_millis(160));
-        assert!(!cache.contains("a/x"), "entry must expire after the TTL");
-    }
-
-    #[test]
-    fn zero_ttl_disables_the_cache() {
-        let cache = NegativeCache::new(Duration::ZERO);
-        assert!(!cache.is_enabled());
-        cache.insert("a/x");
-        assert!(!cache.contains("a/x"));
-        assert!(cache.is_empty());
-    }
-
-    #[test]
-    fn remove_forgets_a_key() {
-        let cache = NegativeCache::new(Duration::from_secs(30));
-        cache.insert("a/x");
-        cache.remove("a/x");
-        assert!(!cache.contains("a/x"));
-    }
-
-    #[test]
-    fn capacity_bounds_entries() {
-        let cache = NegativeCache::with_capacity(Duration::from_secs(30), 4);
-        for i in 0..64 {
-            cache.insert(&format!("k{i}"));
-        }
-        assert!(cache.len() <= 4, "len {} exceeds capacity", cache.len());
-    }
-}

crates/ecstore/src/bucket/on_demand_migration/stats.rs

@@ -1,527 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Per-bucket on-demand migration counters (rustfs/backlog#2152).
-//!
-//! `OdmStats` is lock-free and survives config rebuilds; `snapshot()` turns
-//! it into the serializable `OdmStatsSnapshot` that the metrics collector
-//! and the admin status route (ODM-10/14/15) consume. Field names and label
-//! values are a wire contract: the golden JSON test below pins them.
-
-use super::breaker::BreakerState;
-use super::source_client::SourceError;
-use parking_lot::Mutex;
-use serde::{Deserialize, Serialize};
-use std::collections::BTreeMap;
-use std::sync::Arc;
-use std::sync::atomic::{AtomicU64, Ordering};
-use std::time::Duration;
-use time::OffsetDateTime;
-
-/// Request operations that can enter ODM.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum OdmOp {
-    Get,
-    Head,
-}
-
-impl OdmOp {
-    pub const ALL: [OdmOp; 2] = [OdmOp::Get, OdmOp::Head];
-
-    pub fn as_str(self) -> &'static str {
-        match self {
-            OdmOp::Get => "get",
-            OdmOp::Head => "head",
-        }
-    }
-}
-
-/// How a request that entered ODM ended. `local_hit` is deliberately absent:
-/// requests served locally never reach the runtime.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum OdmOutcome {
-    SourceHit,
-    SourceMiss,
-    SourceError,
-    BreakerOpen,
-    NegativeCached,
-    Filtered,
-    Unsupported,
-}
-
-impl OdmOutcome {
-    pub const ALL: [OdmOutcome; 7] = [
-        OdmOutcome::SourceHit,
-        OdmOutcome::SourceMiss,
-        OdmOutcome::SourceError,
-        OdmOutcome::BreakerOpen,
-        OdmOutcome::NegativeCached,
-        OdmOutcome::Filtered,
-        OdmOutcome::Unsupported,
-    ];
-
-    pub fn as_str(self) -> &'static str {
-        match self {
-            OdmOutcome::SourceHit => "source_hit",
-            OdmOutcome::SourceMiss => "source_miss",
-            OdmOutcome::SourceError => "source_error",
-            OdmOutcome::BreakerOpen => "breaker_open",
-            OdmOutcome::NegativeCached => "negative_cached",
-            OdmOutcome::Filtered => "filtered",
-            OdmOutcome::Unsupported => "unsupported",
-        }
-    }
-}
-
-/// Which pipeline stored a pulled object locally.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum PullPath {
-    /// Streamed to the client and written locally in one pass.
-    Inline,
-    /// Pulled by a background task after a partial/large read.
-    Background,
-    /// Pulled by the backfill job.
-    Backfill,
-}
-
-impl PullPath {
-    pub const ALL: [PullPath; 3] = [PullPath::Inline, PullPath::Background, PullPath::Backfill];
-
-    pub fn as_str(self) -> &'static str {
-        match self {
-            PullPath::Inline => "inline",
-            PullPath::Background => "background",
-            PullPath::Backfill => "backfill",
-        }
-    }
-}
-
-/// Why a pull did not produce a local object.
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub enum PullFailureReason {
-    SourceNotFound,
-    SourceAccessDenied,
-    SourceThrottled,
-    SourceTimeout,
-    SourceConnect,
-    SourceServerError,
-    SourceUnsupported,
-    SourceOther,
-    /// Source bytes did not match the ETag advertised by HEAD/GET.
-    EtagMismatch,
-    /// The local write (internal PUT) failed.
-    LocalWrite,
-    /// The bucket state was removed or the process is shutting down.
-    Canceled,
-    /// The background pull queue was full.
-    QueueFull,
-}
-
-impl PullFailureReason {
-    pub const ALL: [PullFailureReason; 12] = [
-        PullFailureReason::SourceNotFound,
-        PullFailureReason::SourceAccessDenied,
-        PullFailureReason::SourceThrottled,
-        PullFailureReason::SourceTimeout,
-        PullFailureReason::SourceConnect,
-        PullFailureReason::SourceServerError,
-        PullFailureReason::SourceUnsupported,
-        PullFailureReason::SourceOther,
-        PullFailureReason::EtagMismatch,
-        PullFailureReason::LocalWrite,
-        PullFailureReason::Canceled,
-        PullFailureReason::QueueFull,
-    ];
-
-    pub fn as_str(self) -> &'static str {
-        match self {
-            PullFailureReason::SourceNotFound => "source_not_found",
-            PullFailureReason::SourceAccessDenied => "source_access_denied",
-            PullFailureReason::SourceThrottled => "source_throttled",
-            PullFailureReason::SourceTimeout => "source_timeout",
-            PullFailureReason::SourceConnect => "source_connect",
-            PullFailureReason::SourceServerError => "source_server_error",
-            PullFailureReason::SourceUnsupported => "source_unsupported",
-            PullFailureReason::SourceOther => "source_other",
-            PullFailureReason::EtagMismatch => "etag_mismatch",
-            PullFailureReason::LocalWrite => "local_write",
-            PullFailureReason::Canceled => "canceled",
-            PullFailureReason::QueueFull => "queue_full",
-        }
-    }
-}
-
-impl From<&SourceError> for PullFailureReason {
-    fn from(err: &SourceError) -> Self {
-        match err {
-            SourceError::NotFound => PullFailureReason::SourceNotFound,
-            SourceError::AccessDenied => PullFailureReason::SourceAccessDenied,
-            SourceError::Throttled => PullFailureReason::SourceThrottled,
-            SourceError::Timeout => PullFailureReason::SourceTimeout,
-            SourceError::Connect(_) => PullFailureReason::SourceConnect,
-            SourceError::ServerError(_) => PullFailureReason::SourceServerError,
-            SourceError::Unsupported(_) => PullFailureReason::SourceUnsupported,
-            SourceError::Other(_) => PullFailureReason::SourceOther,
-        }
-    }
-}
-
-/// Upper bounds (milliseconds) of the source latency histogram buckets; the
-/// implicit last bucket is unbounded. Roughly logarithmic from 5 ms to 60 s.
-pub const SOURCE_LATENCY_BUCKET_BOUNDS_MS: [u64; 14] = [
-    5, 10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000, 20_000, 30_000, 60_000,
-];
-
-#[derive(Debug, Default)]
-struct LatencyHistogram {
-    /// One counter per bound plus one for the overflow bucket.
-    buckets: [AtomicU64; SOURCE_LATENCY_BUCKET_BOUNDS_MS.len() + 1],
-    count: AtomicU64,
-    sum_ms: AtomicU64,
-}
-
-impl LatencyHistogram {
-    fn observe(&self, latency: Duration) {
-        let ms = u64::try_from(latency.as_millis()).unwrap_or(u64::MAX);
-        let index = SOURCE_LATENCY_BUCKET_BOUNDS_MS
-            .iter()
-            .position(|bound| ms <= *bound)
-            .unwrap_or(SOURCE_LATENCY_BUCKET_BOUNDS_MS.len());
-        self.buckets[index].fetch_add(1, Ordering::Relaxed);
-        self.count.fetch_add(1, Ordering::Relaxed);
-        self.sum_ms.fetch_add(ms, Ordering::Relaxed);
-    }
-
-    fn snapshot(&self) -> SourceLatencySnapshot {
-        let mut cumulative = 0;
-        let buckets = SOURCE_LATENCY_BUCKET_BOUNDS_MS
-            .iter()
-            .zip(self.buckets.iter())
-            .map(|(bound, counter)| {
-                cumulative += counter.load(Ordering::Relaxed);
-                LatencyBucketSnapshot {
-                    le_ms: *bound,
-                    count: cumulative,
-                }
-            })
-            .collect();
-        SourceLatencySnapshot {
-            buckets,
-            count: self.count.load(Ordering::Relaxed),
-            sum_ms: self.sum_ms.load(Ordering::Relaxed),
-        }
-    }
-}
-
-/// The most recent source failure, kept for operators: class only, never the
-/// key or the message (which may echo attacker-controlled input).
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct LastSourceError {
-    pub class: String,
-    #[serde(with = "time::serde::rfc3339")]
-    pub at: OffsetDateTime,
-}
-
-#[derive(Debug, Default)]
-pub struct OdmStats {
-    requests_total: [[AtomicU64; OdmOutcome::ALL.len()]; OdmOp::ALL.len()],
-    pulled_bytes_total: AtomicU64,
-    pulled_objects_total: [AtomicU64; PullPath::ALL.len()],
-    pull_failures_total: [AtomicU64; PullFailureReason::ALL.len()],
-    inflight_pulls: AtomicU64,
-    queue_depth: AtomicU64,
-    source_latency: LatencyHistogram,
-    last_source_error: Mutex<Option<LastSourceError>>,
-}
-
-impl OdmStats {
-    pub fn new() -> Self {
-        Self::default()
-    }
-
-    pub fn record_request(&self, op: OdmOp, outcome: OdmOutcome) {
-        self.requests_total[op as usize][outcome as usize].fetch_add(1, Ordering::Relaxed);
-    }
-
-    pub fn record_pulled_bytes(&self, bytes: u64) {
-        self.pulled_bytes_total.fetch_add(bytes, Ordering::Relaxed);
-    }
-
-    pub fn record_pulled_object(&self, path: PullPath) {
-        self.pulled_objects_total[path as usize].fetch_add(1, Ordering::Relaxed);
-    }
-
-    pub fn record_pull_failure(&self, reason: PullFailureReason) {
-        self.pull_failures_total[reason as usize].fetch_add(1, Ordering::Relaxed);
-    }
-
-    pub fn record_source_latency(&self, latency: Duration) {
-        self.source_latency.observe(latency);
-    }
-
-    pub fn record_source_error(&self, err: &SourceError) {
-        self.record_source_error_at(err, OffsetDateTime::now_utc());
-    }
-
-    pub fn record_source_error_at(&self, err: &SourceError, at: OffsetDateTime) {
-        *self.last_source_error.lock() = Some(LastSourceError {
-            class: err.class_label().to_string(),
-            at,
-        });
-    }
-
-    pub fn last_source_error(&self) -> Option<LastSourceError> {
-        self.last_source_error.lock().clone()
-    }
-
-    pub fn inflight_pulls(&self) -> u64 {
-        self.inflight_pulls.load(Ordering::Relaxed)
-    }
-
-    pub fn queue_depth(&self) -> u64 {
-        self.queue_depth.load(Ordering::Relaxed)
-    }
-
-    /// RAII increment of `inflight_pulls`.
-    pub fn inflight_guard(self: &Arc<Self>) -> GaugeGuard {
-        GaugeGuard::new(Arc::clone(self), OdmGauge::InflightPulls)
-    }
-
-    /// RAII increment of `queue_depth`.
-    pub fn queue_guard(self: &Arc<Self>) -> GaugeGuard {
-        GaugeGuard::new(Arc::clone(self), OdmGauge::QueueDepth)
-    }
-
-    fn gauge(&self, gauge: OdmGauge) -> &AtomicU64 {
-        match gauge {
-            OdmGauge::InflightPulls => &self.inflight_pulls,
-            OdmGauge::QueueDepth => &self.queue_depth,
-        }
-    }
-
-    /// Read-only, side-effect-free copy of every counter. The breaker lives
-    /// next to the stats in the bucket state; its state is passed in so the
-    /// snapshot stays a single document.
-    pub fn snapshot(&self, breaker_state: BreakerState) -> OdmStatsSnapshot {
-        let mut requests_total = BTreeMap::new();
-        for op in OdmOp::ALL {
-            let mut by_outcome = BTreeMap::new();
-            for outcome in OdmOutcome::ALL {
-                by_outcome.insert(
-                    outcome.as_str().to_string(),
-                    self.requests_total[op as usize][outcome as usize].load(Ordering::Relaxed),
-                );
-            }
-            requests_total.insert(op.as_str().to_string(), by_outcome);
-        }
-        let pulled_objects_total = PullPath::ALL
-            .iter()
-            .map(|path| {
-                (
-                    path.as_str().to_string(),
-                    self.pulled_objects_total[*path as usize].load(Ordering::Relaxed),
-                )
-            })
-            .collect();
-        let pull_failures_total = PullFailureReason::ALL
-            .iter()
-            .map(|reason| {
-                (
-                    reason.as_str().to_string(),
-                    self.pull_failures_total[*reason as usize].load(Ordering::Relaxed),
-                )
-            })
-            .collect();
-        OdmStatsSnapshot {
-            requests_total,
-            pulled_bytes_total: self.pulled_bytes_total.load(Ordering::Relaxed),
-            pulled_objects_total,
-            pull_failures_total,
-            inflight_pulls: self.inflight_pulls(),
-            queue_depth: self.queue_depth(),
-            source_latency: self.source_latency.snapshot(),
-            last_source_error: self.last_source_error(),
-            breaker_state,
-        }
-    }
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-enum OdmGauge {
-    InflightPulls,
-    QueueDepth,
-}
-
-/// Increments a gauge on creation and decrements it on drop. Owns its
-/// `OdmStats` so it can live inside the pull slot handed to callers.
-#[derive(Debug)]
-pub struct GaugeGuard {
-    stats: Arc<OdmStats>,
-    gauge: OdmGauge,
-}
-
-impl GaugeGuard {
-    fn new(stats: Arc<OdmStats>, gauge: OdmGauge) -> Self {
-        stats.gauge(gauge).fetch_add(1, Ordering::Relaxed);
-        Self { stats, gauge }
-    }
-}
-
-impl Drop for GaugeGuard {
-    fn drop(&mut self) {
-        self.stats.gauge(self.gauge).fetch_sub(1, Ordering::Relaxed);
-    }
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct LatencyBucketSnapshot {
-    /// Upper bound of the bucket in milliseconds.
-    pub le_ms: u64,
-    /// Cumulative observations at or below `le_ms`.
-    pub count: u64,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct SourceLatencySnapshot {
-    pub buckets: Vec<LatencyBucketSnapshot>,
-    /// Total observations, including those above the last bound.
-    pub count: u64,
-    pub sum_ms: u64,
-}
-
-/// Serializable copy of [`OdmStats`]. Every key is snake_case and every
-/// label set is fixed, so consumers can rely on the document shape.
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct OdmStatsSnapshot {
-    /// `op -> outcome -> count`.
-    pub requests_total: BTreeMap<String, BTreeMap<String, u64>>,
-    pub pulled_bytes_total: u64,
-    /// `path -> count`.
-    pub pulled_objects_total: BTreeMap<String, u64>,
-    /// `reason -> count`.
-    pub pull_failures_total: BTreeMap<String, u64>,
-    pub inflight_pulls: u64,
-    pub queue_depth: u64,
-    pub source_latency: SourceLatencySnapshot,
-    pub last_source_error: Option<LastSourceError>,
-    pub breaker_state: BreakerState,
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use serde_json::json;
-    use time::macros::datetime;
-
-    #[test]
-    fn snapshot_matches_golden_json() {
-        let stats = Arc::new(OdmStats::new());
-        stats.record_request(OdmOp::Get, OdmOutcome::SourceHit);
-        stats.record_request(OdmOp::Get, OdmOutcome::SourceHit);
-        stats.record_request(OdmOp::Head, OdmOutcome::NegativeCached);
-        stats.record_pulled_bytes(4096);
-        stats.record_pulled_object(PullPath::Inline);
-        stats.record_pull_failure(PullFailureReason::from(&SourceError::Timeout));
-        stats.record_source_latency(Duration::from_millis(3));
-        stats.record_source_latency(Duration::from_millis(750));
-        stats.record_source_latency(Duration::from_secs(90));
-        stats.record_source_error_at(&SourceError::ServerError(502), datetime!(2026-09-02 10:00:00 UTC));
-        let _inflight = stats.inflight_guard();
-        let _queued = stats.queue_guard();
-
-        let snapshot = stats.snapshot(BreakerState::HalfOpen);
-        let actual = serde_json::to_value(&snapshot).unwrap();
-        let expected = json!({
-            "requests_total": {
-                "get": {
-                    "breaker_open": 0, "filtered": 0, "negative_cached": 0, "source_error": 0,
-                    "source_hit": 2, "source_miss": 0, "unsupported": 0
-                },
-                "head": {
-                    "breaker_open": 0, "filtered": 0, "negative_cached": 1, "source_error": 0,
-                    "source_hit": 0, "source_miss": 0, "unsupported": 0
-                }
-            },
-            "pulled_bytes_total": 4096,
-            "pulled_objects_total": { "backfill": 0, "background": 0, "inline": 1 },
-            "pull_failures_total": {
-                "canceled": 0, "etag_mismatch": 0, "local_write": 0, "queue_full": 0,
-                "source_access_denied": 0, "source_connect": 0, "source_not_found": 0, "source_other": 0,
-                "source_server_error": 0, "source_throttled": 0, "source_timeout": 1, "source_unsupported": 0
-            },
-            "inflight_pulls": 1,
-            "queue_depth": 1,
-            "source_latency": {
-                "buckets": [
-                    { "le_ms": 5, "count": 1 }, { "le_ms": 10, "count": 1 }, { "le_ms": 20, "count": 1 },
-                    { "le_ms": 50, "count": 1 }, { "le_ms": 100, "count": 1 }, { "le_ms": 200, "count": 1 },
-                    { "le_ms": 500, "count": 1 }, { "le_ms": 1000, "count": 2 }, { "le_ms": 2000, "count": 2 },
-                    { "le_ms": 5000, "count": 2 }, { "le_ms": 10000, "count": 2 }, { "le_ms": 20000, "count": 2 },
-                    { "le_ms": 30000, "count": 2 }, { "le_ms": 60000, "count": 2 }
-                ],
-                "count": 3,
-                "sum_ms": 90753
-            },
-            "last_source_error": { "class": "server_error", "at": "2026-09-02T10:00:00Z" },
-            "breaker_state": "half_open"
-        });
-        assert_eq!(actual, expected);
-
-        let round_trip: OdmStatsSnapshot = serde_json::from_value(actual).unwrap();
-        assert_eq!(round_trip, snapshot);
-    }
-
-    #[test]
-    fn gauges_return_to_zero_when_guards_drop() {
-        let stats = Arc::new(OdmStats::new());
-        {
-            let _a = stats.inflight_guard();
-            let _b = stats.inflight_guard();
-            let _c = stats.queue_guard();
-            assert_eq!(stats.inflight_pulls(), 2);
-            assert_eq!(stats.queue_depth(), 1);
-        }
-        assert_eq!(stats.inflight_pulls(), 0);
-        assert_eq!(stats.queue_depth(), 0);
-    }
-
-    #[test]
-    fn pull_failure_reason_covers_every_source_error_class() {
-        let cases = [
-            (SourceError::NotFound, PullFailureReason::SourceNotFound),
-            (SourceError::AccessDenied, PullFailureReason::SourceAccessDenied),
-            (SourceError::Throttled, PullFailureReason::SourceThrottled),
-            (SourceError::Timeout, PullFailureReason::SourceTimeout),
-            (SourceError::Connect("x".into()), PullFailureReason::SourceConnect),
-            (SourceError::ServerError(500), PullFailureReason::SourceServerError),
-            (SourceError::Unsupported("x".into()), PullFailureReason::SourceUnsupported),
-            (SourceError::Other("x".into()), PullFailureReason::SourceOther),
-        ];
-        for (err, reason) in cases {
-            assert_eq!(PullFailureReason::from(&err), reason, "{err:?}");
-            assert_eq!(serde_json::to_string(&reason).unwrap(), format!("\"{}\"", reason.as_str()));
-        }
-    }
-
-    #[test]
-    fn label_lists_are_exhaustive_and_unique() {
-        let outcomes: std::collections::BTreeSet<_> = OdmOutcome::ALL.iter().map(|o| o.as_str()).collect();
-        assert_eq!(outcomes.len(), OdmOutcome::ALL.len());
-        let reasons: std::collections::BTreeSet<_> = PullFailureReason::ALL.iter().map(|r| r.as_str()).collect();
-        assert_eq!(reasons.len(), PullFailureReason::ALL.len());
-        let paths: std::collections::BTreeSet<_> = PullPath::ALL.iter().map(|p| p.as_str()).collect();
-        assert_eq!(paths.len(), PullPath::ALL.len());
-    }
-}

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -66,7 +66,6 @@ use crate::disk::new_disk;
 use crate::multipart_listing::paginate_multipart_listing;
 #[cfg(test)]
 use crate::object_api::ObjectLockConfigSnapshot;
-use crate::set_disk::core::io_primitives::finish_rename_tail_heal;
 use crate::set_disk::mem;
 use crate::set_disk::metadata_sys;
 use crate::set_disk::runtime_sources;
@@ -3124,8 +3123,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         let commit_object_lock_guard = object_lock_guard.take();
         let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
         let commit_decommission_capacity_guard = decommission_capacity_guard.take();
-        let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation())
-            && (commit_object_lock_guard.is_some() || commit_decommission_object_lock_guard.is_some());
+        // CompleteMultipartUpload is an S3 publication boundary: after a
+        // successful response, the object must be immediately readable and
+        // usable as a CopyObject source. Do not return on rename quorum while
+        // a tail owner may still hold the object guard and finish shard moves.
+        let commit_allows_early_ack = false;
         let detach_commit_owner = commit_allows_early_ack || upload_guard.is_some() || quota_mutation_fence;
         let commit = async move {
             let mut _object_lock_guard = commit_object_lock_guard;
@@ -3256,105 +3258,15 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                 commit_allows_early_ack,
             )
             .await;
-            let mut rename_guard_release = None;
-            let mut needs_immediate_heal = false;
-            let mut tail_owns_staging_cleanup = false;
             if let Ok(rename_commit) = rename_result.as_mut() {
                 commit_set.record_capacity_scope_if_needed(commit_capacity_scope_token, &rename_commit.capacity_disks);
-                // Install the tail watcher before any post-commit await. The
-                // latch keeps namespace guards through their prior handoff point.
-                needs_immediate_heal = rename_commit.needs_immediate_heal();
-                if let Some(rename_tail_drain) = rename_commit.tail_drain.take() {
-                    tail_owns_staging_cleanup = true;
-                    let mut request = rustfs_heal_contracts::heal_channel::create_heal_request_with_options(
-                        commit_bucket.clone(),
-                        Some(commit_object.clone()),
-                        false,
-                        Some(HealChannelPriority::Normal),
-                        Some(commit_set.pool_index),
-                        Some(commit_set.set_index),
-                    );
-                    request.object_version_id = fi
-                        .version_id
-                        .or_else(|| commit_version_suspended.then(Uuid::nil))
-                        .map(|version_id| version_id.to_string());
-                    let object_lock_guard = _object_lock_guard.take();
-                    let upload_guard = _upload_guard.take();
-                    let decommission_object_lock_guard = _decommission_object_lock_guard.take();
-                    let decommission_capacity_guard = _decommission_capacity_guard.take();
-                    let cleanup_bucket = commit_bucket.clone();
-                    let cleanup_object = commit_object.clone();
-                    let heal_set = commit_set.clone();
-                    let cleanup_set = commit_set.clone();
-                    let committed_data_dir = fi.data_dir;
-                    let cleanup_parts = parts.clone();
-                    let cleanup_upload_path = commit_upload_id_path.clone();
-                    let cleanup_upload_id = commit_upload_id.clone();
-                    let fence_disks = commit_disks.clone();
-                    let fence_tokens = quota_fence_tokens.clone();
-                    let fence_bucket = commit_bucket.clone();
-                    let fence_object = commit_object.clone();
-                    let (guard_release_tx, guard_release_rx) = tokio::sync::oneshot::channel();
-                    rename_guard_release = Some(guard_release_tx);
-                    tokio::spawn(finish_rename_tail_heal(
-                        rename_tail_drain,
-                        guard_release_rx,
-                        (
-                            object_lock_guard,
-                            upload_guard,
-                            decommission_object_lock_guard,
-                            decommission_capacity_guard,
-                        ),
-                        request,
-                        move || async move {
-                            if quota_mutation_fence {
-                                let _ = SetDisks::release_quota_mutation_fences(
-                                    &fence_disks,
-                                    &fence_tokens,
-                                    &fence_bucket,
-                                    &fence_object,
-                                    write_quorum,
-                                )
-                                .await;
-                            }
-                        },
-                        move |(object_lock_guard, upload_guard, decommission_object_lock_guard, decommission_capacity_guard),
-                              targets| async move {
-                            drop(object_lock_guard);
-                            cleanup_set.cleanup_multipart_path(&cleanup_parts).await;
-                            cleanup_set
-                                .cleanup_rename_tail(
-                                    targets,
-                                    &cleanup_bucket,
-                                    &cleanup_object,
-                                    committed_data_dir,
-                                    transaction_epoch,
-                                )
-                                .await;
-                            if let Err(err) = cleanup_set
-                                .delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &cleanup_upload_path, write_quorum)
-                                .await
-                            {
-                                warn!(
-                                    bucket = %cleanup_bucket,
-                                    object = %cleanup_object,
-                                    upload_id = %cleanup_upload_id,
-                                    error = ?err,
-                                    "completed multipart upload staging cleanup did not reach write quorum"
-                                );
-                            }
-                            drop(upload_guard);
-                            drop(decommission_object_lock_guard);
-                            drop(decommission_capacity_guard);
-                        },
-                        |request| async move { heal_set.submit_rename_tail_heal(request).await },
-                    ));
-                }
+                debug_assert!(
+                    rename_commit.tail_drain.is_none(),
+                    "multipart completion disables early ACK and must not detach a rename tail"
+                );
             }
-            if !tail_owns_staging_cleanup {
-                drop(_decommission_capacity_guard.take());
-            }
-            if quota_mutation_fence && !tail_owns_staging_cleanup {
+            drop(_decommission_capacity_guard.take());
+            if quota_mutation_fence {
                 let _ = SetDisks::release_quota_mutation_fences(
                     &commit_disks,
                     &quota_fence_tokens,
@@ -3371,6 +3283,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                 Ok(result) => result,
                 Err(err) => return Err(err.into()),
             };
+            let needs_immediate_heal = rename_commit.needs_immediate_heal();
             let op_old_dir = rename_commit.data_dir;
             let cleanup_disks = rename_commit.cleanup_disks;
             let committed_file_info = rename_commit.committed_file_info;
@@ -3413,9 +3326,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             // parts are swept by a retried completion or upload GC (rustfs/backlog#946).
             // Compiles to a no-op outside `#[cfg(test)]`.
             if crash_inject::should_crash_at(CrashPoint::MultipartAfterCommitBeforePartsCleanup, &commit_object) {
-                if let Some(release) = rename_guard_release.take() {
-                    let _ = release.send(false);
-                }
                 return Err(StorageError::Unexpected);
             }
 
@@ -3431,10 +3341,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                 .invalidate_get_object_metadata_cache(&commit_bucket, &commit_object)
                 .await;
 
-            if let Some(release) = rename_guard_release.take() {
-                let _ = release.send(true);
-            }
-            drop(_object_lock_guard.take()); // release the object lock before multipart cleanup tail IO.
+            drop(_object_lock_guard.take()); // release the object lock before multipart cleanup IO.
 
             #[cfg(test)]
             pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::AfterObjectPublication).await;
@@ -3447,9 +3354,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             // parts; deleting them before the commit would strand the upload
             // permanently. This mirrors the "clean up only after commit" pattern
             // already used for the old data-dir GC and the upload-dir delete_all below.
-            if !tail_owns_staging_cleanup {
-                commit_set.cleanup_multipart_path(&parts).await;
-            }
+            commit_set.cleanup_multipart_path(&parts).await;
 
             if let Some(old_dir) = op_old_dir {
                 // backlog#898: best-effort reclaim of the dereferenced old data dir.
@@ -3480,10 +3385,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             #[cfg(test)]
             pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::AfterRename).await;
 
-            if !tail_owns_staging_cleanup
-                && let Err(err) = commit_set
-                    .delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &commit_upload_id_path, write_quorum)
-                    .await
+            if let Err(err) = commit_set
+                .delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &commit_upload_id_path, write_quorum)
+                .await
             {
                 warn!(
                     bucket = %commit_bucket,
@@ -4010,7 +3914,7 @@ mod tests {
 
     #[tokio::test]
     #[serial(capacity_dirty_scope)]
-    async fn early_ack_multipart_holds_quota_fences_and_re_marks_capacity_after_tail_drain() {
+    async fn complete_multipart_waits_for_tail_before_releasing_guards_and_marking_capacity() {
         use rustfs_object_capacity::capacity_scope::drain_global_dirty_scopes;
 
         temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
@@ -4056,10 +3960,9 @@ mod tests {
                 .collect::<HashSet<_>>();
             let _ = drain_global_dirty_scopes();
 
-            let rename_tasks = rename_fanout_barrier::observe_tasks(object);
             let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
             let complete_store = Arc::clone(&set_disks);
-            let complete = tokio::spawn(async move {
+            let mut complete = tokio::spawn(async move {
                 let mut opts = ObjectOptions::default();
                 assert!(opts.set_quota_admission(0, u64::MAX));
                 complete_store
@@ -4069,20 +3972,15 @@ mod tests {
             tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
                 .await
                 .expect("multipart completion should pause one tail disk during rename");
-            let cleanup_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_CLEANUP);
-            complete
-                .await
-                .expect("early-ACK multipart task should join before tail release")
-                .expect("multipart completion should return after write quorum");
             assert!(
-                rename_tasks.running() >= 1,
-                "the paused multipart tail disk must remain in flight after quorum ACK"
+                tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
+                "multipart completion must not publish success while a tail rename is still paused"
             );
 
             let initial = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
             assert!(
-                expected.is_subset(&initial),
-                "the multipart quorum ACK must mark every candidate disk dirty"
+                initial.is_empty(),
+                "capacity must not be marked as committed before the full multipart rename finishes"
             );
 
             let abort_store = Arc::clone(&set_disks);
@@ -4092,7 +3990,7 @@ mod tests {
                     .await
             });
             signaling.wait_for_attempts(2).await;
-            assert!(!abort.is_finished(), "the detached tail owner must retain the multipart upload guard");
+            assert!(!abort.is_finished(), "the in-flight completion must retain the multipart upload guard");
 
             let retained_staging = futures::future::join_all(
                 disk_stores
@@ -4117,25 +4015,20 @@ mod tests {
                     .await
             });
             signaling.wait_for_attempts(object_attempt).await;
-            assert!(!object_probe.is_finished(), "the detached tail owner must retain the object guard");
+            assert!(!object_probe.is_finished(), "the in-flight completion must retain the object guard");
 
             rename_barrier.release();
+            complete
+                .await
+                .expect("multipart task should join after tail release")
+                .expect("multipart completion should return after every tail rename finishes");
             object_probe
                 .await
-                .expect("object guard probe should join after the tail releases")
-                .expect("object guard probe should acquire after the tail releases");
-            tokio::time::timeout(Duration::from_secs(30), cleanup_barrier.wait_until_paused())
-                .await
-                .expect("the multipart tail should pause before reclaiming its old body");
-            let after_tail = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
-            assert!(
-                expected.is_subset(&after_tail),
-                "the multipart rename tail must re-mark capacity after the first scope was drained"
-            );
-            cleanup_barrier.release();
+                .expect("object guard probe should join after completion releases")
+                .expect("object guard probe should acquire after completion releases");
             let abort_err = abort
                 .await
-                .expect("abort task should join after the tail releases")
+                .expect("abort task should join after completion releases")
                 .expect_err("the committed upload should no longer exist");
             assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
 
@@ -4148,7 +4041,7 @@ mod tests {
             let after_cleanup = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
             assert!(
                 expected.is_subset(&after_cleanup),
-                "the multipart tail cleanup must re-mark capacity after its preceding scope was drained"
+                "the completed multipart commit must mark every candidate disk dirty"
             );
         })
         .await;
@@ -4283,23 +4176,26 @@ mod tests {
             ],
             async {
                 let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
-                set_disks
-                    .clone()
-                    .complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &ObjectOptions::default())
-                    .await
-                    .expect("fenced multipart completion should commit with a live proof");
+                let complete_store = Arc::clone(&set_disks);
+                let mut complete = tokio::spawn(async move {
+                    complete_store
+                        .complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &ObjectOptions::default())
+                        .await
+                });
 
                 tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
                     .await
-                    .expect("multipart completion should leave one rename tail in flight after quorum ACK");
-                let disks = disk_stores.clone();
-                let mut epochs = tokio::spawn(async move { object_transaction_epochs(&disks, bucket, object).await });
+                    .expect("multipart completion should pause one tail disk during rename");
                 assert!(
-                    tokio::time::timeout(Duration::from_millis(100), &mut epochs).await.is_err(),
-                    "epoch read-back should wait for the lagging rename tail"
+                    tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
+                    "fenced multipart completion must wait for every rename tail before returning"
                 );
                 rename_barrier.release();
-                epochs.await.expect("epoch read-back should finish after the rename tail")
+                complete
+                    .await
+                    .expect("fenced multipart task should join after tail release")
+                    .expect("fenced multipart completion should commit with a live proof");
+                object_transaction_epochs(&disk_stores, bucket, object).await
             },
         )
         .await;
@@ -8392,29 +8288,18 @@ mod tests {
             let new = payload(0xC3);
             let (u_new, parts_new) = stage_upload(&set_disks, bucket, object, &new).await;
             let parts_retry = parts_new.clone();
-            let rename_tasks = rename_fanout_barrier::observe_tasks(object);
-            let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
             crash_inject::arm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
             let crashed = complete(&set_disks, bucket, object, &u_new, parts_new).await;
             assert!(
                 matches!(crashed, Err(StorageError::Unexpected)),
                 "the armed post-commit crash point must be the failure that surfaced, got {crashed:?}"
             );
-            assert!(rename_tasks.running() >= 1, "the crash must interrupt an actual early-ACK tail handoff");
             crash_inject::disarm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
-            rename_barrier.release();
-            tokio::time::timeout(Duration::from_secs(30), async {
-                while rename_tasks.running() != 0 {
-                    tokio::task::yield_now().await;
-                }
-            })
-            .await
-            .expect("the crash-interrupted rename tail should drain after release");
             drop(
                 set_disks
                     .acquire_write_lock_diag("post_commit_crash_tail_probe", bucket, object)
                     .await
-                    .expect("the crash-interrupted tail should release its object guard"),
+                    .expect("the failed post-commit completion should release its object guard"),
             );
 
             // The commit landed: the new version reads back whole and correct.
@@ -8487,29 +8372,18 @@ mod tests {
 
                     let new = payload(0x52);
                     let (u_new, parts_new) = stage_upload(&set_disks, bucket, object, &new).await;
-                    let rename_tasks = rename_fanout_barrier::observe_tasks(object);
-                    let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
                     crash_inject::arm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
                     let crashed = complete(&set_disks, bucket, object, &u_new, parts_new).await;
                     assert!(
                         matches!(crashed, Err(StorageError::Unexpected)),
                         "the post-commit crash point must surface as unexpected, got {crashed:?}"
                     );
-                    assert!(rename_tasks.running() >= 1, "the crash must interrupt an actual early-ACK tail handoff");
                     crash_inject::disarm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
-                    rename_barrier.release();
-                    tokio::time::timeout(Duration::from_secs(30), async {
-                        while rename_tasks.running() != 0 {
-                            tokio::task::yield_now().await;
-                        }
-                    })
-                    .await
-                    .expect("the crash-interrupted rename tail should drain after release");
                     drop(
                         set_disks
                             .acquire_write_lock_diag("post_commit_receipt_tail_probe", bucket, object)
                             .await
-                            .expect("the crash-interrupted tail should release its object guard"),
+                            .expect("the failed post-commit completion should release its object guard"),
                     );
 
                     let (body, _) = read_object(&set_disks, bucket, object).await;
@@ -8523,8 +8397,8 @@ mod tests {
                         );
                     }
                     assert_eq!(
-                        receipts, 3,
-                        "the committed quorum must persist receipts while the crash-interrupted tail preserves staging"
+                        receipts, 4,
+                        "the completed rename must persist old-data cleanup receipts on every disk before surfacing the post-commit crash"
                     );
 
                     let restarted_endpoints = temp_dirs
@@ -8570,12 +8444,12 @@ mod tests {
                         .reconcile_old_data_cleanup_receipts(bucket, object)
                         .await
                         .expect("restart receipt reconciliation should succeed");
-                    assert_eq!(removed, 3, "restart receipt reconciliation should delete the committed quorum's targets");
+                    assert_eq!(removed, 4, "restart receipt reconciliation should delete every committed target");
                     let reclaimed = restarted_set
                         .reclaim_orphan_data_dirs(bucket, object)
                         .await
                         .expect("restart orphan reconciliation should succeed");
-                    assert_eq!(reclaimed, 1, "the late commit without a receipt must remain reclaimable as an orphan");
+                    assert_eq!(reclaimed, 0, "the post-commit crash should leave no receipt-less late commit orphan");
                     for disk in &reloaded {
                         assert!(
                             !data_dir_exists(disk, bucket, object, old_dir).await,

crates/ecstore/src/store/init.rs

@@ -2305,6 +2305,200 @@ mod tests {
         shutdown.cancel();
     }
 
+    #[cfg(feature = "test-util")]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    #[serial_test::serial(storage_class_env)]
+    async fn copy_object_immediately_reads_small_completed_multipart_source() {
+        let temp_dir = tempfile::tempdir().expect("create small multipart copy store dir");
+        let (_ctx, store, shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "small-multipart-copy", &[1])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+
+        let bucket = format!("small-multipart-copy-{}", Uuid::new_v4());
+        let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
+        let target_object = "docker/registry/v2/blobs/sha256/c0/digest/data";
+        let payload = vec![0xAB; 273];
+
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket for small multipart copy");
+        let upload = store
+            .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
+            .await
+            .expect("create source multipart upload");
+        let mut part_reader = PutObjReader::from_vec(payload.clone());
+        let part = store
+            .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
+            .await
+            .expect("stage small multipart source part");
+        let completed = store
+            .clone()
+            .complete_multipart_upload(
+                &bucket,
+                source_object,
+                &upload.upload_id,
+                vec![crate::storage_api_contracts::multipart::CompletePart {
+                    part_num: part.part_num,
+                    etag: part.etag,
+                    ..Default::default()
+                }],
+                &ObjectOptions::default(),
+            )
+            .await
+            .expect("complete the small multipart source");
+        assert_eq!(completed.get_actual_size().expect("completed object logical size"), payload.len() as i64);
+
+        let source_reader = store
+            .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("completed multipart source should be immediately readable");
+        let mut copy_info = source_reader.object_info.clone();
+        let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
+        assert_eq!(actual_size, payload.len() as i64);
+        let copy_reader = rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
+            .expect("copy source hash reader should build");
+        copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
+
+        store
+            .copy_object(
+                &bucket,
+                source_object,
+                &bucket,
+                target_object,
+                &mut copy_info,
+                &ObjectOptions::default(),
+                &ObjectOptions::default(),
+            )
+            .await
+            .expect("CopyObject should accept a freshly completed multipart source");
+
+        let mut target_reader = store
+            .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("copied target should be readable");
+        let mut target_body = Vec::new();
+        target_reader
+            .stream
+            .read_to_end(&mut target_body)
+            .await
+            .expect("target body should stream");
+        assert_eq!(target_body, payload);
+        shutdown.cancel();
+    }
+
+    #[cfg(feature = "test-util")]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    #[serial_test::serial(storage_class_env)]
+    async fn complete_multipart_waits_for_tail_rename_before_copy_source_visibility() {
+        let temp_dir = tempfile::tempdir().expect("create early-ack multipart copy store dir");
+        let (_ctx, store, shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "early-ack-multipart-copy", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+
+        let bucket = format!("early-ack-multipart-copy-{}", Uuid::new_v4());
+        let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
+        let target_object = "docker/registry/v2/blobs/sha256/c0/digest/data";
+        let payload = vec![0xCD; 273];
+
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket for early-ack multipart copy");
+        let upload = store
+            .new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
+            .await
+            .expect("create source multipart upload");
+        let mut part_reader = PutObjReader::from_vec(payload.clone());
+        let part = store
+            .put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
+            .await
+            .expect("stage small multipart source part");
+        let completed_parts = vec![crate::storage_api_contracts::multipart::CompletePart {
+            part_num: part.part_num,
+            etag: part.etag,
+            ..Default::default()
+        }];
+
+        temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
+            let rename_tasks = crate::set_disk::rename_fanout_barrier::observe_tasks(source_object);
+            let rename_barrier = crate::set_disk::rename_fanout_barrier::arm(
+                source_object,
+                0,
+                crate::set_disk::rename_fanout_barrier::PHASE_RENAME,
+            );
+            let complete_store = Arc::clone(&store);
+            let complete_bucket = bucket.clone();
+            let complete_upload_id = upload.upload_id.clone();
+            let mut complete = tokio::spawn(async move {
+                complete_store
+                    .complete_multipart_upload(
+                        &complete_bucket,
+                        source_object,
+                        &complete_upload_id,
+                        completed_parts,
+                        &ObjectOptions::default(),
+                    )
+                    .await
+            });
+            tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
+                .await
+                .expect("multipart completion should pause one tail disk during rename");
+            assert!(
+                rename_tasks.running() >= 1,
+                "the paused multipart tail disk must remain in flight before completion returns"
+            );
+            assert!(
+                tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
+                "CompleteMultipartUpload must not return while a copy source rename tail is still pending"
+            );
+            rename_barrier.release();
+            complete
+                .await
+                .expect("multipart completion task should join")
+                .expect("multipart completion should return after every rename tail finishes");
+
+            let source_reader = store
+                .get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
+                .await
+                .expect("completed multipart source should be immediately readable after success");
+            let mut copy_info = source_reader.object_info.clone();
+            let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
+            assert_eq!(actual_size, payload.len() as i64);
+            let copy_reader =
+                rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
+                    .expect("copy source hash reader should build");
+            copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
+
+            store
+                .copy_object(
+                    &bucket,
+                    source_object,
+                    &bucket,
+                    target_object,
+                    &mut copy_info,
+                    &ObjectOptions::default(),
+                    &ObjectOptions::default(),
+                )
+                .await
+                .expect("CopyObject should accept a freshly completed multipart source");
+        })
+        .await;
+
+        let mut target_reader = store
+            .get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("copied target should be readable after tail release");
+        let mut target_body = Vec::new();
+        target_reader
+            .stream
+            .read_to_end(&mut target_body)
+            .await
+            .expect("target body should stream");
+        assert_eq!(target_body, payload);
+        shutdown.cancel();
+    }
+
     #[cfg(feature = "test-util")]
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     #[serial_test::serial(storage_class_env)]

crates/protocols/src/common/client/s3.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use async_trait::async_trait;
+use rustfs_credentials::Credentials;
 use s3s::dto::*;
 
 #[cfg(feature = "webdav")]
@@ -27,49 +28,33 @@ pub trait StorageBackend: Send + Sync {
         &self,
         bucket: &str,
         key: &str,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &Credentials,
         start_pos: Option<u64>,
     ) -> Result<GetObjectOutput, Self::Error>;
     async fn get_object_range(
         &self,
         bucket: &str,
         key: &str,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &Credentials,
         start_pos: u64,
         length: u64,
     ) -> Result<GetObjectOutput, Self::Error>;
     /// Put object content with metadata
-    async fn put_object(&self, input: PutObjectInput, access_key: &str, secret_key: &str)
-    -> Result<PutObjectOutput, Self::Error>;
+    async fn put_object(&self, input: PutObjectInput, credentials: &Credentials) -> Result<PutObjectOutput, Self::Error>;
     /// Delete an object
-    async fn delete_object(
-        &self,
-        bucket: &str,
-        key: &str,
-        access_key: &str,
-        secret_key: &str,
-    ) -> Result<DeleteObjectOutput, Self::Error>;
+    async fn delete_object(&self, bucket: &str, key: &str, credentials: &Credentials) -> Result<DeleteObjectOutput, Self::Error>;
     /// Get object metadata without content
-    async fn head_object(
-        &self,
-        bucket: &str,
-        key: &str,
-        access_key: &str,
-        secret_key: &str,
-    ) -> Result<HeadObjectOutput, Self::Error>;
+    async fn head_object(&self, bucket: &str, key: &str, credentials: &Credentials) -> Result<HeadObjectOutput, Self::Error>;
     /// Check if bucket exists and get metadata
-    async fn head_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<HeadBucketOutput, Self::Error>;
+    async fn head_bucket(&self, bucket: &str, credentials: &Credentials) -> Result<HeadBucketOutput, Self::Error>;
     /// List objects in a bucket with pagination
     async fn list_objects_v2(
         &self,
         input: ListObjectsV2Input,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &Credentials,
     ) -> Result<ListObjectsV2Output, Self::Error>;
     /// List all buckets (requires authentication).
-    async fn list_buckets(&self, access_key: &str, secret_key: &str) -> Result<ListBucketsOutput, Self::Error>;
+    async fn list_buckets(&self, credentials: &Credentials) -> Result<ListBucketsOutput, Self::Error>;
     /// List buckets visible to the authenticated session.
     ///
     /// Backends that implement this must apply per-bucket authorization. The default denies the
@@ -87,20 +72,15 @@ pub trait StorageBackend: Send + Sync {
         ))
     }
     /// Create a new bucket
-    async fn create_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<CreateBucketOutput, Self::Error>;
+    async fn create_bucket(&self, bucket: &str, credentials: &Credentials) -> Result<CreateBucketOutput, Self::Error>;
     /// Delete a bucket (must be empty)
-    async fn delete_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<DeleteBucketOutput, Self::Error>;
+    async fn delete_bucket(&self, bucket: &str, credentials: &Credentials) -> Result<DeleteBucketOutput, Self::Error>;
     /// Server-side copy of an object from one bucket+key to another.
     /// The input carries the full S3 surface (content type, metadata map,
     /// metadata directive, storage class, SSE config, conditional-copy
     /// headers) so protocol drivers can map client-supplied metadata
     /// onto the destination object.
-    async fn copy_object(
-        &self,
-        input: CopyObjectInput,
-        access_key: &str,
-        secret_key: &str,
-    ) -> Result<CopyObjectOutput, Self::Error>;
+    async fn copy_object(&self, input: CopyObjectInput, credentials: &Credentials) -> Result<CopyObjectOutput, Self::Error>;
     /// Initiate a multipart upload. Returns an upload_id that identifies
     /// the in-progress upload for subsequent UploadPart, CompleteMultipartUpload,
     /// and AbortMultipartUpload calls. The input carries the full S3 surface
@@ -110,25 +90,18 @@ pub trait StorageBackend: Send + Sync {
     async fn create_multipart_upload(
         &self,
         input: CreateMultipartUploadInput,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &Credentials,
     ) -> Result<CreateMultipartUploadOutput, Self::Error>;
     /// Upload one part of a multipart upload. The part_number must be in
     /// the range 1 to the 10 000-part S3 limit. The returned ETag
     /// identifies the part in the subsequent CompleteMultipartUpload call.
-    async fn upload_part(
-        &self,
-        input: UploadPartInput,
-        access_key: &str,
-        secret_key: &str,
-    ) -> Result<UploadPartOutput, Self::Error>;
+    async fn upload_part(&self, input: UploadPartInput, credentials: &Credentials) -> Result<UploadPartOutput, Self::Error>;
     /// Assemble the parts listed in the input into the final object.
     /// The parts list must be sorted by part_number with no duplicates.
     async fn complete_multipart_upload(
         &self,
         input: CompleteMultipartUploadInput,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &Credentials,
     ) -> Result<CompleteMultipartUploadOutput, Self::Error>;
     /// Abort an in-progress multipart upload. Releases any storage
     /// associated with the upload_id. Idempotent: calling abort on an
@@ -138,8 +111,7 @@ pub trait StorageBackend: Send + Sync {
     async fn abort_multipart_upload(
         &self,
         input: AbortMultipartUploadInput,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &Credentials,
     ) -> Result<AbortMultipartUploadOutput, Self::Error>;
     /// Copy a byte range from an existing object into a part of an
     /// in-progress multipart upload. Used by rename for objects larger
@@ -147,7 +119,6 @@ pub trait StorageBackend: Send + Sync {
     async fn upload_part_copy(
         &self,
         input: UploadPartCopyInput,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &Credentials,
     ) -> Result<UploadPartCopyOutput, Self::Error>;
 }

crates/protocols/src/common/dummy_storage.rs

@@ -35,6 +35,7 @@ use crate::common::session::SessionContext;
 use async_trait::async_trait;
 use bytes::Bytes;
 use futures_util::stream::{self, StreamExt};
+use rustfs_credentials::Credentials;
 use s3s::dto::{
     AbortMultipartUploadInput, AbortMultipartUploadOutput, CompleteMultipartUploadInput, CompleteMultipartUploadOutput,
     CopyObjectInput, CopyObjectOutput, CopyPartResult, CreateBucketOutput, CreateMultipartUploadInput,
@@ -605,8 +606,7 @@ impl StorageBackend for DummyBackend {
         &self,
         bucket: &str,
         key: &str,
-        _ak: &str,
-        _sk: &str,
+        _credentials: &Credentials,
         _start_pos: Option<u64>,
     ) -> Result<GetObjectOutput, Self::Error> {
         match self.inner.lock().expect("lock").get_object.pop_front() {
@@ -619,8 +619,7 @@ impl StorageBackend for DummyBackend {
         &self,
         bucket: &str,
         key: &str,
-        _ak: &str,
-        _sk: &str,
+        _credentials: &Credentials,
         _start_pos: u64,
         _length: u64,
     ) -> Result<GetObjectOutput, Self::Error> {
@@ -630,7 +629,7 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn put_object(&self, input: PutObjectInput, _ak: &str, _sk: &str) -> Result<PutObjectOutput, Self::Error> {
+    async fn put_object(&self, input: PutObjectInput, _credentials: &Credentials) -> Result<PutObjectOutput, Self::Error> {
         // Decide control flow while holding the lock. Release before
         // awaiting so the stall path does not hold the Mutex across
         // an await point.
@@ -659,7 +658,12 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn delete_object(&self, bucket: &str, key: &str, _ak: &str, _sk: &str) -> Result<DeleteObjectOutput, Self::Error> {
+    async fn delete_object(
+        &self,
+        bucket: &str,
+        key: &str,
+        _credentials: &Credentials,
+    ) -> Result<DeleteObjectOutput, Self::Error> {
         let mut inner = self.inner.lock().expect("lock");
         inner.delete_object_calls.push(DeleteObjectCall {
             bucket: bucket.to_string(),
@@ -671,7 +675,7 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn head_object(&self, bucket: &str, key: &str, _ak: &str, _sk: &str) -> Result<HeadObjectOutput, Self::Error> {
+    async fn head_object(&self, bucket: &str, key: &str, _credentials: &Credentials) -> Result<HeadObjectOutput, Self::Error> {
         {
             let mut inner = self.inner.lock().expect("lock");
             inner.head_object_calls.push(HeadObjectCall {
@@ -685,7 +689,7 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn head_bucket(&self, bucket: &str, _ak: &str, _sk: &str) -> Result<HeadBucketOutput, Self::Error> {
+    async fn head_bucket(&self, bucket: &str, _credentials: &Credentials) -> Result<HeadBucketOutput, Self::Error> {
         match self.inner.lock().expect("lock").head_bucket.pop_front() {
             Some(r) => r,
             None => Err(DummyError::NoSuchBucket(bucket.to_string())),
@@ -695,8 +699,7 @@ impl StorageBackend for DummyBackend {
     async fn list_objects_v2(
         &self,
         _input: ListObjectsV2Input,
-        _ak: &str,
-        _sk: &str,
+        _credentials: &Credentials,
     ) -> Result<ListObjectsV2Output, Self::Error> {
         // Decide control flow while holding the lock. Release before
         // awaiting so the stall path does not hold the Mutex across
@@ -721,7 +724,7 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn list_buckets(&self, _access_key: &str, _secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
+    async fn list_buckets(&self, _credentials: &Credentials) -> Result<ListBucketsOutput, Self::Error> {
         match self.inner.lock().expect("lock").list_buckets.pop_front() {
             Some(r) => r,
             None => Ok(ListBucketsOutput::default()),
@@ -744,14 +747,14 @@ impl StorageBackend for DummyBackend {
             .unwrap_or_else(|| Ok(ListBucketsOutput::default()))
     }
 
-    async fn create_bucket(&self, _bucket: &str, _ak: &str, _sk: &str) -> Result<CreateBucketOutput, Self::Error> {
+    async fn create_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<CreateBucketOutput, Self::Error> {
         match self.inner.lock().expect("lock").create_bucket.pop_front() {
             Some(r) => r,
             None => Err(DummyError::Unconfigured("create_bucket")),
         }
     }
 
-    async fn delete_bucket(&self, bucket: &str, _ak: &str, _sk: &str) -> Result<DeleteBucketOutput, Self::Error> {
+    async fn delete_bucket(&self, bucket: &str, _credentials: &Credentials) -> Result<DeleteBucketOutput, Self::Error> {
         let mut inner = self.inner.lock().expect("lock");
         inner.delete_bucket_calls.push(bucket.to_string());
         match inner.delete_bucket.pop_front() {
@@ -760,7 +763,7 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn copy_object(&self, _input: CopyObjectInput, _ak: &str, _sk: &str) -> Result<CopyObjectOutput, Self::Error> {
+    async fn copy_object(&self, _input: CopyObjectInput, _credentials: &Credentials) -> Result<CopyObjectOutput, Self::Error> {
         match self.inner.lock().expect("lock").copy_object.pop_front() {
             Some(r) => r,
             None => Err(DummyError::Unconfigured("copy_object")),
@@ -770,8 +773,7 @@ impl StorageBackend for DummyBackend {
     async fn create_multipart_upload(
         &self,
         input: CreateMultipartUploadInput,
-        _ak: &str,
-        _sk: &str,
+        _credentials: &Credentials,
     ) -> Result<CreateMultipartUploadOutput, Self::Error> {
         {
             let mut inner = self.inner.lock().expect("lock");
@@ -787,7 +789,7 @@ impl StorageBackend for DummyBackend {
         }
     }
 
-    async fn upload_part(&self, input: UploadPartInput, _ak: &str, _sk: &str) -> Result<UploadPartOutput, Self::Error> {
+    async fn upload_part(&self, input: UploadPartInput, _credentials: &Credentials) -> Result<UploadPartOutput, Self::Error> {
         // Record the call and decide the control flow while holding the
         // lock. Release the lock before awaiting so the stall path does
         // not hold the Mutex across an await point.
@@ -821,8 +823,7 @@ impl StorageBackend for DummyBackend {
     async fn complete_multipart_upload(
         &self,
         input: CompleteMultipartUploadInput,
-        _ak: &str,
-        _sk: &str,
+        _credentials: &Credentials,
     ) -> Result<CompleteMultipartUploadOutput, Self::Error> {
         let part_count = input
             .multipart_upload
@@ -847,8 +848,7 @@ impl StorageBackend for DummyBackend {
     async fn abort_multipart_upload(
         &self,
         input: AbortMultipartUploadInput,
-        _ak: &str,
-        _sk: &str,
+        _credentials: &Credentials,
     ) -> Result<AbortMultipartUploadOutput, Self::Error> {
         {
             let mut inner = self.inner.lock().expect("lock");
@@ -867,8 +867,7 @@ impl StorageBackend for DummyBackend {
     async fn upload_part_copy(
         &self,
         _input: UploadPartCopyInput,
-        _ak: &str,
-        _sk: &str,
+        _credentials: &Credentials,
     ) -> Result<UploadPartCopyOutput, Self::Error> {
         match self.inner.lock().expect("lock").upload_part_copy.pop_front() {
             Some(r) => r,
@@ -884,7 +883,8 @@ mod tests {
     #[tokio::test]
     async fn dummy_backend_reports_not_found_by_default() {
         let backend = DummyBackend::new();
-        let result = backend.head_object("b", "k", "ak", "sk").await;
+        let credentials = Credentials::default();
+        let result = backend.head_object("b", "k", &credentials).await;
         let Err(err) = result else {
             panic!("default head_object must return an error");
         };
@@ -897,21 +897,23 @@ mod tests {
     #[tokio::test]
     async fn dummy_backend_returns_queued_head_object_response() {
         let backend = DummyBackend::new();
+        let credentials = Credentials::default();
         backend.queue_head_object_ok(42, None);
-        let out = backend.head_object("b", "k", "ak", "sk").await.expect("queued Ok");
+        let out = backend.head_object("b", "k", &credentials).await.expect("queued Ok");
         assert_eq!(out.content_length, Some(42));
     }
 
     #[tokio::test]
     async fn dummy_backend_logs_abort_multipart_calls() {
         let backend = Arc::new(DummyBackend::new());
+        let credentials = Credentials::default();
         let input = AbortMultipartUploadInput::builder()
             .bucket("b".to_string())
             .key("k".to_string())
             .upload_id("UP-1".to_string())
             .build()
             .expect("build");
-        backend.abort_multipart_upload(input, "ak", "sk").await.expect("Ok");
+        backend.abort_multipart_upload(input, &credentials).await.expect("Ok");
         let calls = backend.abort_multipart_calls();
         assert_eq!(calls.len(), 1);
         assert_eq!(calls[0].upload_id, "UP-1");
@@ -920,6 +922,7 @@ mod tests {
     #[tokio::test]
     async fn dummy_backend_unconfigured_errors_loudly() {
         let backend = DummyBackend::new();
+        let credentials = Credentials::default();
         let err = backend
             .create_multipart_upload(
                 CreateMultipartUploadInput::builder()
@@ -927,8 +930,7 @@ mod tests {
                     .key("k".to_string())
                     .build()
                     .expect("build"),
-                "ak",
-                "sk",
+                &credentials,
             )
             .await
             .expect_err("default create_multipart_upload must error");

crates/protocols/src/common/gateway.rs

@@ -288,12 +288,7 @@ pub async fn is_authorized(
         }
     };
 
-    // Create policy arguments
-    let mut claims = HashMap::new();
-    claims.insert(
-        "principal".to_string(),
-        serde_json::Value::String(session_context.principal.access_key().to_string()),
-    );
+    let claims = policy_claims_for_session(session_context);
 
     let policy_action: rustfs_policy::policy::action::Action = action.clone().into();
 
@@ -315,6 +310,21 @@ pub async fn is_authorized(
     Ok(iam_sys.is_allowed(&args).await)
 }
 
+fn policy_claims_for_session(session_context: &SessionContext) -> HashMap<String, serde_json::Value> {
+    let mut claims = session_context
+        .principal
+        .user_identity
+        .credentials
+        .claims
+        .clone()
+        .unwrap_or_default();
+    claims.insert(
+        "principal".to_string(),
+        serde_json::Value::String(session_context.principal.access_key().to_string()),
+    );
+    claims
+}
+
 /// Authorize an operation and return an error if not authorized.
 /// AccessDenied covers both the protocol-not-supported case and the
 /// policy-denies case. IamUnavailable propagates from is_authorized
@@ -457,7 +467,9 @@ pub use test_auth_override::{with_test_auth_override, with_test_iam_unavailable}
 mod tests {
     use super::*;
     use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext};
+    use rustfs_credentials::{IAM_POLICY_CLAIM_NAME_SA, INHERITED_POLICY_TYPE};
     use rustfs_policy::auth::UserIdentity;
+    use serde_json::Value;
     use std::net::{IpAddr, Ipv4Addr};
     use std::sync::Arc;
 
@@ -466,6 +478,44 @@ mod tests {
         SessionContext::new(principal, Protocol::Sftp, IpAddr::V4(Ipv4Addr::LOCALHOST))
     }
 
+    fn session_with_claims(access_key: &str, claims: HashMap<String, Value>) -> SessionContext {
+        let identity = UserIdentity::new(rustfs_credentials::Credentials {
+            access_key: access_key.to_string(),
+            secret_key: "secret".to_string(),
+            claims: Some(claims),
+            ..Default::default()
+        });
+        let principal = ProtocolPrincipal::new(Arc::new(identity));
+        SessionContext::new(principal, Protocol::WebDav, IpAddr::V4(Ipv4Addr::LOCALHOST))
+    }
+
+    #[test]
+    fn policy_claims_preserve_authenticated_service_account_claims() {
+        let parent = "parent-user";
+        let mut stored_claims = HashMap::new();
+        stored_claims.insert("parent".to_string(), Value::String(parent.to_string()));
+        stored_claims.insert(IAM_POLICY_CLAIM_NAME_SA.to_string(), Value::String(INHERITED_POLICY_TYPE.to_string()));
+        let session = session_with_claims("service-account", stored_claims);
+
+        let claims = policy_claims_for_session(&session);
+
+        assert_eq!(claims.get("parent").and_then(Value::as_str), Some(parent));
+        assert_eq!(claims.get(IAM_POLICY_CLAIM_NAME_SA).and_then(Value::as_str), Some(INHERITED_POLICY_TYPE));
+        assert_eq!(claims.get("principal").and_then(Value::as_str), Some("service-account"));
+    }
+
+    #[test]
+    fn policy_claims_overwrite_untrusted_principal_claim() {
+        let session = session_with_claims(
+            "authenticated-service-account",
+            HashMap::from([("principal".to_string(), Value::String("forged-principal".to_string()))]),
+        );
+
+        let claims = policy_claims_for_session(&session);
+
+        assert_eq!(claims.get("principal").and_then(Value::as_str), Some("authenticated-service-account"));
+    }
+
     #[tokio::test]
     async fn with_test_auth_override_allow_returns_ok() {
         let session = test_session();

crates/protocols/src/common/session.rs

@@ -84,6 +84,11 @@ impl SessionContext {
     pub fn access_key(&self) -> &str {
         self.principal.access_key()
     }
+
+    /// Get the authenticated credentials for this session.
+    pub fn credentials(&self) -> &Credentials {
+        &self.principal.user_identity.credentials
+    }
 }
 
 /// Build a SessionContext suitable for driver-level unit tests. The

crates/protocols/src/ftps/driver.rs

@@ -129,14 +129,7 @@ where
         }
 
         let mut list_result = Vec::new();
-        match self
-            .storage
-            .list_buckets(
-                &session_context.principal.user_identity.credentials.access_key,
-                &session_context.principal.user_identity.credentials.secret_key,
-            )
-            .await
-        {
+        match self.storage.list_buckets(session_context.credentials()).await {
             Ok(output) => {
                 if let Some(buckets) = output.buckets {
                     for bucket in buckets {
@@ -190,15 +183,7 @@ where
                 Error::new(ErrorKind::PermanentFileNotAvailable, format!("Failed to build ListObjectsV2Input: {}", e))
             })?;
 
-            if let Ok(output) = self
-                .storage
-                .list_objects_v2(
-                    list_input,
-                    &session_context.principal.user_identity.credentials.access_key,
-                    &session_context.principal.user_identity.credentials.secret_key,
-                )
-                .await
-            {
+            if let Ok(output) = self.storage.list_objects_v2(list_input, session_context.credentials()).await {
                 // Delete all objects in this page
                 if let Some(objects) = output.contents {
                     for obj in objects {
@@ -209,12 +194,7 @@ where
 
                             let _ = self
                                 .storage
-                                .delete_object(
-                                    bucket,
-                                    &obj_key,
-                                    &session_context.principal.user_identity.credentials.access_key,
-                                    &session_context.principal.user_identity.credentials.secret_key,
-                                )
+                                .delete_object(bucket, &obj_key, session_context.credentials())
                                 .await;
                         }
                     }
@@ -231,15 +211,7 @@ where
         }
 
         // Then delete the bucket
-        match self
-            .storage
-            .delete_bucket(
-                bucket,
-                &session_context.principal.user_identity.credentials.access_key,
-                &session_context.principal.user_identity.credentials.secret_key,
-            )
-            .await
-        {
+        match self.storage.delete_bucket(bucket, session_context.credentials()).await {
             Ok(_) => Ok(()),
             Err(e) if e.to_string().contains("NoSuchBucket") => Ok(()),
             Err(e) => {
@@ -277,16 +249,7 @@ where
                 .await
                 .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
 
-            match self
-                .storage
-                .head_object(
-                    &bucket,
-                    &key,
-                    &session_context.principal.user_identity.credentials.access_key,
-                    &session_context.principal.user_identity.credentials.secret_key,
-                )
-                .await
-            {
+            match self.storage.head_object(&bucket, &key, session_context.credentials()).await {
                 Ok(output) => {
                     let size = output.content_length.unwrap_or(0) as u64;
                     let modified = output.last_modified.map(|dt| {
@@ -323,15 +286,7 @@ where
                 .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
 
             let bucket_clone = bucket.clone();
-            match self
-                .storage
-                .head_bucket(
-                    &bucket,
-                    &session_context.principal.user_identity.credentials.access_key,
-                    &session_context.principal.user_identity.credentials.secret_key,
-                )
-                .await
-            {
+            match self.storage.head_bucket(&bucket, session_context.credentials()).await {
                 Ok(_) => Ok(FtpsMetadata {
                     size: 0,
                     modified: Some(std::time::SystemTime::now()),
@@ -390,15 +345,7 @@ where
                 Error::new(ErrorKind::PermanentFileNotAvailable, format!("Failed to build ListObjectsV2Input: {}", e))
             })?;
 
-        match self
-            .storage
-            .list_objects_v2(
-                list_input,
-                &session_context.principal.user_identity.credentials.access_key,
-                &session_context.principal.user_identity.credentials.secret_key,
-            )
-            .await
-        {
+        match self.storage.list_objects_v2(list_input, session_context.credentials()).await {
             Ok(output) => {
                 let mut fileinfos = Vec::new();
 
@@ -515,8 +462,7 @@ where
             .get_object(
                 &bucket,
                 &key,
-                &session_context.principal.user_identity.credentials.access_key,
-                &session_context.principal.user_identity.credentials.secret_key,
+                session_context.credentials(),
                 Some(start_pos), // Pass start_pos for range request
             )
             .await
@@ -624,15 +570,7 @@ where
             .build()
             .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Failed to build PutObjectInput"))?;
 
-        match self
-            .storage
-            .put_object(
-                put_input,
-                &session_context.principal.user_identity.credentials.access_key,
-                &session_context.principal.user_identity.credentials.secret_key,
-            )
-            .await
-        {
+        match self.storage.put_object(put_input, session_context.credentials()).await {
             Ok(_output) => {
                 Ok(file_size as u64) // Return the size of the uploaded object
             }
@@ -681,16 +619,7 @@ where
                 .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
 
             // Delete file
-            match self
-                .storage
-                .delete_object(
-                    &bucket,
-                    &key,
-                    &session_context.principal.user_identity.credentials.access_key,
-                    &session_context.principal.user_identity.credentials.secret_key,
-                )
-                .await
-            {
+            match self.storage.delete_object(&bucket, &key, session_context.credentials()).await {
                 Ok(_) => Ok(()),
                 Err(e) => {
                     error!(
@@ -748,15 +677,7 @@ where
             .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
 
         // Create bucket for directory
-        match self
-            .storage
-            .create_bucket(
-                &bucket,
-                &session_context.principal.user_identity.credentials.access_key,
-                &session_context.principal.user_identity.credentials.secret_key,
-            )
-            .await
-        {
+        match self.storage.create_bucket(&bucket, session_context.credentials()).await {
             Ok(_) => {
                 debug!(
                     event = EVENT_FTPS_DIRECTORY_STATE,
@@ -856,15 +777,7 @@ where
             .map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
 
         // Check if bucket exists
-        match self
-            .storage
-            .head_bucket(
-                &bucket,
-                &session_context.principal.user_identity.credentials.access_key,
-                &session_context.principal.user_identity.credentials.secret_key,
-            )
-            .await
-        {
+        match self.storage.head_bucket(&bucket, session_context.credentials()).await {
             Ok(_) => Ok(()),
             Err(e) => {
                 error!(

crates/protocols/src/sftp/attrs.rs

@@ -137,7 +137,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             // on success. Size and mtime are not returned by HeadBucket.
             None => {
                 self.authorize(&S3Action::HeadBucket, &bucket, None).await?;
-                self.run_backend("head_bucket", self.storage.head_bucket(&bucket, self.access_key(), self.secret_key()))
+                self.run_backend("head_bucket", self.storage.head_bucket(&bucket, self.credentials()))
                     .await?;
                 Ok(s3_attrs_to_sftp(0, None, true))
             }
@@ -154,11 +154,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             Some(object_key) => {
                 self.authorize(&S3Action::HeadObject, &bucket, Some(&object_key)).await?;
                 match self
-                    .run_backend_with_err(
-                        "head_object",
-                        self.storage
-                            .head_object(&bucket, &object_key, self.access_key(), self.secret_key()),
-                    )
+                    .run_backend_with_err("head_object", self.storage.head_object(&bucket, &object_key, self.credentials()))
                     .await?
                 {
                     Ok(out) => {
@@ -183,10 +179,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
                             .build()
                             .map_err(|e| s3_error_to_sftp("build_list_objects", e))?;
                         let out = self
-                            .run_backend(
-                                "list_objects_v2",
-                                self.storage.list_objects_v2(input, self.access_key(), self.secret_key()),
-                            )
+                            .run_backend("list_objects_v2", self.storage.list_objects_v2(input, self.credentials()))
                             .await?;
 
                         let has_contents = out.contents.map(|c| !c.is_empty()).unwrap_or(false);

crates/protocols/src/sftp/dir.rs

@@ -102,10 +102,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
         let input = builder.build().map_err(|e| s3_error_to_sftp("build_list_objects", e))?;
 
         let out = self
-            .run_backend(
-                "list_objects_v2",
-                self.storage.list_objects_v2(input, self.access_key(), self.secret_key()),
-            )
+            .run_backend("list_objects_v2", self.storage.list_objects_v2(input, self.credentials()))
             .await?;
 
         let mut entries = Vec::new();
@@ -196,10 +193,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
         // Issue list_objects_v2. On Err the destructive caller never
         // runs because validate_directory_empty returns the Err.
         let out = self
-            .run_backend(
-                "list_objects_v2",
-                self.storage.list_objects_v2(input, self.access_key(), self.secret_key()),
-            )
+            .run_backend("list_objects_v2", self.storage.list_objects_v2(input, self.credentials()))
             .await?;
 
         // Count content entries that are not the directory's own marker.
@@ -234,7 +228,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
         self.authorize(&S3Action::ListBuckets, "", None).await?;
 
         let out = self
-            .run_backend("list_buckets", self.storage.list_buckets(self.access_key(), self.secret_key()))
+            .run_backend("list_buckets", self.storage.list_buckets(self.credentials()))
             .await?;
 
         let mut entries = Vec::new();
@@ -280,7 +274,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
     /// MKDIR for a bucket-level path: authorise and issue CreateBucket.
     pub(super) async fn mkdir_bucket(&self, bucket: &str) -> Result<(), SftpError> {
         self.authorize(&S3Action::CreateBucket, bucket, None).await?;
-        self.run_backend("create_bucket", self.storage.create_bucket(bucket, self.access_key(), self.secret_key()))
+        self.run_backend("create_bucket", self.storage.create_bucket(bucket, self.credentials()))
             .await?;
         Ok(())
     }
@@ -302,7 +296,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             .body(Some(streaming))
             .build()
             .map_err(|e| s3_error_to_sftp("build_put_object", e))?;
-        self.run_backend("put_object", self.storage.put_object(input, self.access_key(), self.secret_key()))
+        self.run_backend("put_object", self.storage.put_object(input, self.credentials()))
             .await?;
         Ok(())
     }
@@ -312,7 +306,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
     pub(super) async fn rmdir_bucket(&self, bucket: &str) -> Result<(), SftpError> {
         self.validate_directory_empty(bucket, "").await?;
         self.authorize(&S3Action::DeleteBucket, bucket, None).await?;
-        self.run_backend("delete_bucket", self.storage.delete_bucket(bucket, self.access_key(), self.secret_key()))
+        self.run_backend("delete_bucket", self.storage.delete_bucket(bucket, self.credentials()))
             .await?;
         Ok(())
     }
@@ -326,12 +320,8 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
 
         let marker_key = path::encode_dir_object(&prefix);
         self.authorize(&S3Action::DeleteObject, bucket, Some(&marker_key)).await?;
-        self.run_backend(
-            "delete_object",
-            self.storage
-                .delete_object(bucket, &marker_key, self.access_key(), self.secret_key()),
-        )
-        .await?;
+        self.run_backend("delete_object", self.storage.delete_object(bucket, &marker_key, self.credentials()))
+            .await?;
         Ok(())
     }
 
@@ -398,7 +388,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
                 if prefix.is_empty() { None } else { Some(prefix.as_str()) },
             )
             .await?;
-            self.run_backend("head_bucket", self.storage.head_bucket(&bucket, self.access_key(), self.secret_key()))
+            self.run_backend("head_bucket", self.storage.head_bucket(&bucket, self.credentials()))
                 .await?;
             DirCursor::Listing {
                 bucket,

crates/protocols/src/sftp/driver.rs

@@ -34,6 +34,7 @@ use crate::common::client::s3::StorageBackend;
 use crate::common::gateway::{AuthorizationError, S3Action, authorize_operation};
 use crate::common::session::SessionContext;
 use russh_sftp::protocol::{Attrs, Data, File, FileAttributes, Handle, Name, OpenFlags, Packet, Status, StatusCode, Version};
+use rustfs_credentials::Credentials;
 use rustfs_utils::MaskedAccessKey;
 use s3s::dto::{AbortMultipartUploadInput, CopyObjectInput, CopySource};
 use std::collections::HashMap;
@@ -165,16 +166,14 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
         super::read_cache::ReadCache::new(Arc::clone(&self.read_cache_in_use))
     }
 
-    /// Borrow the authenticated principal's S3 access key. Each StorageBackend
-    /// call needs this alongside the secret key for signing.
+    /// Borrow the authenticated principal's S3 access key for diagnostics.
     pub(super) fn access_key(&self) -> &str {
-        &self.session_context.principal.user_identity.credentials.access_key
+        &self.credentials().access_key
     }
 
-    /// Borrow the authenticated principal's S3 secret key. Used together with
-    /// access_key for signing every backend call.
-    pub(super) fn secret_key(&self) -> &str {
-        &self.session_context.principal.user_identity.credentials.secret_key
+    /// Borrow the authenticated principal credentials for backend calls.
+    pub(super) fn credentials(&self) -> &Credentials {
+        self.session_context.credentials()
     }
 
     /// Returns Err(PermissionDenied) when the driver is read-only,
@@ -787,12 +786,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
 
         self.authorize(&S3Action::DeleteObject, &bucket, Some(&object_key)).await?;
 
-        self.run_backend(
-            "delete_object",
-            self.storage
-                .delete_object(&bucket, &object_key, self.access_key(), self.secret_key()),
-        )
-        .await?;
+        self.run_backend("delete_object", self.storage.delete_object(&bucket, &object_key, self.credentials()))
+            .await?;
         Ok(ok_status(id))
     }
 
@@ -898,11 +893,7 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
         // single-shot vs multipart-copy branch below.
         self.authorize(&S3Action::HeadObject, &src_bucket, Some(&src_object)).await?;
         let head = self
-            .run_backend(
-                "head_object",
-                self.storage
-                    .head_object(&src_bucket, &src_object, self.access_key(), self.secret_key()),
-            )
+            .run_backend("head_object", self.storage.head_object(&src_bucket, &src_object, self.credentials()))
             .await?;
         let content_length = head.content_length.unwrap_or(0).max(0) as u64;
 
@@ -920,7 +911,7 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
                 .key(dst_object.clone())
                 .build()
                 .map_err(|e| s3_error_to_sftp("build_copy_object", e))?;
-            self.run_backend("copy_object", self.storage.copy_object(input, self.access_key(), self.secret_key()))
+            self.run_backend("copy_object", self.storage.copy_object(input, self.credentials()))
                 .await?;
         } else {
             self.multipart_copy(&src_bucket, &src_object, &dst_bucket, &dst_object, content_length)
@@ -932,12 +923,8 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
         // delete separately.
         self.authorize(&S3Action::DeleteObject, &src_bucket, Some(&src_object))
             .await?;
-        self.run_backend(
-            "delete_object",
-            self.storage
-                .delete_object(&src_bucket, &src_object, self.access_key(), self.secret_key()),
-        )
-        .await?;
+        self.run_backend("delete_object", self.storage.delete_object(&src_bucket, &src_object, self.credentials()))
+            .await?;
 
         Ok(ok_status(id))
     }
@@ -1029,14 +1016,12 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
 impl<S: StorageBackend + Send + Sync + 'static> Drop for SftpDriver<S> {
     fn drop(&mut self) {
         // Snapshot credentials, peer IP, and the per-call backend
-        // timeout before draining the handle table. self.access_key()
-        // and self.secret_key() borrow self.session_context immutably,
-        // which conflicts with the mutable borrow of self.handles
-        // inside the loop. The timeout is copied into each spawned
-        // abort task so the deadline applies uniformly to inline calls
-        // and Drop-time aborts.
-        let access_key = self.session_context.principal.user_identity.credentials.access_key.clone();
-        let secret_key = self.session_context.principal.user_identity.credentials.secret_key.clone();
+        // timeout before draining the handle table. Borrowing
+        // self.session_context inside the loop would conflict with the
+        // mutable borrow of self.handles. The timeout is copied into each
+        // spawned abort task so the deadline applies uniformly to inline
+        // calls and Drop-time aborts.
+        let credentials = self.session_context.credentials().clone();
         let peer = self.session_context.source_ip;
         let backend_op_timeout_secs = self.backend_op_timeout_secs;
 
@@ -1056,7 +1041,7 @@ impl<S: StorageBackend + Send + Sync + 'static> Drop for SftpDriver<S> {
                             key = %key,
                             upload_id = %upload_id,
                             peer = %peer,
-                            access_key = %access_key,
+                            access_key = %MaskedAccessKey(&credentials.access_key),
                             "skipped abort of orphaned multipart upload on session drop, principal lacks s3:AbortMultipartUpload, bucket lifecycle rules must reclaim parts",
                         );
                     }
@@ -1065,8 +1050,7 @@ impl<S: StorageBackend + Send + Sync + 'static> Drop for SftpDriver<S> {
             };
 
             let storage = Arc::clone(&self.storage);
-            let access_key = access_key.clone();
-            let secret_key = secret_key.clone();
+            let credentials = credentials.clone();
             let upload_id = upload_id_owned;
 
             // Cap the global abort fan-out so a burst of session
@@ -1122,7 +1106,7 @@ impl<S: StorageBackend + Send + Sync + 'static> Drop for SftpDriver<S> {
                     };
                     match tokio::time::timeout(
                         std::time::Duration::from_secs(backend_op_timeout_secs),
-                        storage.abort_multipart_upload(input, &access_key, &secret_key),
+                        storage.abort_multipart_upload(input, &credentials),
                     )
                     .await
                     {

crates/protocols/src/sftp/read.rs

@@ -46,11 +46,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
         // the body. These are cached on the handle so READ can detect EOF
         // and FSTAT can answer without another backend call.
         let head = self
-            .run_backend(
-                "head_object",
-                self.storage
-                    .head_object(&bucket, &object_key, self.access_key(), self.secret_key()),
-            )
+            .run_backend("head_object", self.storage.head_object(&bucket, &object_key, self.credentials()))
             .await?;
         let size = head.content_length.unwrap_or(0).max(0) as u64;
         let mtime = timestamp_to_mtime(head.last_modified);
@@ -166,7 +162,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             .run_backend(
                 "get_object_range",
                 self.storage
-                    .get_object_range(bucket, key, self.access_key(), self.secret_key(), offset, fetch_len),
+                    .get_object_range(bucket, key, self.credentials(), offset, fetch_len),
             )
             .await?;
 

crates/protocols/src/sftp/write.rs

@@ -293,11 +293,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             // not-found error means the key is free. Any other error is
             // propagated rather than misinterpreted as "does not exist".
             match self
-                .run_backend_with_err(
-                    "head_object",
-                    self.storage
-                        .head_object(&bucket, &object_key, self.access_key(), self.secret_key()),
-                )
+                .run_backend_with_err("head_object", self.storage.head_object(&bucket, &object_key, self.credentials()))
                 .await?
             {
                 Ok(_) => return Err(SftpError::code(StatusCode::Failure)),
@@ -385,7 +381,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
                 .map_err(|e| s3_error_to_sftp("build_put_object", e))?;
 
             let outcome = self
-                .run_backend_with_err("put_object", self.storage.put_object(input, self.access_key(), self.secret_key()))
+                .run_backend_with_err("put_object", self.storage.put_object(input, self.credentials()))
                 .await?;
 
             let backend_err = match outcome {
@@ -448,7 +444,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             .map_err(|e| s3_error_to_sftp("build_upload_part", e))?;
 
         let out = self
-            .run_backend("upload_part", self.storage.upload_part(input, self.access_key(), self.secret_key()))
+            .run_backend("upload_part", self.storage.upload_part(input, self.credentials()))
             .await?;
 
         let e_tag = out.e_tag.ok_or_else(|| {
@@ -528,11 +524,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             .map_err(|e| s3_error_to_sftp("build_create_multipart_upload", e))?;
 
         let out = self
-            .run_backend(
-                "create_multipart_upload",
-                self.storage
-                    .create_multipart_upload(input, self.access_key(), self.secret_key()),
-            )
+            .run_backend("create_multipart_upload", self.storage.create_multipart_upload(input, self.credentials()))
             .await?;
 
         let upload_id = out.upload_id.ok_or_else(|| {
@@ -585,8 +577,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
         let result = self
             .run_backend(
                 "complete_multipart_upload",
-                self.storage
-                    .complete_multipart_upload(input, self.access_key(), self.secret_key()),
+                self.storage.complete_multipart_upload(input, self.credentials()),
             )
             .await;
         result?;
@@ -852,12 +843,8 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
             .build()
             .map_err(|e| s3_error_to_sftp("build_abort_multipart_upload", e))?;
 
-        self.run_backend(
-            "abort_multipart_upload",
-            self.storage
-                .abort_multipart_upload(input, self.access_key(), self.secret_key()),
-        )
-        .await?;
+        self.run_backend("abort_multipart_upload", self.storage.abort_multipart_upload(input, self.credentials()))
+            .await?;
         Ok(())
     }
 
@@ -1066,10 +1053,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
                     .map_err(|e| s3_error_to_sftp("build_upload_part_copy", e))?;
 
                 let out = self
-                    .run_backend(
-                        "upload_part_copy",
-                        self.storage.upload_part_copy(input, self.access_key(), self.secret_key()),
-                    )
+                    .run_backend("upload_part_copy", self.storage.upload_part_copy(input, self.credentials()))
                     .await?;
 
                 let e_tag = out.copy_part_result.and_then(|r| r.e_tag).ok_or_else(|| {
@@ -1125,6 +1109,7 @@ mod tests {
     use crate::common::dummy_storage::{AbortCall, DummyBackend, DummyError};
     use crate::common::gateway::with_test_auth_override;
     use russh_sftp::protocol::{FileAttributes, OpenFlags, StatusCode};
+    use rustfs_credentials::Credentials;
     use s3s::dto::ETag;
     use std::sync::Arc;
     use std::time::Duration;
@@ -2324,9 +2309,10 @@ mod tests {
         let backend = Arc::new(DummyBackend::new());
         backend.queue_head_object_err(DummyError::AccessDenied("pinned".to_string()));
         let driver = build_driver(backend, TEST_PART_SIZE);
+        let credentials = Credentials::default();
 
         let result = driver
-            .run_backend_with_err("head_object", driver.storage.head_object("b", "k", "ak", "sk"))
+            .run_backend_with_err("head_object", driver.storage.head_object("b", "k", &credentials))
             .await;
 
         match result {

crates/protocols/src/webdav/driver.rs

@@ -22,6 +22,7 @@ use dav_server::fs::{
 };
 use futures_util::{FutureExt, StreamExt, stream};
 use percent_encoding::percent_decode_str;
+use rustfs_credentials::Credentials;
 use rustfs_utils::MaskedAccessKey;
 use rustfs_utils::path;
 use s3s::S3ErrorCode;
@@ -198,15 +199,7 @@ where
         let key = self.key.clone();
 
         async move {
-            match storage
-                .head_object(
-                    &bucket,
-                    &key,
-                    &session_context.principal.user_identity.credentials.access_key,
-                    &session_context.principal.user_identity.credentials.secret_key,
-                )
-                .await
-            {
+            match storage.head_object(&bucket, &key, session_context.credentials()).await {
                 Ok(output) => {
                     let size = output.content_length.unwrap_or(0) as u64;
                     let modified = output
@@ -288,14 +281,7 @@ where
         async move {
             let start_pos = *position.read().await;
             match storage
-                .get_object_range(
-                    &bucket,
-                    &key,
-                    &session_context.principal.user_identity.credentials.access_key,
-                    &session_context.principal.user_identity.credentials.secret_key,
-                    start_pos,
-                    count as u64,
-                )
+                .get_object_range(&bucket, &key, session_context.credentials(), start_pos, count as u64)
                 .await
             {
                 Ok(output) => {
@@ -407,14 +393,7 @@ where
                 .build()
                 .map_err(|_| FsError::GeneralFailure)?;
 
-            match storage
-                .put_object(
-                    put_input,
-                    &session_context.principal.user_identity.credentials.access_key,
-                    &session_context.principal.user_identity.credentials.secret_key,
-                )
-                .await
-            {
+            match storage.put_object(put_input, session_context.credentials()).await {
                 Ok(_) => {
                     debug!(
                         event = EVENT_WEBDAV_OBJECT_WRITE_STATE,
@@ -522,11 +501,8 @@ where
         self
     }
 
-    fn credentials(&self) -> (&str, &str) {
-        (
-            &self.session_context.principal.user_identity.credentials.access_key,
-            &self.session_context.principal.user_identity.credentials.secret_key,
-        )
+    fn credentials(&self) -> &Credentials {
+        self.session_context.credentials()
     }
 
     fn is_missing_head_object_error(error: &str) -> bool {
@@ -538,7 +514,7 @@ where
     }
 
     async fn prefix_has_entries(&self, bucket: &str, prefix: &str) -> FsResult<bool> {
-        let (access_key, secret_key) = self.credentials();
+        let credentials = self.credentials();
         let list_input = ListObjectsV2Input::builder()
             .bucket(bucket.to_string())
             .prefix(Some(prefix.to_string()))
@@ -546,32 +522,28 @@ where
             .build()
             .map_err(|_| FsError::GeneralFailure)?;
 
-        let output = self
-            .storage
-            .list_objects_v2(list_input, access_key, secret_key)
-            .await
-            .map_err(|e| {
-                error!(
-                    event = EVENT_WEBDAV_LIST_FAILED,
-                    component = LOG_COMPONENT_PROTOCOLS,
-                    subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
-                    bucket = %bucket,
-                    prefix = %prefix,
-                    error = %e,
-                    "webdav list failed"
-                );
-                FsError::GeneralFailure
-            })?;
+        let output = self.storage.list_objects_v2(list_input, credentials).await.map_err(|e| {
+            error!(
+                event = EVENT_WEBDAV_LIST_FAILED,
+                component = LOG_COMPONENT_PROTOCOLS,
+                subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
+                bucket = %bucket,
+                prefix = %prefix,
+                error = %e,
+                "webdav list failed"
+            );
+            FsError::GeneralFailure
+        })?;
 
         Ok(output.contents.map(|c| !c.is_empty()).unwrap_or(false)
             || output.common_prefixes.map(|c| !c.is_empty()).unwrap_or(false))
     }
 
     async fn copy_object_streaming(&self, src_bucket: &str, src_key: &str, dst_bucket: &str, dst_key: &str) -> FsResult<()> {
-        let (access_key, secret_key) = self.credentials();
+        let credentials = self.credentials();
         let get_output = self
             .storage
-            .get_object(src_bucket, src_key, access_key, secret_key, None)
+            .get_object(src_bucket, src_key, credentials, None)
             .await
             .map_err(|e| {
                 error!(
@@ -625,24 +597,21 @@ where
 
         let put_input = put_builder.build().map_err(|_| FsError::GeneralFailure)?;
 
-        self.storage
-            .put_object(put_input, access_key, secret_key)
-            .await
-            .map_err(|e| {
-                error!(
-                    event = EVENT_WEBDAV_COPY_FAILED,
-                    component = LOG_COMPONENT_PROTOCOLS,
-                    subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
-                    state = "destination_write_failed",
-                    src_bucket = %src_bucket,
-                    src_object = %src_key,
-                    dst_bucket = %dst_bucket,
-                    dst_object = %dst_key,
-                    error = %e,
-                    "webdav copy failed"
-                );
-                FsError::GeneralFailure
-            })?;
+        self.storage.put_object(put_input, credentials).await.map_err(|e| {
+            error!(
+                event = EVENT_WEBDAV_COPY_FAILED,
+                component = LOG_COMPONENT_PROTOCOLS,
+                subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
+                state = "destination_write_failed",
+                src_bucket = %src_bucket,
+                src_object = %src_key,
+                dst_bucket = %dst_bucket,
+                dst_object = %dst_key,
+                error = %e,
+                "webdav copy failed"
+            );
+            FsError::GeneralFailure
+        })?;
 
         Ok(())
     }
@@ -653,7 +622,7 @@ where
         dst_bucket: &str,
         rename_pairs: &[(String, String)],
     ) -> FsResult<()> {
-        let (access_key, secret_key) = self.credentials();
+        let credentials = self.credentials();
 
         for (src_obj_key, dst_obj_key) in rename_pairs {
             self.copy_object_streaming(src_bucket, src_obj_key, dst_bucket, dst_obj_key)
@@ -662,7 +631,7 @@ where
 
         for (src_obj_key, _) in rename_pairs {
             self.storage
-                .delete_object(src_bucket, src_obj_key, access_key, secret_key)
+                .delete_object(src_bucket, src_obj_key, credentials)
                 .await
                 .map_err(|e| {
                     error!(
@@ -683,7 +652,7 @@ where
     }
 
     async fn probe_head_object(&self, bucket: &str, key: &str) -> FsResult<HeadObjectProbe> {
-        let (access_key, secret_key) = self.credentials();
+        let credentials = self.credentials();
 
         if authorize_operation(&self.session_context, &S3Action::HeadObject, bucket, Some(key))
             .await
@@ -692,7 +661,7 @@ where
             return Ok(HeadObjectProbe::Forbidden);
         }
 
-        match self.storage.head_object(bucket, key, access_key, secret_key).await {
+        match self.storage.head_object(bucket, key, credentials).await {
             Ok(output) => Ok(HeadObjectProbe::Found(Box::new(output))),
             Err(e) => {
                 let err_msg = e.to_string();
@@ -816,8 +785,8 @@ where
     async fn list_buckets(&self) -> FsResult<Vec<WebDavDirEntry>> {
         match authorize_operation(&self.session_context, &S3Action::ListBuckets, "", None).await {
             Ok(()) => {
-                let (access_key, secret_key) = self.credentials();
-                return match self.storage.list_buckets(access_key, secret_key).await {
+                let credentials = self.credentials();
+                return match self.storage.list_buckets(credentials).await {
                     Ok(output) => Ok(Self::bucket_entries(output)),
                     Err(error) => {
                         error!(
@@ -825,7 +794,7 @@ where
                             component = LOG_COMPONENT_PROTOCOLS,
                             subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
                             error = %error,
-                            access_key = %MaskedAccessKey(access_key),
+                            access_key = %MaskedAccessKey(credentials.access_key.as_str()),
                             "webdav bucket list failed"
                         );
                         Err(FsError::GeneralFailure)
@@ -908,15 +877,7 @@ where
             .build()
             .map_err(|_| FsError::GeneralFailure)?;
 
-        match self
-            .storage
-            .list_objects_v2(
-                list_input,
-                &self.session_context.principal.user_identity.credentials.access_key,
-                &self.session_context.principal.user_identity.credentials.secret_key,
-            )
-            .await
-        {
+        match self.storage.list_objects_v2(list_input, self.credentials()).await {
             Ok(output) => {
                 let mut entries = Vec::new();
 
@@ -1054,15 +1015,7 @@ where
 
             let list_input = list_input.build().map_err(|_| FsError::GeneralFailure)?;
 
-            if let Ok(output) = self
-                .storage
-                .list_objects_v2(
-                    list_input,
-                    &self.session_context.principal.user_identity.credentials.access_key,
-                    &self.session_context.principal.user_identity.credentials.secret_key,
-                )
-                .await
-            {
+            if let Ok(output) = self.storage.list_objects_v2(list_input, self.credentials()).await {
                 // Delete all objects in this page
                 if let Some(objects) = output.contents {
                     for obj in objects {
@@ -1071,15 +1024,7 @@ where
                                 .await
                                 .map_err(|_| FsError::Forbidden)?;
 
-                            let _ = self
-                                .storage
-                                .delete_object(
-                                    bucket,
-                                    &obj_key,
-                                    &self.session_context.principal.user_identity.credentials.access_key,
-                                    &self.session_context.principal.user_identity.credentials.secret_key,
-                                )
-                                .await;
+                            let _ = self.storage.delete_object(bucket, &obj_key, self.credentials()).await;
                         }
                     }
                 }
@@ -1095,15 +1040,7 @@ where
         }
 
         // Then delete the bucket
-        match self
-            .storage
-            .delete_bucket(
-                bucket,
-                &self.session_context.principal.user_identity.credentials.access_key,
-                &self.session_context.principal.user_identity.credentials.secret_key,
-            )
-            .await
-        {
+        match self.storage.delete_bucket(bucket, self.credentials()).await {
             Ok(_) => Ok(()),
             Err(e) if e.to_string().contains("NoSuchBucket") => Ok(()),
             Err(e) => {
@@ -1250,15 +1187,7 @@ where
                     .await
                     .map_err(|_| FsError::Forbidden)?;
 
-                match self
-                    .storage
-                    .head_bucket(
-                        &bucket,
-                        &self.session_context.principal.user_identity.credentials.access_key,
-                        &self.session_context.principal.user_identity.credentials.secret_key,
-                    )
-                    .await
-                {
+                match self.storage.head_bucket(&bucket, self.credentials()).await {
                     Ok(_) => Ok(Box::new(WebDavMetaData {
                         size: 0,
                         modified: SystemTime::now(),
@@ -1318,15 +1247,7 @@ where
                     .build()
                     .map_err(|_| FsError::GeneralFailure)?;
 
-                match self
-                    .storage
-                    .put_object(
-                        put_input,
-                        &self.session_context.principal.user_identity.credentials.access_key,
-                        &self.session_context.principal.user_identity.credentials.secret_key,
-                    )
-                    .await
-                {
+                match self.storage.put_object(put_input, self.credentials()).await {
                     Ok(_) => {
                         debug!(
                             event = EVENT_WEBDAV_DIRECTORY_STATE,
@@ -1360,15 +1281,7 @@ where
                 .await
                 .map_err(|_| FsError::Forbidden)?;
 
-            match self
-                .storage
-                .create_bucket(
-                    &bucket,
-                    &self.session_context.principal.user_identity.credentials.access_key,
-                    &self.session_context.principal.user_identity.credentials.secret_key,
-                )
-                .await
-            {
+            match self.storage.create_bucket(&bucket, self.credentials()).await {
                 Ok(_) => {
                     debug!(
                         event = EVENT_WEBDAV_DIRECTORY_STATE,
@@ -1438,15 +1351,7 @@ where
 
                     let list_input = list_input.build().map_err(|_| FsError::GeneralFailure)?;
 
-                    if let Ok(output) = self
-                        .storage
-                        .list_objects_v2(
-                            list_input,
-                            &self.session_context.principal.user_identity.credentials.access_key,
-                            &self.session_context.principal.user_identity.credentials.secret_key,
-                        )
-                        .await
-                    {
+                    if let Ok(output) = self.storage.list_objects_v2(list_input, self.credentials()).await {
                         if let Some(objects) = output.contents {
                             for obj in objects {
                                 if let Some(obj_key) = obj.key {
@@ -1454,15 +1359,7 @@ where
                                         .await
                                         .map_err(|_| FsError::Forbidden)?;
 
-                                    let _ = self
-                                        .storage
-                                        .delete_object(
-                                            &bucket,
-                                            &obj_key,
-                                            &self.session_context.principal.user_identity.credentials.access_key,
-                                            &self.session_context.principal.user_identity.credentials.secret_key,
-                                        )
-                                        .await;
+                                    let _ = self.storage.delete_object(&bucket, &obj_key, self.credentials()).await;
                                 }
                             }
                         }
@@ -1479,12 +1376,7 @@ where
                 // Also delete the directory marker itself
                 let _ = self
                     .storage
-                    .delete_object(
-                        &bucket,
-                        &prefix_with_slash,
-                        &self.session_context.principal.user_identity.credentials.access_key,
-                        &self.session_context.principal.user_identity.credentials.secret_key,
-                    )
+                    .delete_object(&bucket, &prefix_with_slash, self.credentials())
                     .await;
 
                 return Ok(());
@@ -1515,16 +1407,7 @@ where
                 .await
                 .map_err(|_| FsError::Forbidden)?;
 
-            match self
-                .storage
-                .delete_object(
-                    &bucket,
-                    &key,
-                    &self.session_context.principal.user_identity.credentials.access_key,
-                    &self.session_context.principal.user_identity.credentials.secret_key,
-                )
-                .await
-            {
+            match self.storage.delete_object(&bucket, &key, self.credentials()).await {
                 Ok(_) => {
                     debug!(
                         event = EVENT_WEBDAV_OBJECT_DELETE_STATE,
@@ -1566,7 +1449,7 @@ where
 
             let src_key = src_key.ok_or(FsError::Forbidden)?;
             let dst_key = dst_key.ok_or(FsError::Forbidden)?;
-            let (access_key, secret_key) = self.credentials();
+            let credentials = self.credentials();
             let resolved_src = self.resolve_path(&src_bucket, &src_key).await?;
             let (src_prefix, include_src_marker) = match resolved_src {
                 ResolvedPath::File(_) => {
@@ -1584,7 +1467,7 @@ where
                         .await?;
 
                     self.storage
-                        .delete_object(&src_bucket, &src_key, access_key, secret_key)
+                        .delete_object(&src_bucket, &src_key, credentials)
                         .await
                         .map_err(|e| {
                             error!(
@@ -1656,25 +1539,21 @@ where
                 }
 
                 let list_input = list_builder.build().map_err(|_| FsError::GeneralFailure)?;
-                let output = self
-                    .storage
-                    .list_objects_v2(list_input, access_key, secret_key)
-                    .await
-                    .map_err(|e| {
-                        error!(
-                            event = EVENT_WEBDAV_RENAME_STATE,
-                            component = LOG_COMPONENT_PROTOCOLS,
-                            subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
-                            state = "directory_list_failed",
-                            src_bucket = %src_bucket,
-                            src_prefix = %src_prefix,
-                            dst_bucket = %dst_bucket,
-                            dst_prefix = %dst_prefix,
-                            error = %e,
-                            "WebDAV rename directory listing failed"
-                        );
-                        FsError::GeneralFailure
-                    })?;
+                let output = self.storage.list_objects_v2(list_input, credentials).await.map_err(|e| {
+                    error!(
+                        event = EVENT_WEBDAV_RENAME_STATE,
+                        component = LOG_COMPONENT_PROTOCOLS,
+                        subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
+                        state = "directory_list_failed",
+                        src_bucket = %src_bucket,
+                        src_prefix = %src_prefix,
+                        dst_bucket = %dst_bucket,
+                        dst_prefix = %dst_prefix,
+                        error = %e,
+                        "WebDAV rename directory listing failed"
+                    );
+                    FsError::GeneralFailure
+                })?;
 
                 let mut page_pairs: Vec<(String, String)> = Vec::new();
                 if let Some(objects) = output.contents {
@@ -1785,8 +1664,7 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
             _start_pos: Option<u64>,
         ) -> Result<GetObjectOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
@@ -1796,20 +1674,14 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
             _start_pos: u64,
             _length: u64,
         ) -> Result<GetObjectOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
-        async fn put_object(
-            &self,
-            _input: PutObjectInput,
-            _access_key: &str,
-            _secret_key: &str,
-        ) -> Result<PutObjectOutput, Self::Error> {
+        async fn put_object(&self, _input: PutObjectInput, _credentials: &Credentials) -> Result<PutObjectOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
@@ -1817,8 +1689,7 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<DeleteObjectOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
@@ -1827,57 +1698,39 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<HeadObjectOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
-        async fn head_bucket(
-            &self,
-            _bucket: &str,
-            _access_key: &str,
-            _secret_key: &str,
-        ) -> Result<HeadBucketOutput, Self::Error> {
+        async fn head_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<HeadBucketOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
         async fn list_objects_v2(
             &self,
             _input: ListObjectsV2Input,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<ListObjectsV2Output, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
-        async fn list_buckets(&self, _access_key: &str, _secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
+        async fn list_buckets(&self, _credentials: &Credentials) -> Result<ListBucketsOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
-        async fn create_bucket(
-            &self,
-            _bucket: &str,
-            _access_key: &str,
-            _secret_key: &str,
-        ) -> Result<CreateBucketOutput, Self::Error> {
+        async fn create_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<CreateBucketOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
-        async fn delete_bucket(
-            &self,
-            _bucket: &str,
-            _access_key: &str,
-            _secret_key: &str,
-        ) -> Result<DeleteBucketOutput, Self::Error> {
+        async fn delete_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<DeleteBucketOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
 
         async fn copy_object(
             &self,
             _input: CopyObjectInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<CopyObjectOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
@@ -1885,8 +1738,7 @@ mod tests {
         async fn create_multipart_upload(
             &self,
             _input: CreateMultipartUploadInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<CreateMultipartUploadOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
@@ -1894,8 +1746,7 @@ mod tests {
         async fn upload_part(
             &self,
             _input: UploadPartInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<UploadPartOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
@@ -1903,8 +1754,7 @@ mod tests {
         async fn complete_multipart_upload(
             &self,
             _input: CompleteMultipartUploadInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<CompleteMultipartUploadOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
@@ -1912,8 +1762,7 @@ mod tests {
         async fn abort_multipart_upload(
             &self,
             _input: AbortMultipartUploadInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<AbortMultipartUploadOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
@@ -1921,8 +1770,7 @@ mod tests {
         async fn upload_part_copy(
             &self,
             _input: UploadPartCopyInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<UploadPartCopyOutput, Self::Error> {
             unreachable!("parse_path tests should not hit storage")
         }
@@ -2099,8 +1947,7 @@ mod tests {
             &self,
             bucket: &str,
             key: &str,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
             _start_pos: Option<u64>,
         ) -> Result<GetObjectOutput, Self::Error> {
             let data = self
@@ -2127,8 +1974,7 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
             _start_pos: u64,
             _length: u64,
         ) -> Result<GetObjectOutput, Self::Error> {
@@ -2138,8 +1984,7 @@ mod tests {
         async fn put_object(
             &self,
             mut input: PutObjectInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<PutObjectOutput, Self::Error> {
             let bucket = input.bucket.clone();
             let key = input.key.clone();
@@ -2163,8 +2008,7 @@ mod tests {
             &self,
             bucket: &str,
             key: &str,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<DeleteObjectOutput, Self::Error> {
             let mut state = self.state.lock().expect("recording storage lock poisoned");
             state.delete_keys.push(key.to_string());
@@ -2179,26 +2023,19 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<HeadObjectOutput, Self::Error> {
             unreachable!("head_object is not used in rename regression tests")
         }
 
-        async fn head_bucket(
-            &self,
-            _bucket: &str,
-            _access_key: &str,
-            _secret_key: &str,
-        ) -> Result<HeadBucketOutput, Self::Error> {
+        async fn head_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<HeadBucketOutput, Self::Error> {
             unreachable!("head_bucket is not used in rename regression tests")
         }
 
         async fn list_objects_v2(
             &self,
             input: ListObjectsV2Input,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<ListObjectsV2Output, Self::Error> {
             let prefix = input.prefix.unwrap_or_default();
             let mut keys: Vec<String> = self
@@ -2226,25 +2063,15 @@ mod tests {
             })
         }
 
-        async fn list_buckets(&self, _access_key: &str, _secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
+        async fn list_buckets(&self, _credentials: &Credentials) -> Result<ListBucketsOutput, Self::Error> {
             unreachable!("list_buckets is not used in rename regression tests")
         }
 
-        async fn create_bucket(
-            &self,
-            _bucket: &str,
-            _access_key: &str,
-            _secret_key: &str,
-        ) -> Result<CreateBucketOutput, Self::Error> {
+        async fn create_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<CreateBucketOutput, Self::Error> {
             unreachable!("create_bucket is not used in rename regression tests")
         }
 
-        async fn delete_bucket(
-            &self,
-            bucket: &str,
-            _access_key: &str,
-            _secret_key: &str,
-        ) -> Result<DeleteBucketOutput, Self::Error> {
+        async fn delete_bucket(&self, bucket: &str, _credentials: &Credentials) -> Result<DeleteBucketOutput, Self::Error> {
             self.state
                 .lock()
                 .expect("recording storage lock poisoned")
@@ -2256,8 +2083,7 @@ mod tests {
         async fn copy_object(
             &self,
             _input: CopyObjectInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<CopyObjectOutput, Self::Error> {
             unreachable!("copy_object is not used in rename regression tests")
         }
@@ -2265,8 +2091,7 @@ mod tests {
         async fn create_multipart_upload(
             &self,
             _input: CreateMultipartUploadInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<CreateMultipartUploadOutput, Self::Error> {
             unreachable!("create_multipart_upload is not used in rename regression tests")
         }
@@ -2274,8 +2099,7 @@ mod tests {
         async fn upload_part(
             &self,
             _input: UploadPartInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<UploadPartOutput, Self::Error> {
             unreachable!("upload_part is not used in rename regression tests")
         }
@@ -2283,8 +2107,7 @@ mod tests {
         async fn complete_multipart_upload(
             &self,
             _input: CompleteMultipartUploadInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<CompleteMultipartUploadOutput, Self::Error> {
             unreachable!("complete_multipart_upload is not used in rename regression tests")
         }
@@ -2292,8 +2115,7 @@ mod tests {
         async fn abort_multipart_upload(
             &self,
             _input: AbortMultipartUploadInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<AbortMultipartUploadOutput, Self::Error> {
             unreachable!("abort_multipart_upload is not used in rename regression tests")
         }
@@ -2301,8 +2123,7 @@ mod tests {
         async fn upload_part_copy(
             &self,
             _input: UploadPartCopyInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<UploadPartCopyOutput, Self::Error> {
             unreachable!("upload_part_copy is not used in rename regression tests")
         }

crates/protocols/src/webdav/server.rs

@@ -687,6 +687,7 @@ mod tests {
     use futures_util::stream;
     use http_body_util::StreamBody;
     use hyper::body::Frame;
+    use rustfs_credentials::Credentials;
     use s3s::dto::*;
     use std::fmt::{Debug, Formatter};
     use std::net::{Ipv4Addr, SocketAddr};
@@ -715,8 +716,7 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
             _start_pos: Option<u64>,
         ) -> Result<GetObjectOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
@@ -726,20 +726,14 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
             _start_pos: u64,
             _length: u64,
         ) -> Result<GetObjectOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
-        async fn put_object(
-            &self,
-            _input: PutObjectInput,
-            _access_key: &str,
-            _secret_key: &str,
-        ) -> Result<PutObjectOutput, Self::Error> {
+        async fn put_object(&self, _input: PutObjectInput, _credentials: &Credentials) -> Result<PutObjectOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
@@ -747,8 +741,7 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<DeleteObjectOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
@@ -757,57 +750,39 @@ mod tests {
             &self,
             _bucket: &str,
             _key: &str,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<HeadObjectOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
-        async fn head_bucket(
-            &self,
-            _bucket: &str,
-            _access_key: &str,
-            _secret_key: &str,
-        ) -> Result<HeadBucketOutput, Self::Error> {
+        async fn head_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<HeadBucketOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
         async fn list_objects_v2(
             &self,
             _input: ListObjectsV2Input,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<ListObjectsV2Output, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
-        async fn list_buckets(&self, _access_key: &str, _secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
+        async fn list_buckets(&self, _credentials: &Credentials) -> Result<ListBucketsOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
-        async fn create_bucket(
-            &self,
-            _bucket: &str,
-            _access_key: &str,
-            _secret_key: &str,
-        ) -> Result<CreateBucketOutput, Self::Error> {
+        async fn create_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<CreateBucketOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
-        async fn delete_bucket(
-            &self,
-            _bucket: &str,
-            _access_key: &str,
-            _secret_key: &str,
-        ) -> Result<DeleteBucketOutput, Self::Error> {
+        async fn delete_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<DeleteBucketOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
 
         async fn copy_object(
             &self,
             _input: CopyObjectInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<CopyObjectOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
@@ -815,8 +790,7 @@ mod tests {
         async fn create_multipart_upload(
             &self,
             _input: CreateMultipartUploadInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<CreateMultipartUploadOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
@@ -824,8 +798,7 @@ mod tests {
         async fn upload_part(
             &self,
             _input: UploadPartInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<UploadPartOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
@@ -833,8 +806,7 @@ mod tests {
         async fn complete_multipart_upload(
             &self,
             _input: CompleteMultipartUploadInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<CompleteMultipartUploadOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
@@ -842,8 +814,7 @@ mod tests {
         async fn abort_multipart_upload(
             &self,
             _input: AbortMultipartUploadInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<AbortMultipartUploadOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }
@@ -851,8 +822,7 @@ mod tests {
         async fn upload_part_copy(
             &self,
             _input: UploadPartCopyInput,
-            _access_key: &str,
-            _secret_key: &str,
+            _credentials: &Credentials,
         ) -> Result<UploadPartCopyOutput, Self::Error> {
             unreachable!("connection tests should not hit storage")
         }

crates/rio/Cargo.toml

@@ -90,8 +90,15 @@ s3s = { workspace = true, features = ["minio"] }
 hex-simd.workspace = true
 
 [dev-dependencies]
+tokio = { workspace = true, features = ["test-util"] }
 tokio-test = { workspace = true }
+criterion = { workspace = true, features = ["html_reports"] }
+proptest = { workspace = true }
 axum = { workspace = true }
 hyper = { workspace = true, features = ["http2", "server"] }
 hyper-util = { workspace = true, features = ["tokio"] }
 http-body-util = { workspace = true }
+
+[[bench]]
+name = "tee_reader"
+harness = false

crates/rio/benches/tee_reader.rs

@@ -0,0 +1,103 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Throughput of `tee_reader` versus reading the same source directly:
+//! 64 MiB of data served in 1 MiB chunks, consumed with 1 MiB reads.
+
+use bytes::Bytes;
+use criterion::{Criterion, Throughput, criterion_group, criterion_main};
+use rustfs_rio::tee_reader;
+use std::hint::black_box;
+use std::io;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf};
+
+const CHUNK_BYTES: usize = 1024 * 1024;
+const TOTAL_BYTES: usize = 64 * 1024 * 1024;
+const TEE_BUFFER_BYTES: usize = 4 * CHUNK_BYTES;
+
+/// In-memory source that serves at most `CHUNK_BYTES` per poll.
+struct ChunkedSource {
+    data: Bytes,
+    pos: usize,
+}
+
+impl AsyncRead for ChunkedSource {
+    fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
+        let remaining = self.data.len() - self.pos;
+        let n = CHUNK_BYTES.min(remaining).min(buf.remaining());
+        buf.put_slice(&self.data[self.pos..self.pos + n]);
+        self.pos += n;
+        Poll::Ready(Ok(()))
+    }
+}
+
+async fn consume<R: AsyncRead + Unpin>(mut reader: R) -> usize {
+    let mut buf = vec![0u8; CHUNK_BYTES];
+    let mut total = 0;
+    loop {
+        let n = reader.read(&mut buf).await.expect("read");
+        if n == 0 {
+            return total;
+        }
+        total += n;
+    }
+}
+
+fn bench_tee_reader(c: &mut Criterion) {
+    let runtime = tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(2)
+        .enable_all()
+        .build()
+        .expect("build tokio runtime for tee_reader benchmark");
+    let data = Bytes::from(vec![0xA5u8; TOTAL_BYTES]);
+
+    let mut group = c.benchmark_group("tee_reader_64mib_1mib_chunks");
+    group.throughput(Throughput::Bytes(TOTAL_BYTES as u64));
+    group.sample_size(10);
+
+    group.bench_function("direct_read", |b| {
+        b.iter(|| {
+            let source = ChunkedSource {
+                data: data.clone(),
+                pos: 0,
+            };
+            let total = runtime.block_on(consume(source));
+            black_box(total)
+        })
+    });
+
+    group.bench_function("tee_primary_plus_secondary", |b| {
+        b.iter(|| {
+            let source = ChunkedSource {
+                data: data.clone(),
+                pos: 0,
+            };
+            let (primary, secondary) = tee_reader(source, TEE_BUFFER_BYTES);
+            let totals = runtime.block_on(async {
+                let secondary_task = tokio::spawn(consume(secondary));
+                let primary_total = consume(primary).await;
+                let secondary_total = secondary_task.await.expect("secondary task");
+                (primary_total, secondary_total)
+            });
+            black_box(totals)
+        })
+    });
+
+    group.finish();
+}
+
+criterion_group!(benches, bench_tee_reader);
+criterion_main!(benches);

crates/rio/src/lib.rs

@@ -118,6 +118,12 @@ pub use hardlimit_reader::HardLimitReader;
 
 mod hash_reader;
 pub use hash_reader::*;
+
+mod tee_reader;
+pub use tee_reader::{
+    DEFAULT_TEE_MAX_DRAIN_BYTES, TeeDrainLimitExceeded, TeeOptions, TeePrimary, TeeSecondary, TeeStream, tee_reader,
+    tee_reader_with_options,
+};
 mod checksum;
 pub use checksum::*;
 

crates/scanner/src/scanner.rs

@@ -552,18 +552,21 @@ pub(super) fn data_usage_info_has_persisted_baseline_identity(info: &DataUsageIn
 }
 
 pub(super) fn data_usage_info_is_bootstrap_pending(info: &DataUsageInfo) -> bool {
-    if info.last_update.is_none() || info.scanner_cycle.is_some() {
+    let Some(last_update) = info.last_update else {
         return false;
-    }
+    };
 
-    let expected = DataUsageInfo {
-        last_update: info.last_update,
-        scanner_epoch: info.scanner_epoch,
+    info == &scanner_usage_bootstrap_marker(last_update, info.scanner_epoch)
+}
+
+pub(super) fn scanner_usage_bootstrap_marker(last_update: std::time::SystemTime, scanner_epoch: Option<u64>) -> DataUsageInfo {
+    DataUsageInfo {
+        last_update: Some(last_update),
+        scanner_epoch,
         usage_snapshot_converged: Some(false),
         usage_snapshot_bootstrap_pending: true,
         ..Default::default()
-    };
-    info == &expected
+    }
 }
 
 fn usage_cache_needs_prompt_scan(authoritative: &DataUsageInfo, observed: Option<&DataUsageInfo>) -> bool {
@@ -915,8 +918,8 @@ fn prepare_cycle_for_usage_floor_bootstrap(
                 },
             )
         }
-        PersistedUsageFloorStartup::RecoveredLegacyEmptyFence => {
-            // The legacy empty fence proves only its leader epoch, not
+        PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence => {
+            // The legacy incomplete fence proves only its leader epoch, not
             // namespace coverage. Clear coverage while retaining the durable
             // cycle number so surviving caches cannot force a regression.
             let next = cycle_info.next;
@@ -1438,6 +1441,7 @@ async fn fence_scanner_epoch_after_cycle_timeout<Store, LockLost>(
     cycle_info: &mut CurrentCycle,
     cycle_revision: &mut DataUsageCacheRevision,
     leader_epoch: &mut u64,
+    allow_bootstrap_pending: bool,
     lock_lost: LockLost,
 ) -> bool
 where
@@ -1451,7 +1455,7 @@ where
         cycle_info,
         cycle_revision,
         leader_epoch,
-        false,
+        allow_bootstrap_pending,
         ScannerCycleResetPolicy::None,
     );
     tokio::pin!(claim);
@@ -1473,6 +1477,7 @@ struct ScannerCycleDeadlineState<'a> {
     cycle_revision: &'a mut DataUsageCacheRevision,
     leader_epoch: &'a mut u64,
     cycle_budget: &'a ScannerCycleBudget,
+    allow_bootstrap_pending: bool,
 }
 
 fn cycle_timeout_requires_recovery(worker_stopped: bool, cycle_state_persisted: bool, generation_fenced: bool) -> bool {
@@ -1494,6 +1499,7 @@ async fn handle_scanner_cycle_deadline<Store>(
         state.cycle_info,
         state.cycle_revision,
         state.leader_epoch,
+        state.allow_bootstrap_pending,
         guard.lock_lost_notified(),
     )
     .await;
@@ -2575,7 +2581,7 @@ async fn run_data_scanner_with_maintenance_state(
     match usage_floor_startup {
         PersistedUsageFloorStartup::Authoritative
         | PersistedUsageFloorStartup::BootstrapPending
-        | PersistedUsageFloorStartup::RecoveredLegacyEmptyFence => {}
+        | PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence => {}
         PersistedUsageFloorStartup::Missing => {
             if ctx.is_cancelled() || guard.is_lock_lost() {
                 global_metrics().set_cycle(None).await;
@@ -2670,8 +2676,8 @@ async fn run_data_scanner_with_maintenance_state(
         finish_scanner_leader_iteration(false, "epoch_claim_failed", "leadership epoch claim failed".to_string()).await;
         return Ok(());
     }
-    if usage_floor_startup == PersistedUsageFloorStartup::RecoveredLegacyEmptyFence
-        && let Err(err) = complete_legacy_empty_usage_floor_recovery(storeapi.clone(), leader_epoch).await
+    if usage_floor_startup == PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence
+        && let Err(err) = complete_legacy_incomplete_usage_floor_recovery(storeapi.clone(), leader_epoch).await
     {
         let error = err.to_string();
         warn!(
@@ -2744,6 +2750,7 @@ async fn run_data_scanner_with_maintenance_state(
                         cycle_revision: &mut cycle_revision,
                         leader_epoch: &mut leader_epoch,
                         cycle_budget: &cycle_budget,
+                        allow_bootstrap_pending: allow_usage_floor_bootstrap_pending,
                     },
                     worker_stopped,
                     &mut guard,
@@ -3033,6 +3040,7 @@ async fn run_data_scanner_with_maintenance_state(
                         cycle_revision: &mut cycle_revision,
                         leader_epoch: &mut leader_epoch,
                         cycle_budget: &cycle_budget,
+                        allow_bootstrap_pending: allow_usage_floor_bootstrap_pending,
                     },
                     worker_stopped,
                     &mut guard,

crates/scanner/src/scanner/cycle_state.rs

@@ -26,7 +26,10 @@ pub(super) const MAX_SCANNER_CYCLE_RECOVERY_RETRIES: u32 = 5;
 const METRIC_SCANNER_CYCLE_RECOVERY_REQUIRED: &str = "rustfs_scanner_cycle_recovery_required";
 const METRIC_SCANNER_CYCLE_RECOVERY_RETRY_COUNT: &str = "rustfs_scanner_cycle_recovery_retry_count";
 const USAGE_FLOOR_LOAD_FAILED: &str = "usage_floor_load_failed";
-const LEGACY_EMPTY_USAGE_FLOOR_RECOVERY: &str = "legacy_empty_usage_floor";
+// Keep the published status value stable for operators that already alert on
+// the empty-fence recovery introduced by backlog-2102. The same durable marker
+// now also covers strictly validated data-bearing legacy fences.
+const LEGACY_INCOMPLETE_USAGE_FLOOR_RECOVERY: &str = "legacy_empty_usage_floor";
 const CACHE_CYCLE_AHEAD: &str = "cache_cycle_ahead";
 
 const SCANNER_USAGE_STATE_RESET_MODE_FULL_REBUILD: &str = "full-rebuild";
@@ -182,9 +185,9 @@ pub(super) fn clear_scanner_cache_cycle_ahead() {
     }
 }
 
-pub(super) fn record_legacy_empty_usage_floor_recovery_pending(leader_epoch: u64) {
+pub(super) fn record_legacy_incomplete_usage_floor_recovery_pending(leader_epoch: u64) {
     let previous = scanner_cycle_recovery_status();
-    let same_recovery = previous.classification.as_deref() == Some(LEGACY_EMPTY_USAGE_FLOOR_RECOVERY)
+    let same_recovery = previous.classification.as_deref() == Some(LEGACY_INCOMPLETE_USAGE_FLOOR_RECOVERY)
         && previous.leader_epoch == Some(leader_epoch);
     let now = unix_now_secs();
     let (first_detected_at_unix_secs, retry_count) = if same_recovery {
@@ -196,20 +199,20 @@ pub(super) fn record_legacy_empty_usage_floor_recovery_pending(leader_epoch: u64
         path: DATA_USAGE_OBJ_NAME_PATH.clone(),
         quarantine_path: Some(DATA_USAGE_RECOVERY_PATH.clone()),
         state: "usage_floor_recovery_pending".to_string(),
-        classification: Some(LEGACY_EMPTY_USAGE_FLOOR_RECOVERY.to_string()),
+        classification: Some(LEGACY_INCOMPLETE_USAGE_FLOOR_RECOVERY.to_string()),
         leader_epoch: Some(leader_epoch),
         first_detected_at_unix_secs,
         last_attempt_at_unix_secs: Some(now),
         retry_count,
         max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES,
         retryable: true,
-        reason: Some("legacy empty usage floor recovery is awaiting a fenced leadership claim".to_string()),
+        reason: Some("legacy incomplete usage floor recovery is awaiting a fenced leadership claim".to_string()),
         ..Default::default()
     });
 }
 
-pub(super) fn clear_legacy_empty_usage_floor_recovery_status() {
-    if scanner_cycle_recovery_status().classification.as_deref() == Some(LEGACY_EMPTY_USAGE_FLOOR_RECOVERY) {
+pub(super) fn clear_legacy_incomplete_usage_floor_recovery_status() {
+    if scanner_cycle_recovery_status().classification.as_deref() == Some(LEGACY_INCOMPLETE_USAGE_FLOOR_RECOVERY) {
         set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
     }
 }
@@ -1431,6 +1434,101 @@ async fn delete_usage_state_reset_slot(
     }
 }
 
+#[derive(Clone, Copy)]
+pub(super) enum ScannerUsageBootstrapPublishContext {
+    Initial,
+    Recovery,
+    Reset,
+}
+
+enum ScannerUsageBootstrapPublishError {
+    Encode(serde_json::Error),
+    Reconcile(EcstoreError),
+    MissingEtag,
+    Save(EcstoreError),
+}
+
+impl ScannerUsageBootstrapPublishError {
+    fn into_scanner_error(self, context: ScannerUsageBootstrapPublishContext) -> ScannerError {
+        let message = match context {
+            ScannerUsageBootstrapPublishContext::Initial => match self {
+                Self::Encode(err) => format!("failed to encode scanner usage baseline bootstrap: {err}"),
+                Self::Reconcile(err) => format!("failed to reconcile scanner usage bootstrap: {err}"),
+                Self::MissingEtag => "scanner usage bootstrap returned no ETag and could not be confirmed".to_string(),
+                Self::Save(err) => format!("failed to persist scanner usage bootstrap: {err}"),
+            },
+            ScannerUsageBootstrapPublishContext::Recovery => match self {
+                Self::Encode(err) => format!("failed to encode recovered scanner usage bootstrap: {err}"),
+                Self::Reconcile(err) => format!("failed to reconcile recovered scanner usage bootstrap: {err}"),
+                Self::MissingEtag => "recovered scanner usage bootstrap returned no ETag and could not be confirmed".to_string(),
+                Self::Save(err) => format!("failed to recover legacy incomplete scanner usage floor: {err}"),
+            },
+            ScannerUsageBootstrapPublishContext::Reset => match self {
+                Self::Encode(err) => format!("failed to encode scanner usage reset bootstrap marker: {err}"),
+                Self::Reconcile(err) => format!("failed to reconcile scanner usage reset bootstrap marker: {err}"),
+                Self::MissingEtag => "scanner usage reset bootstrap returned no ETag and could not be confirmed".to_string(),
+                Self::Save(err) if scanner_publication_epoch_changed(&err) => {
+                    "scanner usage reset deferred by a movement epoch change".to_string()
+                }
+                Self::Save(EcstoreError::PreconditionFailed) => {
+                    "scanner usage reset primary slot changed before bootstrap publish".to_string()
+                }
+                Self::Save(err) => format!("failed to persist scanner usage reset bootstrap: {err}"),
+            },
+        };
+        ScannerError::Other(message)
+    }
+}
+
+pub(super) async fn publish_scanner_usage_bootstrap_primary(
+    storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
+    expected_revision: &DataUsageCacheRevision,
+    expected_publication_epoch: u64,
+    leader_epoch: Option<u64>,
+    context: ScannerUsageBootstrapPublishContext,
+) -> Result<(), ScannerError> {
+    async fn inner(
+        storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
+        expected_revision: &DataUsageCacheRevision,
+        expected_publication_epoch: u64,
+        leader_epoch: Option<u64>,
+    ) -> Result<(), ScannerUsageBootstrapPublishError> {
+        let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::now(), leader_epoch);
+        let data = serde_json::to_vec(&marker).map_err(ScannerUsageBootstrapPublishError::Encode)?;
+        let save_result = save_config_with_publication_admission_for_epoch(
+            storeapi.clone(),
+            DATA_USAGE_OBJ_NAME_PATH.as_str(),
+            data.clone(),
+            expected_revision.preconditions(),
+            expected_publication_epoch,
+        )
+        .await;
+        if save_result
+            .as_ref()
+            .ok()
+            .and_then(|info| info.etag.as_deref())
+            .is_some_and(|etag| !etag.is_empty())
+        {
+            return Ok(());
+        }
+
+        let (persisted, revision) = read_config_with_revision(storeapi, DATA_USAGE_OBJ_NAME_PATH.as_str())
+            .await
+            .map_err(ScannerUsageBootstrapPublishError::Reconcile)?;
+        if persisted.as_deref() == Some(data.as_slice()) && matches!(revision, DataUsageCacheRevision::Etag(_)) {
+            return Ok(());
+        }
+        Err(match save_result {
+            Ok(_) => ScannerUsageBootstrapPublishError::MissingEtag,
+            Err(err) => ScannerUsageBootstrapPublishError::Save(err),
+        })
+    }
+
+    inner(storeapi, expected_revision, expected_publication_epoch, leader_epoch)
+        .await
+        .map_err(|err| err.into_scanner_error(context))
+}
+
 pub(super) async fn reset_scanner_usage_state_slots_for_full_rebuild(
     storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
     slots: &[ScannerUsageStateResetSlot],
@@ -1442,48 +1540,15 @@ pub(super) async fn reset_scanner_usage_state_slots_for_full_rebuild(
         .iter()
         .find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str())
         .ok_or_else(|| ScannerError::Other("scanner usage reset primary slot was not inspected".to_string()))?;
-    let marker = DataUsageInfo {
-        last_update: Some(std::time::SystemTime::now()),
-        scanner_epoch: Some(leader_epoch),
-        usage_snapshot_converged: Some(false),
-        usage_snapshot_bootstrap_pending: true,
-        ..Default::default()
-    };
-    let data = serde_json::to_vec(&marker)
-        .map_err(|err| ScannerError::Other(format!("failed to encode scanner usage reset bootstrap marker: {err}")))?;
-    let save_result = save_config_with_publication_admission_for_epoch(
+    publish_scanner_usage_bootstrap_primary(
         storeapi.clone(),
-        DATA_USAGE_OBJ_NAME_PATH.as_str(),
-        data.clone(),
-        primary.revision.preconditions(),
+        &primary.revision,
         expected_epoch,
+        Some(leader_epoch),
+        ScannerUsageBootstrapPublishContext::Reset,
     )
-    .await;
-    if save_result
-        .as_ref()
-        .ok()
-        .and_then(|info| info.etag.as_deref())
-        .is_some_and(|etag| !etag.is_empty())
-    {
-        reset_paths.push(DATA_USAGE_OBJ_NAME_PATH.as_str().to_string());
-    } else {
-        let (persisted, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
-            .await
-            .map_err(|err| ScannerError::Other(format!("failed to reconcile scanner usage reset bootstrap marker: {err}")))?;
-        if persisted.as_deref() != Some(data.as_slice()) || !matches!(revision, DataUsageCacheRevision::Etag(_)) {
-            return Err(ScannerError::Other(match save_result {
-                Ok(_) => "scanner usage reset bootstrap returned no ETag and could not be confirmed".to_string(),
-                Err(err) if scanner_publication_epoch_changed(&err) => {
-                    "scanner usage reset deferred by a movement epoch change".to_string()
-                }
-                Err(EcstoreError::PreconditionFailed) => {
-                    "scanner usage reset primary slot changed before bootstrap publish".to_string()
-                }
-                Err(err) => format!("failed to persist scanner usage reset bootstrap: {err}"),
-            }));
-        }
-        reset_paths.push(DATA_USAGE_OBJ_NAME_PATH.as_str().to_string());
-    }
+    .await?;
+    reset_paths.push(DATA_USAGE_OBJ_NAME_PATH.as_str().to_string());
 
     for slot in slots.iter().filter(|slot| slot.path != DATA_USAGE_OBJ_NAME_PATH.as_str()) {
         if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch).await? {
@@ -1567,7 +1632,7 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
     }
 
     clear_scanner_usage_floor_failure();
-    clear_legacy_empty_usage_floor_recovery_status();
+    clear_legacy_incomplete_usage_floor_recovery_status();
     set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
     super::notify_scanner_cycle_recovery_wake();
     info!(
@@ -1614,33 +1679,48 @@ pub(super) enum PersistedUsageFloorStartup {
     Authoritative,
     Missing,
     BootstrapPending,
-    RecoveredLegacyEmptyFence,
+    RecoveredLegacyIncompleteFence,
 }
 
 #[derive(Clone, Debug)]
-struct LegacyEmptyUsageFloorPrimary {
+struct LegacyIncompleteUsageFloorPrimary {
     revision: DataUsageCacheRevision,
     epoch: u64,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(deny_unknown_fields)]
-struct LegacyEmptyUsageFloorRecoveryMarker {
+struct LegacyIncompleteUsageFloorRecoveryMarker {
     schema_version: u16,
     primary_revision: String,
     leader_epoch: u64,
 }
 
-async fn read_legacy_empty_usage_floor_recovery_marker(
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct LegacyIncompleteUsageFence {
+    claimable_epoch: Option<u64>,
+}
+
+impl LegacyIncompleteUsageFence {
+    fn new(claimable_epoch: Option<u64>) -> Self {
+        Self { claimable_epoch }
+    }
+
+    fn claimable_epoch(self) -> Option<u64> {
+        self.claimable_epoch
+    }
+}
+
+async fn read_legacy_incomplete_usage_floor_recovery_marker(
     storeapi: Arc<impl ScannerObjectIO>,
-) -> Result<Option<(LegacyEmptyUsageFloorRecoveryMarker, DataUsageCacheRevision)>, ScannerError> {
+) -> Result<Option<(LegacyIncompleteUsageFloorRecoveryMarker, DataUsageCacheRevision)>, ScannerError> {
     let (data, revision) = read_config_with_revision(storeapi, DATA_USAGE_RECOVERY_PATH.as_str())
         .await
         .map_err(|err| ScannerError::Other(format!("failed to read scanner usage recovery marker: {err}")))?;
     let Some(data) = data else {
         return Ok(None);
     };
-    let marker = serde_json::from_slice::<LegacyEmptyUsageFloorRecoveryMarker>(&data)
+    let marker = serde_json::from_slice::<LegacyIncompleteUsageFloorRecoveryMarker>(&data)
         .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage recovery marker: {err}")))?;
     if marker.schema_version != 1 || marker.primary_revision.is_empty() || marker.leader_epoch == 0 {
         return Err(ScannerError::Other("scanner usage recovery marker is invalid".to_string()));
@@ -1651,7 +1731,7 @@ async fn read_legacy_empty_usage_floor_recovery_marker(
     Ok(Some((marker, revision)))
 }
 
-async fn clear_legacy_empty_usage_floor_recovery_marker(
+async fn clear_legacy_incomplete_usage_floor_recovery_marker(
     storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
     marker_revision: &DataUsageCacheRevision,
     expected_publication_epoch: u64,
@@ -1685,11 +1765,11 @@ async fn clear_legacy_empty_usage_floor_recovery_marker(
     }
 }
 
-pub(super) async fn complete_legacy_empty_usage_floor_recovery(
+pub(super) async fn complete_legacy_incomplete_usage_floor_recovery(
     storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
     claimed_epoch: u64,
 ) -> Result<(), ScannerError> {
-    let Some((marker, marker_revision)) = read_legacy_empty_usage_floor_recovery_marker(storeapi.clone()).await? else {
+    let Some((marker, marker_revision)) = read_legacy_incomplete_usage_floor_recovery_marker(storeapi.clone()).await? else {
         return Ok(());
     };
     if claimed_epoch <= marker.leader_epoch {
@@ -1709,12 +1789,220 @@ pub(super) async fn complete_legacy_empty_usage_floor_recovery(
     let expected_publication_epoch = scanner_publication_epoch(storeapi.clone())
         .await
         .ok_or_else(|| ScannerError::Other("scanner usage recovery cleanup is blocked by data movement".to_string()))?;
-    clear_legacy_empty_usage_floor_recovery_marker(storeapi, &marker_revision, expected_publication_epoch).await?;
-    clear_legacy_empty_usage_floor_recovery_status();
+    clear_legacy_incomplete_usage_floor_recovery_marker(storeapi, &marker_revision, expected_publication_epoch).await?;
+    clear_legacy_incomplete_usage_floor_recovery_status();
     Ok(())
 }
 
-fn legacy_empty_usage_fence_epoch(data: &[u8], usage: &DataUsageInfo) -> Option<Option<u64>> {
+struct LegacyOptional<T> {
+    present: bool,
+    value: Option<T>,
+}
+
+impl<T> Default for LegacyOptional<T> {
+    fn default() -> Self {
+        Self {
+            present: false,
+            value: None,
+        }
+    }
+}
+
+fn deserialize_legacy_optional<'de, D, T>(deserializer: D) -> Result<LegacyOptional<T>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+    T: Deserialize<'de>,
+{
+    Option::<T>::deserialize(deserializer).map(|value| LegacyOptional { present: true, value })
+}
+
+struct LegacyUniqueMap<V>(std::collections::HashMap<String, V>);
+
+impl<'de, V> Deserialize<'de> for LegacyUniqueMap<V>
+where
+    V: Deserialize<'de>,
+{
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        struct UniqueMapVisitor<V>(std::marker::PhantomData<V>);
+
+        impl<'de, V> serde::de::Visitor<'de> for UniqueMapVisitor<V>
+        where
+            V: Deserialize<'de>,
+        {
+            type Value = LegacyUniqueMap<V>;
+
+            fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                formatter.write_str("a JSON object without duplicate keys")
+            }
+
+            fn visit_map<A>(self, mut entries: A) -> Result<Self::Value, A::Error>
+            where
+                A: serde::de::MapAccess<'de>,
+            {
+                let mut values = std::collections::HashMap::new();
+                while let Some((key, value)) = entries.next_entry::<String, V>()? {
+                    match values.entry(key) {
+                        std::collections::hash_map::Entry::Vacant(entry) => {
+                            entry.insert(value);
+                        }
+                        std::collections::hash_map::Entry::Occupied(entry) => {
+                            return Err(serde::de::Error::custom(format!("duplicate map key `{}`", entry.key())));
+                        }
+                    }
+                }
+                Ok(LegacyUniqueMap(values))
+            }
+        }
+
+        deserializer.deserialize_map(UniqueMapVisitor(std::marker::PhantomData))
+    }
+}
+
+impl<V> LegacyUniqueMap<V> {
+    fn len(&self) -> usize {
+        self.0.len()
+    }
+
+    fn get(&self, key: &str) -> Option<&V> {
+        self.0.get(key)
+    }
+
+    fn iter(&self) -> impl Iterator<Item = (&String, &V)> {
+        self.0.iter()
+    }
+
+    fn values(&self) -> impl Iterator<Item = &V> {
+        self.0.values()
+    }
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct LegacyBucketTargetUsageWire {
+    #[serde(rename = "replication_pending_size")]
+    _replication_pending_size: serde::de::IgnoredAny,
+    #[serde(rename = "replication_failed_size")]
+    _replication_failed_size: serde::de::IgnoredAny,
+    #[serde(rename = "replicated_size")]
+    _replicated_size: serde::de::IgnoredAny,
+    #[serde(rename = "replica_size")]
+    _replica_size: serde::de::IgnoredAny,
+    #[serde(rename = "replication_pending_count")]
+    _replication_pending_count: serde::de::IgnoredAny,
+    #[serde(rename = "replication_failed_count")]
+    _replication_failed_count: serde::de::IgnoredAny,
+    #[serde(rename = "replicated_count")]
+    _replicated_count: serde::de::IgnoredAny,
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct LegacyBucketUsageWire {
+    size: u64,
+    #[serde(rename = "replication_pending_size_v1")]
+    _replication_pending_size_v1: serde::de::IgnoredAny,
+    #[serde(rename = "replication_failed_size_v1")]
+    _replication_failed_size_v1: serde::de::IgnoredAny,
+    #[serde(rename = "replicated_size_v1")]
+    _replicated_size_v1: serde::de::IgnoredAny,
+    #[serde(rename = "replication_pending_count_v1")]
+    _replication_pending_count_v1: serde::de::IgnoredAny,
+    #[serde(rename = "replication_failed_count_v1")]
+    _replication_failed_count_v1: serde::de::IgnoredAny,
+    objects_count: u64,
+    #[serde(rename = "object_size_histogram")]
+    _object_size_histogram: LegacyUniqueMap<u64>,
+    #[serde(rename = "object_versions_histogram")]
+    _object_versions_histogram: LegacyUniqueMap<u64>,
+    versions_count: u64,
+    delete_markers_count: u64,
+    #[serde(rename = "replica_size")]
+    _replica_size: serde::de::IgnoredAny,
+    #[serde(rename = "replica_count")]
+    _replica_count: serde::de::IgnoredAny,
+    #[serde(rename = "replication_info")]
+    _replication_info: LegacyUniqueMap<LegacyBucketTargetUsageWire>,
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct LegacyDiskUsageStatusWire {
+    #[serde(rename = "disk_id")]
+    _disk_id: serde::de::IgnoredAny,
+    #[serde(rename = "pool_index")]
+    _pool_index: serde::de::IgnoredAny,
+    #[serde(rename = "set_index")]
+    _set_index: serde::de::IgnoredAny,
+    #[serde(rename = "disk_index")]
+    _disk_index: serde::de::IgnoredAny,
+    #[serde(rename = "last_update")]
+    _last_update: serde::de::IgnoredAny,
+    #[serde(rename = "snapshot_exists")]
+    _snapshot_exists: serde::de::IgnoredAny,
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct LegacyTierStatsWire {
+    #[serde(rename = "total_size")]
+    _total_size: serde::de::IgnoredAny,
+    #[serde(rename = "num_versions")]
+    _num_versions: serde::de::IgnoredAny,
+    #[serde(rename = "num_objects")]
+    _num_objects: serde::de::IgnoredAny,
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct LegacyAllTierStatsWire {
+    #[serde(rename = "tiers")]
+    _tiers: LegacyUniqueMap<LegacyTierStatsWire>,
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct LegacyUsageWire {
+    #[serde(rename = "total_capacity")]
+    _total_capacity: serde::de::IgnoredAny,
+    #[serde(rename = "total_used_capacity")]
+    _total_used_capacity: serde::de::IgnoredAny,
+    #[serde(rename = "total_free_capacity")]
+    _total_free_capacity: serde::de::IgnoredAny,
+    #[serde(rename = "last_update")]
+    _last_update: serde::de::IgnoredAny,
+    #[serde(default, deserialize_with = "deserialize_legacy_optional")]
+    scanner_epoch: LegacyOptional<u64>,
+    objects_total_count: u64,
+    versions_total_count: u64,
+    delete_markers_total_count: u64,
+    objects_total_size: u64,
+    #[serde(rename = "replication_info")]
+    _replication_info: LegacyUniqueMap<LegacyBucketTargetUsageWire>,
+    #[serde(default, deserialize_with = "deserialize_legacy_optional")]
+    tier_stats: LegacyOptional<LegacyAllTierStatsWire>,
+    buckets_count: u64,
+    buckets_usage: LegacyUniqueMap<LegacyBucketUsageWire>,
+    usage_snapshot_complete: bool,
+    bucket_sizes: LegacyUniqueMap<u64>,
+    #[serde(rename = "disk_usage_status")]
+    _disk_usage_status: Vec<LegacyDiskUsageStatusWire>,
+}
+
+fn decode_legacy_usage_wire(data: &[u8], usage: &DataUsageInfo) -> Option<LegacyUsageWire> {
+    let wire = serde_json::from_slice::<LegacyUsageWire>(data).ok()?;
+    if wire.scanner_epoch.present != usage.scanner_epoch.is_some()
+        || wire.scanner_epoch.value != usage.scanner_epoch
+        || wire.tier_stats.present != usage.tier_stats.is_some()
+    {
+        return None;
+    }
+    Some(wire)
+}
+
+fn legacy_empty_usage_fence(data: &[u8], usage: &DataUsageInfo) -> Option<LegacyIncompleteUsageFence> {
     if usage.last_update.is_none() || usage.scanner_cycle.is_some() {
         return None;
     }
@@ -1730,55 +2018,88 @@ fn legacy_empty_usage_fence_epoch(data: &[u8], usage: &DataUsageInfo) -> Option<
         return None;
     }
 
-    let serde_json::Value::Object(fields) = serde_json::from_slice::<serde_json::Value>(data).ok()? else {
-        return None;
-    };
     // RUSTFS_COMPAT_TODO(backlog-2102): accept only the exact empty usage fence serialized by rc.2/rc.3. Remove after those releases are no longer supported direct-upgrade sources.
-    const REQUIRED_FIELDS: &[&str] = &[
-        "total_capacity",
-        "total_used_capacity",
-        "total_free_capacity",
-        "last_update",
-        "objects_total_count",
-        "versions_total_count",
-        "delete_markers_total_count",
-        "objects_total_size",
-        "replication_info",
-        "buckets_count",
-        "buckets_usage",
-        "usage_snapshot_complete",
-        "bucket_sizes",
-        "disk_usage_status",
-    ];
-    let expected_len = REQUIRED_FIELDS.len() + if usage.scanner_epoch.is_some() { 1 } else { 0 };
-    if fields.len() != expected_len
-        || REQUIRED_FIELDS.iter().any(|field| !fields.contains_key(*field))
-        || (usage.scanner_epoch.is_some() != fields.contains_key("scanner_epoch"))
+    let wire = decode_legacy_usage_wire(data, usage)?;
+    Some(LegacyIncompleteUsageFence::new(wire.scanner_epoch.value))
+}
+
+fn legacy_incomplete_usage_fence(data: &[u8], usage: &DataUsageInfo) -> Option<LegacyIncompleteUsageFence> {
+    legacy_empty_usage_fence(data, usage).or_else(|| legacy_non_empty_usage_fence(data, usage))
+}
+
+// RUSTFS_COMPAT_TODO(backlog-2122): accept rc.1-rc.3 usage floors that were fenced before a scanner cycle completed. Remove after those releases are no longer supported direct-upgrade sources.
+fn legacy_non_empty_usage_fence(data: &[u8], usage: &DataUsageInfo) -> Option<LegacyIncompleteUsageFence> {
+    if usage.last_update.is_none()
+        || usage.scanner_cycle.is_some()
+        || usage.usage_snapshot_bootstrap_pending
+        || usage.usage_snapshot_complete
+        || usage.usage_snapshot_converged.is_some()
+        || usage.usage_snapshot_authoritative_baseline.is_some()
+        || !usage.usage_snapshot_set_states.is_empty()
+        || usage.usage_snapshot_partial
+        || usage.buckets_count == 0
+        || u64::try_from(usage.buckets_usage.len()).ok() != Some(usage.buckets_count)
     {
         return None;
     }
-    Some(usage.scanner_epoch)
+    if usage.scanner_epoch.is_some_and(|epoch| epoch == 0 || epoch >= u64::MAX - 1) {
+        return None;
+    }
+    let wire = decode_legacy_usage_wire(data, usage)?;
+    if wire.usage_snapshot_complete
+        || wire.buckets_count == 0
+        || u64::try_from(wire.buckets_usage.len()).ok() != Some(wire.buckets_count)
+        || wire.bucket_sizes.len() != wire.buckets_usage.len()
+        || wire
+            .buckets_usage
+            .iter()
+            .any(|(bucket, bucket_usage)| wire.bucket_sizes.get(bucket) != Some(&bucket_usage.size))
+    {
+        return None;
+    }
+    let (objects, versions, delete_markers, size) = wire.buckets_usage.values().try_fold(
+        (0_u64, 0_u64, 0_u64, 0_u64),
+        |(objects, versions, delete_markers, size), bucket| {
+            Some((
+                objects.checked_add(bucket.objects_count)?,
+                versions.checked_add(bucket.versions_count)?,
+                delete_markers.checked_add(bucket.delete_markers_count)?,
+                size.checked_add(bucket.size)?,
+            ))
+        },
+    )?;
+    if (objects, versions, delete_markers, size)
+        != (
+            wire.objects_total_count,
+            wire.versions_total_count,
+            wire.delete_markers_total_count,
+            wire.objects_total_size,
+        )
+    {
+        return None;
+    }
+    Some(LegacyIncompleteUsageFence::new(wire.scanner_epoch.value))
 }
 
-async fn recover_legacy_empty_usage_floor(
+async fn recover_legacy_incomplete_usage_floor(
     storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
-    primary: LegacyEmptyUsageFloorPrimary,
+    primary: LegacyIncompleteUsageFloorPrimary,
     expected_publication_epoch: u64,
 ) -> Result<(), ScannerError> {
     let DataUsageCacheRevision::Etag(primary_revision) = &primary.revision else {
-        return Err(ScannerError::Other("legacy empty scanner usage floor has no revision".to_string()));
+        return Err(ScannerError::Other("legacy incomplete scanner usage floor has no revision".to_string()));
     };
-    let marker = LegacyEmptyUsageFloorRecoveryMarker {
+    let marker = LegacyIncompleteUsageFloorRecoveryMarker {
         schema_version: 1,
         primary_revision: primary_revision.clone(),
         leader_epoch: primary.epoch,
     };
     let marker_data = serde_json::to_vec(&marker)
         .map_err(|err| ScannerError::Other(format!("failed to encode scanner usage recovery marker: {err}")))?;
-    match read_legacy_empty_usage_floor_recovery_marker(storeapi.clone()).await? {
+    match read_legacy_incomplete_usage_floor_recovery_marker(storeapi.clone()).await? {
         Some((persisted, _)) if persisted != marker => {
             return Err(ScannerError::Other(
-                "scanner usage recovery marker conflicts with the persisted empty floor".to_string(),
+                "scanner usage recovery marker conflicts with the persisted incomplete floor".to_string(),
             ));
         }
         Some(_) => {}
@@ -1797,7 +2118,7 @@ async fn recover_legacy_empty_usage_floor(
                 .and_then(|info| info.etag.as_deref())
                 .is_some_and(|etag| !etag.is_empty())
             {
-                let persisted = read_legacy_empty_usage_floor_recovery_marker(storeapi.clone()).await?;
+                let persisted = read_legacy_incomplete_usage_floor_recovery_marker(storeapi.clone()).await?;
                 if persisted.as_ref().map(|(persisted, _)| persisted) != Some(&marker) {
                     return Err(ScannerError::Other(match marker_save {
                         Ok(_) => "scanner usage recovery marker returned no ETag and could not be confirmed".to_string(),
@@ -1808,52 +2129,26 @@ async fn recover_legacy_empty_usage_floor(
         }
     }
 
-    let marker = DataUsageInfo {
-        last_update: Some(std::time::SystemTime::now()),
-        scanner_epoch: Some(primary.epoch),
-        usage_snapshot_converged: Some(false),
-        usage_snapshot_bootstrap_pending: true,
-        ..Default::default()
-    };
-    let data = serde_json::to_vec(&marker)
-        .map_err(|err| ScannerError::Other(format!("failed to encode recovered scanner usage bootstrap: {err}")))?;
-    let save_result = save_config_with_publication_admission_for_epoch(
+    publish_scanner_usage_bootstrap_primary(
         storeapi.clone(),
-        DATA_USAGE_OBJ_NAME_PATH.as_str(),
-        data.clone(),
-        primary.revision.preconditions(),
+        &primary.revision,
         expected_publication_epoch,
+        Some(primary.epoch),
+        ScannerUsageBootstrapPublishContext::Recovery,
     )
-    .await;
-    if save_result
-        .as_ref()
-        .ok()
-        .and_then(|info| info.etag.as_deref())
-        .is_some_and(|etag| !etag.is_empty())
-    {
-        warn!(
-            target: "rustfs::scanner",
-            event = EVENT_SCANNER_PERSIST_STATE,
-            component = LOG_COMPONENT_SCANNER,
-            subsystem = LOG_SUBSYSTEM_RUNTIME,
-            state = "legacy_empty_usage_floor_recovered",
-            path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
-            scanner_epoch = primary.epoch,
-            "Scanner recovered a legacy empty usage floor"
-        );
-        return Ok(());
-    }
-
-    let (persisted, revision) = read_config_with_revision(storeapi, DATA_USAGE_OBJ_NAME_PATH.as_str())
-        .await
-        .map_err(|err| ScannerError::Other(format!("failed to reconcile recovered scanner usage bootstrap: {err}")))?;
-    if persisted.as_deref() == Some(data.as_slice()) && matches!(revision, DataUsageCacheRevision::Etag(_)) {
-        return Ok(());
-    }
-    Err(ScannerError::Other(match save_result {
-        Ok(_) => "recovered scanner usage bootstrap returned no ETag and could not be confirmed".to_string(),
-        Err(err) => format!("failed to recover legacy empty scanner usage floor: {err}"),
-    }))
+    .await?;
+    warn!(
+        target: "rustfs::scanner",
+        event = EVENT_SCANNER_PERSIST_STATE,
+        component = LOG_COMPONENT_SCANNER,
+        subsystem = LOG_SUBSYSTEM_RUNTIME,
+        // Keep the published state stable for existing empty-floor alerts.
+        state = "legacy_empty_usage_floor_recovered",
+        path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        scanner_epoch = primary.epoch,
+        "Scanner recovered a legacy incomplete usage floor"
+    );
+    Ok(())
 }
 
 pub(super) fn encode_scanner_cycle_state(
@@ -2045,8 +2340,8 @@ fn resolve_bootstrap_backup_slot(
             if backup_epoch >= primary_epoch {
                 update_persisted_usage_floor(resolution.floor, slot.usage, slot.backup_path)?;
             }
-        } else if let Some(epoch) = legacy_empty_usage_fence_epoch(slot.data, slot.usage) {
-            if let Some(epoch) = epoch {
+        } else if let Some(fence) = legacy_incomplete_usage_fence(slot.data, slot.usage) {
+            if let Some(epoch) = fence.claimable_epoch() {
                 resolution.floor.leader_epoch = resolution.floor.leader_epoch.max(epoch);
             }
         } else {
@@ -2056,10 +2351,12 @@ fn resolve_bootstrap_backup_slot(
         }
         return Ok(BootstrapBackupAction::Resume);
     }
-    let compatible_empty_fence = legacy_empty_usage_fence_epoch(slot.data, slot.usage).is_some_and(|epoch| {
-        epoch.is_none_or(|epoch| slot.bootstrap_epoch.is_some_and(|bootstrap_epoch| epoch <= bootstrap_epoch))
+    let compatible_incomplete_fence = legacy_incomplete_usage_fence(slot.data, slot.usage).is_some_and(|fence| {
+        fence
+            .claimable_epoch()
+            .is_none_or(|epoch| slot.bootstrap_epoch.is_some_and(|bootstrap_epoch| epoch <= bootstrap_epoch))
     });
-    if compatible_empty_fence {
+    if compatible_incomplete_fence {
         return Ok(BootstrapBackupAction::Resume);
     }
     if slot.recovered_bootstrap && data_usage_info_has_persisted_baseline_identity(slot.usage) {
@@ -2084,7 +2381,7 @@ pub(super) async fn persisted_usage_floor_for_startup(
     let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
         return Err(ScannerError::Other("scanner usage floor read is blocked by data movement".to_string()));
     };
-    let recovery_marker = read_legacy_empty_usage_floor_recovery_marker(storeapi.clone()).await?;
+    let recovery_marker = read_legacy_incomplete_usage_floor_recovery_marker(storeapi.clone()).await?;
     let mut floor = PersistedUsageFloor::default();
     let mut found_any = false;
     let mut bootstrap_pending = false;
@@ -2099,7 +2396,7 @@ pub(super) async fn persisted_usage_floor_for_startup(
     let mut invalid_baseline_epoch = recovery_marker.as_ref().map(|(marker, _)| marker.leader_epoch);
     let mut unrecoverable_baseline_path: Option<String> = None;
     let mut stale_authoritative_path: Option<String> = None;
-    let mut legacy_empty_primary: Option<LegacyEmptyUsageFloorPrimary> = None;
+    let mut legacy_incomplete_primary: Option<LegacyIncompleteUsageFloorPrimary> = None;
     for primary_path in [DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()] {
         let backup_path = format!("{primary_path}.bkp");
         let is_v2_path = primary_path == DATA_USAGE_OBJ_NAME_PATH.as_str();
@@ -2148,14 +2445,12 @@ pub(super) async fn persisted_usage_floor_for_startup(
                     } else if !data_usage_info_has_persisted_baseline_identity(&usage) {
                         invalid_baseline_path.get_or_insert_with(|| primary_path.to_string());
                         invalid_baseline_epoch = invalid_baseline_epoch.max(usage.scanner_epoch);
-                        match legacy_empty_usage_fence_epoch(&data, &usage) {
-                            Some(Some(epoch)) if is_v2_path => {
-                                legacy_empty_primary = Some(LegacyEmptyUsageFloorPrimary { revision, epoch });
-                            }
-                            Some(_) => {}
-                            None => {
-                                unrecoverable_baseline_path.get_or_insert_with(|| primary_path.to_string());
+                        if let Some(fence) = legacy_incomplete_usage_fence(&data, &usage) {
+                            if is_v2_path && let Some(epoch) = fence.claimable_epoch() {
+                                legacy_incomplete_primary = Some(LegacyIncompleteUsageFloorPrimary { revision, epoch });
                             }
+                        } else {
+                            unrecoverable_baseline_path.get_or_insert_with(|| primary_path.to_string());
                         }
                         None
                     } else {
@@ -2243,7 +2538,7 @@ pub(super) async fn persisted_usage_floor_for_startup(
                 if !data_usage_info_has_persisted_baseline_identity(&usage) {
                     invalid_baseline_path.get_or_insert_with(|| backup_path.clone());
                     invalid_baseline_epoch = invalid_baseline_epoch.max(usage.scanner_epoch);
-                    if legacy_empty_usage_fence_epoch(&data, &usage).is_none() {
+                    if legacy_incomplete_usage_fence(&data, &usage).is_none() {
                         unrecoverable_baseline_path.get_or_insert_with(|| backup_path.clone());
                     }
                     // This is still persisted state, so it must not enable a
@@ -2294,17 +2589,18 @@ pub(super) async fn persisted_usage_floor_for_startup(
         if allow_missing_for_bootstrap
             && unrecoverable_baseline_path.is_none()
             && stale_authoritative_path.is_none()
-            && let Some(mut primary) = legacy_empty_primary
+            && let Some(mut primary) = legacy_incomplete_primary
         {
             primary.epoch = primary.epoch.max(invalid_baseline_epoch.unwrap_or_default());
-            recover_legacy_empty_usage_floor(storeapi.clone(), primary.clone(), read_epoch).await?;
-            record_legacy_empty_usage_floor_recovery_pending(primary.epoch);
+            let leader_epoch = primary.epoch;
+            recover_legacy_incomplete_usage_floor(storeapi.clone(), primary, read_epoch).await?;
+            record_legacy_incomplete_usage_floor_recovery_pending(leader_epoch);
             return Ok((
                 PersistedUsageFloor {
                     next_cycle: 0,
-                    leader_epoch: primary.epoch,
+                    leader_epoch,
                 },
-                PersistedUsageFloorStartup::RecoveredLegacyEmptyFence,
+                PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence,
             ));
         }
         if let Some(path) = stale_authoritative_path {
@@ -2317,7 +2613,7 @@ pub(super) async fn persisted_usage_floor_for_startup(
         }
         if let Some(path) = invalid_baseline_path {
             return Err(ScannerError::Other(format!(
-                "persisted scanner usage floor from {path} has no authoritative baseline or newer valid backup"
+                "persisted scanner usage floor from {path} has no authoritative baseline or newer valid backup; recover with POST /rustfs/admin/v3/scanner/usage-state/reset using mode full-rebuild"
             )));
         }
         if !allow_missing_for_bootstrap {
@@ -2369,8 +2665,8 @@ pub(super) async fn persisted_usage_floor_for_startup(
             .as_ref()
             .map(|(marker, _)| marker.leader_epoch)
             .unwrap_or(floor.leader_epoch);
-        record_legacy_empty_usage_floor_recovery_pending(recovery_epoch);
-        PersistedUsageFloorStartup::RecoveredLegacyEmptyFence
+        record_legacy_incomplete_usage_floor_recovery_pending(recovery_epoch);
+        PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence
     } else if bootstrap_pending {
         if let Some(path) = unrecoverable_baseline_path {
             return Err(ScannerError::Other(format!(
@@ -2384,7 +2680,7 @@ pub(super) async fn persisted_usage_floor_for_startup(
     if found_any && let Some((_, marker_revision)) = recovery_marker.as_ref() {
         drop(publication_admission);
         let marker_cleared =
-            match clear_legacy_empty_usage_floor_recovery_marker(storeapi.clone(), marker_revision, read_epoch).await {
+            match clear_legacy_incomplete_usage_floor_recovery_marker(storeapi.clone(), marker_revision, read_epoch).await {
                 Ok(()) => true,
                 Err(err) => {
                     warn!(
@@ -2407,7 +2703,7 @@ pub(super) async fn persisted_usage_floor_for_startup(
             ));
         };
         if marker_cleared {
-            clear_legacy_empty_usage_floor_recovery_status();
+            clear_legacy_incomplete_usage_floor_recovery_status();
         }
         clear_scanner_usage_floor_failure();
         return Ok((floor, state));

crates/scanner/src/scanner/leadership.rs

@@ -185,42 +185,14 @@ pub(super) async fn initialize_usage_baseline_bootstrap(
             "scanner usage baseline bootstrap is blocked by data movement".to_string(),
         ));
     };
-    let baseline = DataUsageInfo {
-        last_update: Some(std::time::SystemTime::now()),
-        usage_snapshot_converged: Some(false),
-        usage_snapshot_bootstrap_pending: true,
-        ..Default::default()
-    };
-    let data = serde_json::to_vec(&baseline)
-        .map_err(|err| ScannerError::Other(format!("failed to encode scanner usage baseline bootstrap: {err}")))?;
-    let save_result = save_config_with_publication_admission_for_epoch(
-        storeapi.clone(),
-        DATA_USAGE_OBJ_NAME_PATH.as_str(),
-        data.clone(),
-        DataUsageCacheRevision::Missing.preconditions(),
+    publish_scanner_usage_bootstrap_primary(
+        storeapi,
+        &DataUsageCacheRevision::Missing,
         expected_epoch,
+        None,
+        ScannerUsageBootstrapPublishContext::Initial,
     )
-    .await;
-    if save_result
-        .as_ref()
-        .ok()
-        .and_then(|info| info.etag.as_deref())
-        .is_some_and(|etag| !etag.is_empty())
-    {
-        return Ok(());
-    }
-
-    let (persisted, revision) = read_config_with_revision(storeapi, DATA_USAGE_OBJ_NAME_PATH.as_str())
-        .await
-        .map_err(|err| ScannerError::Other(format!("failed to reconcile scanner usage bootstrap: {err}")))?;
-    if persisted.as_deref() == Some(data.as_slice()) && matches!(revision, DataUsageCacheRevision::Etag(_)) {
-        return Ok(());
-    }
-
-    Err(ScannerError::Other(match save_result {
-        Ok(_) => "scanner usage bootstrap returned no ETag and could not be confirmed".to_string(),
-        Err(err) => format!("failed to persist scanner usage bootstrap: {err}"),
-    }))
+    .await
 }
 
 pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(

crates/scanner/src/scanner/tests.rs

@@ -462,6 +462,7 @@ async fn cycle_budget_persist_cursor_failure_is_recovery_required() {
         &mut cycle,
         &mut revision,
         &mut leader_epoch,
+        false,
         std::future::pending(),
     )
     .await;
@@ -478,6 +479,52 @@ async fn cycle_budget_persist_cursor_failure_is_recovery_required() {
     assert!(report.leader_lease_without_progress);
 }
 
+#[tokio::test]
+async fn cycle_budget_fence_accepts_bootstrap_pending_usage_marker() {
+    let store = Arc::new(MemoryConfigStore::default());
+    initialize_usage_baseline_bootstrap(store.clone())
+        .await
+        .expect("usage reset should publish a bootstrap marker");
+
+    let ctx = CancellationToken::new();
+    let mut revision = DataUsageCacheRevision::Missing;
+    let mut cycle = CurrentCycle {
+        current: 12,
+        next: 12,
+        ..Default::default()
+    };
+    let mut leader_epoch = 0;
+
+    let fenced = fence_scanner_epoch_after_cycle_timeout(
+        &ctx,
+        store.clone(),
+        &mut cycle,
+        &mut revision,
+        &mut leader_epoch,
+        true,
+        std::future::pending(),
+    )
+    .await;
+
+    assert!(fenced, "a valid reset bootstrap marker must not force cycle recovery after budget expiry");
+    assert!(!cycle_timeout_requires_recovery(true, true, fenced));
+    assert_eq!(leader_epoch, 1);
+
+    let persisted_cycle = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH)
+        .await
+        .expect("timeout fence should persist the next leader epoch");
+    let (_, persisted_epoch) = decode_scanner_cycle_state(&persisted_cycle).expect("persisted epoch fence should decode");
+    assert_eq!(persisted_epoch, 1);
+
+    let usage = read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("timeout fence should keep the bootstrap usage marker");
+    let usage = serde_json::from_slice::<DataUsageInfo>(&usage).expect("bootstrap marker should decode");
+    assert!(data_usage_info_is_bootstrap_pending(&usage));
+    assert_eq!(usage.scanner_epoch, Some(1));
+    assert!(!data_usage_info_has_persisted_baseline_identity(&usage));
+}
+
 #[tokio::test]
 async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
     let (_temp_dir, store) = setup_scanner_cycle_store().await;
@@ -515,6 +562,7 @@ async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
             cycle_revision: &mut cycle_revision,
             leader_epoch: &mut leader_epoch,
             cycle_budget: &budget,
+            allow_bootstrap_pending: false,
         },
         true,
         &mut guard,
@@ -2237,6 +2285,53 @@ fn rc3_legacy_empty_usage_fence(epoch: Option<u64>) -> Vec<u8> {
     serde_json::to_vec(&value).expect("rc.3 legacy empty usage fence fixture should encode")
 }
 
+fn rc3_legacy_non_empty_usage_fence(epoch: Option<u64>) -> Vec<u8> {
+    // Pinned rc.3 field set. Leadership preserved this data and added only
+    // scanner_epoch when the producing scanner cycle had not completed.
+    const RC3_NON_EMPTY_USAGE_FENCE: &str = r#"{
+        "total_capacity":2000000000,
+        "total_used_capacity":1000000000,
+        "total_free_capacity":1000000000,
+        "last_update":{"secs_since_epoch":1,"nanos_since_epoch":0},
+        "objects_total_count":156382067,
+        "versions_total_count":156382070,
+        "delete_markers_total_count":3,
+        "objects_total_size":987654321,
+        "replication_info":{},
+        "buckets_count":1,
+        "buckets_usage":{
+            "photos":{
+                "size":987654321,
+                "replication_pending_size_v1":0,
+                "replication_failed_size_v1":0,
+                "replicated_size_v1":0,
+                "replication_pending_count_v1":0,
+                "replication_failed_count_v1":0,
+                "objects_count":156382067,
+                "object_size_histogram":{},
+                "object_versions_histogram":{},
+                "versions_count":156382070,
+                "delete_markers_count":3,
+                "replica_size":0,
+                "replica_count":0,
+                "replication_info":{}
+            }
+        },
+        "usage_snapshot_complete":false,
+        "bucket_sizes":{"photos":987654321},
+        "disk_usage_status":[]
+    }"#;
+    let mut value = serde_json::from_str::<serde_json::Value>(RC3_NON_EMPTY_USAGE_FENCE)
+        .expect("pinned rc.3 non-empty usage fence should decode");
+    if let Some(epoch) = epoch {
+        value
+            .as_object_mut()
+            .expect("legacy non-empty usage fence should be a JSON object")
+            .insert("scanner_epoch".to_string(), serde_json::Value::from(epoch));
+    }
+    serde_json::to_vec(&value).expect("rc.3 legacy non-empty usage fence fixture should encode")
+}
+
 #[tokio::test]
 async fn scanner_usage_floor_recovers_rc3_empty_fences_and_preserves_cycle_number() {
     let store = Arc::new(MemoryConfigStore::default());
@@ -2265,7 +2360,7 @@ async fn scanner_usage_floor_recovers_rc3_empty_fences_and_preserves_cycle_numbe
             leader_epoch: 7,
         }
     );
-    assert_eq!(startup, PersistedUsageFloorStartup::RecoveredLegacyEmptyFence);
+    assert_eq!(startup, PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence);
 
     let primary = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
         .await
@@ -2279,7 +2374,7 @@ async fn scanner_usage_floor_recovers_rc3_empty_fences_and_preserves_cycle_numbe
         .await
         .expect("recovery marker should survive a restart before leadership claim");
     assert_eq!(restart_floor.leader_epoch, 7);
-    assert_eq!(restart_state, PersistedUsageFloorStartup::RecoveredLegacyEmptyFence);
+    assert_eq!(restart_state, PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence);
     let mut cycle = CurrentCycle {
         current: 17_117,
         next: 17_118,
@@ -2368,7 +2463,7 @@ async fn scanner_usage_floor_recovers_rc3_empty_fences_and_preserves_cycle_numbe
         assert!(!persisted_reset.info.snapshot_complete);
         assert!(persisted_reset.cache.is_empty());
     }
-    complete_legacy_empty_usage_floor_recovery(store.clone(), leader_epoch)
+    complete_legacy_incomplete_usage_floor_recovery(store.clone(), leader_epoch)
         .await
         .expect("leadership claim should retire the recovery marker");
     assert!(matches!(
@@ -2382,6 +2477,277 @@ async fn scanner_usage_floor_recovers_rc3_empty_fences_and_preserves_cycle_numbe
     assert_eq!(claimed_state, PersistedUsageFloorStartup::BootstrapPending);
 }
 
+#[tokio::test]
+async fn scanner_usage_floor_recovers_rc3_non_empty_incomplete_fence() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
+    store
+        .objects
+        .lock()
+        .await
+        .insert(primary_key.clone(), rc3_legacy_non_empty_usage_fence(Some(13)));
+    store.revisions.lock().await.insert(primary_key, 1);
+
+    save_config(
+        store.clone(),
+        LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
+        rc3_legacy_non_empty_usage_fence(None),
+    )
+    .await
+    .expect("legacy usage should persist");
+
+    let (floor, startup) = persisted_usage_floor_for_startup(store.clone(), true)
+        .await
+        .expect("rc.3 non-empty incomplete fence should enter recovery");
+    assert_eq!(
+        floor,
+        PersistedUsageFloor {
+            next_cycle: 0,
+            leader_epoch: 13,
+        }
+    );
+    assert_eq!(startup, PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence);
+
+    let primary = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("recovered usage bootstrap should replace the old floor");
+    let pending = serde_json::from_slice::<DataUsageInfo>(&primary).expect("recovered usage bootstrap should decode");
+    assert!(data_usage_info_is_bootstrap_pending(&pending));
+    assert!(!data_usage_info_has_persisted_baseline_identity(&pending));
+    assert_eq!(pending.scanner_epoch, Some(13));
+    assert!(read_config(store, DATA_USAGE_RECOVERY_PATH.as_str()).await.is_ok());
+}
+
+#[tokio::test]
+async fn scanner_usage_floor_prefers_newer_backup_over_rc3_non_empty_incomplete_fence() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
+    store
+        .objects
+        .lock()
+        .await
+        .insert(primary_key, rc3_legacy_non_empty_usage_fence(Some(13)));
+
+    let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 2);
+    backup.scanner_epoch = Some(14);
+    backup.scanner_cycle = Some(9845);
+    save_config(
+        store.clone(),
+        &format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()),
+        serde_json::to_vec(&backup).expect("newer backup should encode"),
+    )
+    .await
+    .expect("newer backup should persist");
+
+    let (floor, startup) = persisted_usage_floor_for_startup(store.clone(), true)
+        .await
+        .expect("newer authoritative backup should win over the old incomplete floor");
+    assert_eq!(
+        floor,
+        PersistedUsageFloor {
+            next_cycle: 9846,
+            leader_epoch: 14,
+        }
+    );
+    assert_eq!(startup, PersistedUsageFloorStartup::Authoritative);
+    assert!(matches!(
+        read_config(store, DATA_USAGE_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
+#[tokio::test]
+async fn scanner_usage_floor_rejects_noncanonical_non_empty_incomplete_fences() {
+    let base = serde_json::from_slice::<serde_json::Value>(&rc3_legacy_non_empty_usage_fence(Some(13)))
+        .expect("pinned rc.3 usage fence should decode");
+    let mut cases = Vec::new();
+
+    let mut unknown_top_level = base.clone();
+    unknown_top_level["future_field"] = serde_json::Value::Bool(true);
+    cases.push(("unknown top-level field", unknown_top_level));
+
+    let mut unknown_bucket_field = base.clone();
+    unknown_bucket_field["buckets_usage"]["photos"]["future_field"] = serde_json::Value::Bool(true);
+    cases.push(("unknown bucket field", unknown_bucket_field));
+
+    let mut wrong_bucket_size = base.clone();
+    wrong_bucket_size["bucket_sizes"]["photos"] = serde_json::Value::from(987_654_320_u64);
+    cases.push(("bucket size mismatch", wrong_bucket_size));
+
+    let mut wrong_total = base.clone();
+    wrong_total["objects_total_count"] = serde_json::Value::from(156_382_068_u64);
+    cases.push(("object total mismatch", wrong_total));
+
+    let mut wrong_versions = base.clone();
+    wrong_versions["versions_total_count"] = serde_json::Value::from(156_382_071_u64);
+    cases.push(("version total mismatch", wrong_versions));
+
+    let mut wrong_delete_markers = base.clone();
+    wrong_delete_markers["delete_markers_total_count"] = serde_json::Value::from(4_u64);
+    cases.push(("delete marker total mismatch", wrong_delete_markers));
+
+    let mut wrong_total_size = base.clone();
+    wrong_total_size["objects_total_size"] = serde_json::Value::from(987_654_320_u64);
+    cases.push(("object size total mismatch", wrong_total_size));
+
+    let mut wrong_cardinality = base.clone();
+    wrong_cardinality["buckets_count"] = serde_json::Value::from(2_u64);
+    cases.push(("bucket cardinality mismatch", wrong_cardinality));
+
+    let mut overflow = base.clone();
+    let mut overflow_bucket = overflow["buckets_usage"]["photos"].clone();
+    overflow_bucket["objects_count"] = serde_json::Value::from(u64::MAX);
+    overflow_bucket["size"] = serde_json::Value::from(0_u64);
+    overflow["buckets_usage"]["overflow"] = overflow_bucket;
+    overflow["bucket_sizes"]["overflow"] = serde_json::Value::from(0_u64);
+    overflow["buckets_count"] = serde_json::Value::from(2_u64);
+    cases.push(("checked total overflow", overflow));
+
+    let mut invalid_epoch = base;
+    invalid_epoch["scanner_epoch"] = serde_json::Value::from(0_u64);
+    cases.push(("invalid epoch", invalid_epoch));
+
+    for (case, value) in cases {
+        let store = Arc::new(MemoryConfigStore::default());
+        let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
+        let original = serde_json::to_vec(&value).expect("noncanonical usage fixture should encode");
+        store.objects.lock().await.insert(primary_key.clone(), original.clone());
+        store.revisions.lock().await.insert(primary_key, 1);
+
+        let err = persisted_usage_floor_for_startup(store.clone(), true)
+            .await
+            .expect_err("noncanonical incomplete usage must remain fail-closed");
+        assert!(err.to_string().contains("usage-state/reset"), "unexpected error for {case}: {err}");
+        assert_eq!(
+            read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+                .await
+                .expect("rejected usage primary should remain"),
+            original,
+            "rejected primary changed for {case}"
+        );
+        assert!(
+            matches!(
+                read_config(store, DATA_USAGE_RECOVERY_PATH.as_str()).await,
+                Err(EcstoreError::ConfigNotFound)
+            ),
+            "recovery marker should not be written for {case}"
+        );
+    }
+}
+
+#[tokio::test]
+async fn scanner_usage_floor_rejects_duplicate_legacy_fields() {
+    let base = String::from_utf8(rc3_legacy_non_empty_usage_fence(Some(13))).expect("pinned rc.3 usage fence should be UTF-8");
+    let base_value = serde_json::from_str::<serde_json::Value>(&base).expect("pinned rc.3 usage fence should decode");
+    let duplicate_top_level = base.replacen(
+        "\"objects_total_count\":156382067",
+        "\"objects_total_count\":156382067,\"objects_total_count\":156382067",
+        1,
+    );
+    let duplicate_bucket = base.replacen("\"size\":987654321", "\"size\":987654321,\"size\":987654321", 1);
+    let bucket = serde_json::to_string(&base_value["buckets_usage"]["photos"]).expect("pinned rc.3 bucket usage should encode");
+    let bucket_map = format!("\"buckets_usage\":{{\"photos\":{bucket}}}");
+    let duplicate_bucket_key =
+        base.replacen(&bucket_map, &format!("\"buckets_usage\":{{\"photos\":{bucket},\"photos\":{bucket}}}"), 1);
+    let duplicate_bucket_size_key = base.replacen(
+        "\"bucket_sizes\":{\"photos\":987654321}",
+        "\"bucket_sizes\":{\"photos\":987654321,\"photos\":987654321}",
+        1,
+    );
+    let duplicate_histogram_key =
+        base.replacen("\"object_size_histogram\":{}", "\"object_size_histogram\":{\"small\":1,\"small\":1}", 1);
+    let mut duplicate_target_field = base.clone();
+    let target_map = "\"replication_info\":{\"target\":{\"replication_pending_size\":0,\"replication_failed_size\":0,\"replicated_size\":0,\"replica_size\":0,\"replication_pending_count\":0,\"replication_failed_count\":0,\"replicated_count\":0,\"replicated_count\":0}}";
+    let target_offset = duplicate_target_field
+        .rfind("\"replication_info\":{}")
+        .expect("pinned fixture should contain bucket replication info");
+    duplicate_target_field.replace_range(target_offset..target_offset + "\"replication_info\":{}".len(), target_map);
+    let target = "{\"replication_pending_size\":0,\"replication_failed_size\":0,\"replicated_size\":0,\"replica_size\":0,\"replication_pending_count\":0,\"replication_failed_count\":0,\"replicated_count\":0}";
+    let mut duplicate_target_key = base.clone();
+    let target_offset = duplicate_target_key
+        .rfind("\"replication_info\":{}")
+        .expect("pinned fixture should contain bucket replication info");
+    let target_map = format!("\"replication_info\":{{\"target\":{target},\"target\":{target}}}");
+    duplicate_target_key.replace_range(target_offset..target_offset + "\"replication_info\":{}".len(), &target_map);
+
+    let tier = "{\"total_size\":0,\"num_versions\":0,\"num_objects\":0}";
+    let duplicate_tier_key = base.replacen(
+        '{',
+        &format!("{{\"tier_stats\":{{\"tiers\":{{\"STANDARD\":{tier},\"STANDARD\":{tier}}}}},"),
+        1,
+    );
+
+    for (case, original, expected_error) in [
+        ("top-level field", duplicate_top_level.into_bytes(), "duplicate field"),
+        ("bucket field", duplicate_bucket.into_bytes(), "duplicate field"),
+        ("replication target field", duplicate_target_field.into_bytes(), "duplicate field"),
+        ("bucket map key", duplicate_bucket_key.into_bytes(), "usage-state/reset"),
+        ("bucket size map key", duplicate_bucket_size_key.into_bytes(), "usage-state/reset"),
+        ("histogram map key", duplicate_histogram_key.into_bytes(), "usage-state/reset"),
+        ("replication target map key", duplicate_target_key.into_bytes(), "usage-state/reset"),
+        ("tier map key", duplicate_tier_key.into_bytes(), "usage-state/reset"),
+    ] {
+        let store = Arc::new(MemoryConfigStore::default());
+        let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
+        store.objects.lock().await.insert(primary_key.clone(), original.clone());
+        store.revisions.lock().await.insert(primary_key, 1);
+
+        let err = match persisted_usage_floor_for_startup(store.clone(), true).await {
+            Err(err) => err,
+            Ok(result) => panic!("duplicate {case} must remain fail-closed: {result:?}"),
+        };
+        assert!(err.to_string().contains(expected_error), "unexpected error for {case}: {err}");
+        assert_eq!(
+            read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+                .await
+                .expect("rejected duplicate-field primary should remain"),
+            original,
+            "rejected primary changed for {case}"
+        );
+        assert!(
+            matches!(
+                read_config(store, DATA_USAGE_RECOVERY_PATH.as_str()).await,
+                Err(EcstoreError::ConfigNotFound)
+            ),
+            "recovery marker should not be written for {case}"
+        );
+    }
+}
+
+#[tokio::test]
+async fn scanner_usage_floor_rejects_current_schema_incomplete_snapshot() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
+    let mut current = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 2);
+    current.usage_snapshot_complete = false;
+    current.scanner_epoch = Some(13);
+    current.scanner_cycle = None;
+    let original = serde_json::to_vec(&current).expect("current incomplete usage should encode");
+    assert!(
+        serde_json::from_slice::<serde_json::Value>(&original)
+            .expect("current incomplete usage should decode")
+            .get("usage_snapshot_partial")
+            .is_some()
+    );
+    store.objects.lock().await.insert(primary_key.clone(), original.clone());
+    store.revisions.lock().await.insert(primary_key, 1);
+
+    let err = persisted_usage_floor_for_startup(store.clone(), true)
+        .await
+        .expect_err("current schema incomplete usage must remain fail-closed");
+    assert!(err.to_string().contains("usage-state/reset"), "unexpected error: {err}");
+    assert_eq!(
+        read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+            .await
+            .expect("rejected current usage primary should remain"),
+        original
+    );
+    assert!(matches!(
+        read_config(store, DATA_USAGE_RECOVERY_PATH.as_str()).await,
+        Err(EcstoreError::ConfigNotFound)
+    ));
+}
+
 #[tokio::test]
 async fn scanner_usage_floor_recovery_preserves_newer_authoritative_companion_floor() {
     for companion_path in [
@@ -2430,7 +2796,7 @@ async fn scanner_usage_floor_recovery_preserves_newer_authoritative_companion_fl
             .expect("a newer authoritative companion should advance the recovery floor");
         assert_eq!(floor.leader_epoch, 8, "unexpected companion path: {companion_path}");
         assert_eq!(floor.next_cycle, 12, "unexpected companion path: {companion_path}");
-        assert_eq!(state, PersistedUsageFloorStartup::RecoveredLegacyEmptyFence);
+        assert_eq!(state, PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence);
     }
 }
 
@@ -2485,7 +2851,7 @@ async fn scanner_usage_floor_recovery_fences_non_authoritative_legacy_backup() {
                 .expect("an exact empty backup should contribute its epoch fence");
             assert_eq!(floor.leader_epoch, 9);
             assert_eq!(floor.next_cycle, 12);
-            assert_eq!(state, PersistedUsageFloorStartup::RecoveredLegacyEmptyFence);
+            assert_eq!(state, PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence);
         }
     }
 }
@@ -2514,7 +2880,7 @@ async fn scanner_usage_floor_recovery_resumes_after_marker_only_crash_point() {
         .await
         .expect("the durable marker should resume the primary conversion");
     assert_eq!(floor.leader_epoch, 7);
-    assert_eq!(state, PersistedUsageFloorStartup::RecoveredLegacyEmptyFence);
+    assert_eq!(state, PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence);
     let recovered = read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
         .await
         .expect("recovered bootstrap should replace the legacy primary");
@@ -2540,7 +2906,7 @@ async fn scanner_usage_floor_recovery_reconciles_marker_post_commit_error() {
         .await
         .expect("a committed recovery marker should reconcile after an ambiguous error");
     assert_eq!(floor.leader_epoch, 7);
-    assert_eq!(state, PersistedUsageFloorStartup::RecoveredLegacyEmptyFence);
+    assert_eq!(state, PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence);
     assert!(read_config(store, DATA_USAGE_RECOVERY_PATH.as_str()).await.is_ok());
 }
 
@@ -2579,7 +2945,7 @@ async fn scanner_usage_floor_recovery_reconciles_marker_delete_post_commit_error
         .await
         .insert(memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_RECOVERY_PATH.as_str()));
 
-    complete_legacy_empty_usage_floor_recovery(store.clone(), leader_epoch)
+    complete_legacy_incomplete_usage_floor_recovery(store.clone(), leader_epoch)
         .await
         .expect("a committed marker delete should reconcile after an ambiguous error");
     assert!(matches!(
@@ -2621,12 +2987,12 @@ async fn scanner_usage_floor_recovery_retry_budget_uses_marker_epoch_identity()
         .await
         .expect("claimed bootstrap should retain its recovery identity");
     assert_eq!(floor.leader_epoch, 8);
-    assert_eq!(state, PersistedUsageFloorStartup::RecoveredLegacyEmptyFence);
+    assert_eq!(state, PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence);
     let status = scanner_cycle_recovery_status();
     assert_eq!(status.leader_epoch, Some(7));
     assert_eq!(status.retry_count, 3);
     assert_eq!(status.first_detected_at_unix_secs, first_detected);
-    clear_legacy_empty_usage_floor_recovery_status();
+    clear_legacy_incomplete_usage_floor_recovery_status();
 }
 
 #[tokio::test]
@@ -2782,7 +3148,7 @@ fn scanner_usage_floor_failure_is_exposed_and_cleared() {
 #[test]
 #[serial]
 fn scanner_usage_floor_recovery_stays_retryable_until_claim_cleanup() {
-    record_legacy_empty_usage_floor_recovery_pending(7);
+    record_legacy_incomplete_usage_floor_recovery_pending(7);
     let pending = scanner_cycle_recovery_status();
     assert_eq!(pending.state, "usage_floor_recovery_pending");
     assert_eq!(pending.classification.as_deref(), Some("legacy_empty_usage_floor"));
@@ -2792,12 +3158,12 @@ fn scanner_usage_floor_recovery_stays_retryable_until_claim_cleanup() {
     let first_detected = pending.first_detected_at_unix_secs;
 
     assert!(record_scanner_cycle_recovery_retry(3));
-    record_legacy_empty_usage_floor_recovery_pending(7);
+    record_legacy_incomplete_usage_floor_recovery_pending(7);
     let retried = scanner_cycle_recovery_status();
     assert_eq!(retried.retry_count, 3);
     assert_eq!(retried.first_detected_at_unix_secs, first_detected);
 
-    clear_legacy_empty_usage_floor_recovery_status();
+    clear_legacy_incomplete_usage_floor_recovery_status();
     assert_eq!(scanner_cycle_recovery_status().state, "healthy");
 }
 

crates/utils/src/http/metadata_compat.rs

@@ -88,6 +88,21 @@ pub const SUFFIX_TIER_SKIP_FV_ID: &str = "tier-skip-fvid";
 /// Per-target delete-marker version ids are stored one key per target ARN.
 pub const SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX: &str = "replication-delete-marker-version-";
 
+// On-demand migration provenance. Written by the migration write-back onto
+// every pulled object so operators and later tooling can tell a migrated
+// object from a client write and audit where it came from. Internal keys, so
+// the existing internal-key filter keeps them out of client-visible metadata.
+/// Source of a pulled object as `<provider>:<bucket>`.
+pub const SUFFIX_ODM_SOURCE: &str = "odm-source";
+/// ETag the source reported for the pulled object, verbatim.
+pub const SUFFIX_ODM_SOURCE_ETAG: &str = "odm-source-etag";
+/// Source `Last-Modified` of the pulled object, RFC 3339.
+pub const SUFFIX_ODM_SOURCE_LAST_MODIFIED: &str = "odm-source-last-modified";
+/// Source version id of the pulled object; empty for an unversioned source.
+pub const SUFFIX_ODM_SOURCE_VERSION_ID: &str = "odm-source-version-id";
+/// When the object was pulled from the source, RFC 3339.
+pub const SUFFIX_ODM_PULLED_AT: &str = "odm-pulled-at";
+
 /// Case-insensitive (ASCII) check that `s` begins with `prefix`. Equivalent to
 /// `s.to_lowercase().starts_with(prefix)` when `prefix` is ASCII (as both internal prefixes are),
 /// but without allocating.
@@ -591,6 +606,80 @@ mod tests {
         assert!(!contains_key_bytes(&meta_sys, &long_suffix));
     }
 
+    const ODM_PROVENANCE_SUFFIXES: [&str; 5] = [
+        SUFFIX_ODM_SOURCE,
+        SUFFIX_ODM_SOURCE_ETAG,
+        SUFFIX_ODM_SOURCE_LAST_MODIFIED,
+        SUFFIX_ODM_SOURCE_VERSION_ID,
+        SUFFIX_ODM_PULLED_AT,
+    ];
+
+    #[test]
+    fn odm_provenance_suffixes_are_reserved_internal_keys_under_both_prefixes() {
+        for suffix in ODM_PROVENANCE_SUFFIXES {
+            for prefix in [RUSTFS_INTERNAL_PREFIX, MINIO_INTERNAL_PREFIX] {
+                let key = format!("{prefix}{suffix}");
+                assert!(is_internal_key(&key), "{key} must be an internal key");
+                assert!(has_internal_suffix(&key, suffix), "{key} must match its own suffix");
+                assert!(has_internal_suffix(&key.to_uppercase(), suffix), "{key} must match case-insensitively");
+                assert_eq!(strip_internal_prefix(&key).as_deref(), Some(suffix));
+            }
+            assert!(
+                !is_internal_key(&format!("x-amz-meta-{suffix}")),
+                "{suffix} must not leak as user metadata"
+            );
+            assert!(suffix.starts_with("odm-"), "{suffix} must stay in the odm- namespace");
+            assert!(
+                suffix.bytes().all(|b| b.is_ascii_lowercase() || b == b'-'),
+                "{suffix} must be lowercase-hyphenated"
+            );
+        }
+        let distinct: std::collections::HashSet<&str> = ODM_PROVENANCE_SUFFIXES.into_iter().collect();
+        assert_eq!(distinct.len(), ODM_PROVENANCE_SUFFIXES.len(), "provenance suffixes must be distinct");
+    }
+
+    #[test]
+    fn odm_provenance_values_round_trip_under_both_prefixes() {
+        let mut metadata = HashMap::new();
+        insert_str(&mut metadata, SUFFIX_ODM_SOURCE, "s3:legacy-bucket".to_string());
+        insert_str(&mut metadata, SUFFIX_ODM_SOURCE_ETAG, "0123456789abcdef0123456789abcdef-3".to_string());
+        insert_str(&mut metadata, SUFFIX_ODM_SOURCE_LAST_MODIFIED, "2026-01-02T03:04:05Z".to_string());
+        insert_str(&mut metadata, SUFFIX_ODM_SOURCE_VERSION_ID, String::new());
+        insert_str(&mut metadata, SUFFIX_ODM_PULLED_AT, "2026-09-02T00:00:00Z".to_string());
+
+        assert_eq!(
+            metadata.len(),
+            2 * ODM_PROVENANCE_SUFFIXES.len(),
+            "every marker is written under both prefixes"
+        );
+        for suffix in ODM_PROVENANCE_SUFFIXES {
+            assert!(metadata.contains_key(&internal_key_rustfs(suffix)), "missing RustFS key for {suffix}");
+            assert!(
+                metadata.contains_key(&format!("{MINIO_INTERNAL_PREFIX}{suffix}")),
+                "missing MinIO key for {suffix}"
+            );
+            assert!(contains_key_str(&metadata, suffix));
+        }
+        assert_eq!(get_str(&metadata, SUFFIX_ODM_SOURCE).as_deref(), Some("s3:legacy-bucket"));
+        assert_eq!(get_str(&metadata, SUFFIX_ODM_SOURCE_VERSION_ID).as_deref(), Some(""));
+        assert_eq!(
+            get_consistent_str(&metadata, SUFFIX_ODM_SOURCE_ETAG),
+            Some("0123456789abcdef0123456789abcdef-3")
+        );
+
+        // A MinIO-only reader must still find the marker.
+        let minio_only: HashMap<String, String> = metadata
+            .iter()
+            .filter(|(key, _)| starts_with_ignore_ascii_case(key, MINIO_INTERNAL_PREFIX))
+            .map(|(key, value)| (key.clone(), value.clone()))
+            .collect();
+        assert_eq!(get_str(&minio_only, SUFFIX_ODM_PULLED_AT).as_deref(), Some("2026-09-02T00:00:00Z"));
+
+        remove_str(&mut metadata, SUFFIX_ODM_SOURCE);
+        assert!(!contains_key_str(&metadata, SUFFIX_ODM_SOURCE));
+        assert!(contains_key_str(&metadata, SUFFIX_ODM_SOURCE_ETAG));
+    }
+
     #[test]
     fn target_delete_marker_versions_preserve_arn_case_and_report_conflicts() {
         let arn = "arn:rustfs:replication::Target:Bucket";

docs/architecture/compat-cleanup-register.md

@@ -13,6 +13,7 @@
 
 - `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on per-entry and cumulative GNU long-name, GNU long-link, and PAX extension limits; physical-entry, GNU sparse-map, and sparse-continuation limits; cancellation-safe sparse parsing; and fused entry streams after parser errors. The released tokio-tar API does not provide this complete boundary. Keep the reviewed fork pin until astral-sh/tokio-tar#118 is merged and one published tokio-tar release contains every listed capability with the Snowball regression fixtures passing against that release.
 - `backlog-2102` rc.2/rc.3 empty scanner usage floor recovery: old DeleteBucket cleanup could synthesize an empty incomplete v2 usage primary/backup before leadership added an epoch, while newer scanners require a durable authoritative baseline identity. New scanners recognize only that exact serialized empty-fence shape, preserve its epoch through a CAS-protected recovery marker, and rebuild namespace coverage without treating zero usage as authoritative. Remove this recovery path and marker after rc.2 and rc.3 are no longer supported direct-upgrade sources.
+- `backlog-2122` rc.1-rc.3 non-empty scanner usage floor recovery: leadership fencing in those releases can stamp scanner_epoch onto a real bucket-usage snapshot before any scanner cycle completed, leaving a non-empty floor with no scanner_cycle and no authoritative baseline identity. New scanners recognize only this consistent incomplete fenced shape, preserve the epoch through the CAS-protected recovery marker, and rebuild namespace coverage without treating the old usage data as authoritative. Remove this recovery path after rc.1, rc.2, and rc.3 are no longer supported direct-upgrade sources.
 - `s3gate-metadata-xml` persisted bucket XML migration: mixed-version site-replication peers, retained `.metadata.bin` objects, and backup archives can all carry XML written by the s3s codec, so the gateway migration must keep the legacy codec available until every stored form has crossed a verified rewrite boundary. Remove the legacy s3s parser and serializer only after the minimum supported direct-upgrade release reads and writes every persisted XML configuration family through the gateway codec, every supported mixed-version site-replication topology has completed its writer upgrade, and migration tooling has verified or rewritten every retained bucket metadata object and restorable backup archive.
 - `rustfs-6339` legacy bucket policy ID casing: earlier RustFS releases persisted the top-level policy identifier as "ID", while current writes use the S3-compatible "Id" spelling. Readers accept both spellings so retained bucket metadata remains usable after upgrade. Remove the legacy alias after migration tooling has rewritten every retained bucket policy using "ID".
 - `table-publication-fence-v1` table publication fencing: nodes that predate table and table-bucket publication fences can mutate live files while a new node is publishing a catalog pointer. New nodes retain exact object guards until the operator confirms that every serving node uses the new fences. Fleet confirmation also requires non-overlapping active warehouse prefixes and lifecycle workers that exclude table buckets. Remove the exact live-file fallback and the fleet-confirmation gate after the minimum supported RustFS release acquires table fences for registered-table mutations and table-bucket fences for unresolved-prefix mutations.

rustfs/src/app/object/internal_put.rs

@@ -0,0 +1,968 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Internal object write entry points for trusted in-process callers.
+//!
+//! A system write (on-demand migration write-back, future replays) must look
+//! like an ordinary client write: bucket default SSE, quota, versioning,
+//! Object Lock defaults, replication scheduling and creation events all apply.
+//! The single-object entry point runs the same [`DefaultObjectUsecase::put_object_core`]
+//! as the S3 PutObject handler; the multipart entry points mirror the S3
+//! multipart handlers' policy steps against the same storage contract.
+
+use super::*;
+
+use crate::app::object_data_cache::invalidate_object_data_cache_after_complete_multipart_success;
+use crate::app::storage_api::multipart_usecase::contract::multipart::{CompletePart, MultipartOperations as _};
+use crate::app::storage_api::object_usecase::compression::is_multipart_disk_compression_enabled;
+use crate::app::storage_api::object_usecase::io::WriteEncryption;
+use crate::app::storage_api::object_usecase::options::{
+    extract_metadata_from_mime, get_complete_multipart_upload_opts_with_replication_authorization,
+};
+use crate::app::storage_api::object_usecase::sse::{
+    EncryptionKeyKind, PrepareEncryptionRequest, mark_encrypted_multipart_metadata, sse_decryption, sse_prepare_encryption,
+};
+use crate::capacity::record_capacity_write;
+use crate::runtime_sources::NotifyInterface;
+use http::HeaderName;
+
+/// Inputs of an internal object write. Content and user metadata follow the
+/// S3 request shape so the shared write path treats them exactly like a
+/// client PUT: `content_headers` are the standard object headers
+/// (`Content-Type`, `Cache-Control`, `Content-Encoding`, `Content-Disposition`,
+/// `Content-Language`, `Expires`), `user_metadata` carries `x-amz-meta-*`
+/// entries with the prefix stripped, `tags` is the `x-amz-tagging` query
+/// string and `internal_metadata` holds `x-rustfs-internal-*` /
+/// `x-minio-internal-*` keys written verbatim.
+pub(crate) struct InternalPutContext {
+    pub(crate) bucket: String,
+    pub(crate) key: String,
+    /// Plaintext object length. The single-object path requires it, exactly
+    /// like S3 PutObject rejects an unknown `Content-Length`.
+    pub(crate) size: Option<u64>,
+    /// Lowercase hex MD5 the body must hash to; the write fails with
+    /// `BadDigest` otherwise and nothing is committed.
+    pub(crate) expected_md5_hex: Option<String>,
+    /// ETag to store instead of the computed one.
+    pub(crate) preserve_etag: Option<String>,
+    pub(crate) content_headers: HashMap<String, String>,
+    pub(crate) user_metadata: HashMap<String, String>,
+    pub(crate) tags: Option<String>,
+    pub(crate) internal_metadata: HashMap<String, String>,
+    /// Publish the `s3:ObjectCreated:*` event for the write.
+    pub(crate) emit_events: bool,
+    /// `userIdentity.principalId` of the creation event.
+    pub(crate) principal_id: &'static str,
+}
+
+const INTERNAL_PUT_METHOD_NAME: &str = "PUT";
+
+fn api_error_from_s3(err: S3Error) -> ApiError {
+    ApiError {
+        code: err.code().clone(),
+        message: err.message().unwrap_or_default().to_string(),
+        source: None,
+    }
+}
+
+fn not_initialized() -> ApiError {
+    ApiError {
+        code: S3ErrorCode::InternalError,
+        message: "Not init".to_string(),
+        source: None,
+    }
+}
+
+/// Build the header view of an internal write so header-driven policy
+/// (content-type detection, compressibility, standard metadata capture) runs
+/// unchanged.
+fn internal_put_headers(content_headers: &HashMap<String, String>) -> Result<HeaderMap, ApiError> {
+    let mut headers = HeaderMap::with_capacity(content_headers.len());
+    for (name, value) in content_headers {
+        let name = HeaderName::from_bytes(name.as_bytes())
+            .map_err(|err| ApiError::invalid_request(format!("invalid content header name {name:?}: {err}")))?;
+        let value = HeaderValue::from_str(value)
+            .map_err(|err| ApiError::invalid_request(format!("invalid content header value for {name}: {err}")))?;
+        headers.insert(name, value);
+    }
+    Ok(headers)
+}
+
+fn header_string(headers: &HeaderMap, name: &str) -> Option<String> {
+    headers.get(name).and_then(|value| value.to_str().ok()).map(ToOwned::to_owned)
+}
+
+fn internal_put_content_input(headers: &HeaderMap, tags: Option<String>) -> PutObjectContentInput {
+    PutObjectContentInput {
+        cache_control: header_string(headers, "cache-control"),
+        content_disposition: header_string(headers, "content-disposition"),
+        content_encoding: header_string(headers, "content-encoding"),
+        content_language: header_string(headers, "content-language"),
+        content_type: header_string(headers, "content-type"),
+        expires: header_string(headers, "expires"),
+        website_redirect_location: None,
+        tagging: tags,
+        storage_class: None,
+    }
+}
+
+async fn validate_internal_write_target(key: &str, bucket: &str, headers: &HeaderMap) -> Result<(), ApiError> {
+    validate_object_key(key, INTERNAL_PUT_METHOD_NAME).map_err(api_error_from_s3)?;
+    validate_table_catalog_object_mutation(bucket, key)
+        .await
+        .map_err(api_error_from_s3)?;
+    validate_archive_content_encoding(
+        key,
+        headers.get("content-type").and_then(|value| value.to_str().ok()),
+        headers.get("content-encoding").and_then(|value| value.to_str().ok()),
+    )
+    .map_err(api_error_from_s3)
+}
+
+fn internal_events_wanted() -> bool {
+    crate::module_switches::is_notify_module_enabled()
+        || rustfs_notify::notification_system().is_some_and(|system| system.has_live_listeners())
+}
+
+/// Creation event of an internal write, published on successful completion
+/// the way [`OperationHelper`] publishes it for an S3 request.
+pub(super) struct InternalPutObjectEvent {
+    builder: EventArgsBuilder,
+    notify: Arc<dyn NotifyInterface>,
+    request_context: request_context::RequestContext,
+}
+
+impl InternalPutObjectEvent {
+    /// `None` when neither the notify module nor a live listener wants events.
+    pub(super) fn new(
+        notify: Arc<dyn NotifyInterface>,
+        request_context: request_context::RequestContext,
+        event_name: EventName,
+        bucket: &str,
+        key: &str,
+        principal_id: &'static str,
+    ) -> Option<Self> {
+        if !internal_events_wanted() {
+            return None;
+        }
+        Some(Self {
+            builder: Self::builder(event_name, bucket, key, principal_id),
+            notify,
+            request_context,
+        })
+    }
+
+    fn builder(event_name: EventName, bucket: &str, key: &str, principal_id: &'static str) -> EventArgsBuilder {
+        // The object is a placeholder until `object()` supplies the committed
+        // ObjectInfo, matching the S3 helper.
+        let placeholder = ObjectInfo {
+            bucket: bucket.to_string(),
+            name: key.to_string(),
+            ..Default::default()
+        };
+        EventArgsBuilder::new(event_name, bucket.to_string(), convert_ecstore_object_info(placeholder))
+            .req_param("principalId", principal_id)
+    }
+
+    pub(super) fn object(self, obj_info: ObjectInfo) -> Self {
+        let Self {
+            builder,
+            notify,
+            request_context,
+        } = self;
+        Self {
+            builder: builder.object(convert_ecstore_object_info(obj_info)),
+            notify,
+            request_context,
+        }
+    }
+
+    pub(super) fn version_id(self, version_id: String) -> Self {
+        let Self {
+            builder,
+            notify,
+            request_context,
+        } = self;
+        Self {
+            builder: builder.version_id(version_id),
+            notify,
+            request_context,
+        }
+    }
+
+    /// Publish the event when `result` is a success; failures publish nothing.
+    pub(super) fn complete<T>(self, result: &S3Result<S3Response<T>>) {
+        let Ok(response) = result else {
+            return;
+        };
+        let Self {
+            builder,
+            notify,
+            request_context,
+        } = self;
+        let event_args = builder
+            .resp_elements(build_event_resp_elements(response, &request_context.request_id))
+            .build();
+        spawn_background_with_context(Some(request_context), async move {
+            notify.notify(event_args).await;
+        });
+    }
+}
+
+impl DefaultObjectUsecase {
+    /// Write one object through the ordinary PutObject path on behalf of a
+    /// trusted internal caller.
+    ///
+    /// The body is consumed exactly like a client request body: hashed
+    /// against `expected_md5_hex`, compressed and encrypted per bucket policy,
+    /// and committed under quota admission. On any failure nothing is left
+    /// behind.
+    pub(crate) async fn internal_put_object<B>(&self, ctx: InternalPutContext, body: B) -> Result<ObjectInfo, ApiError>
+    where
+        B: Stream<Item = io::Result<Bytes>> + Send + Sync + 'static,
+    {
+        let start_time = Instant::now();
+        let InternalPutContext {
+            bucket,
+            key,
+            size,
+            expected_md5_hex,
+            preserve_etag,
+            content_headers,
+            user_metadata,
+            tags,
+            internal_metadata,
+            emit_events,
+            principal_id,
+        } = ctx;
+        let Some(size) = size else {
+            return Err(ApiError::invalid_request("internal put requires a known object size"));
+        };
+        let size = i64::try_from(size).map_err(|_| ApiError::invalid_request("internal put size exceeds the supported range"))?;
+
+        let headers = internal_put_headers(&content_headers)?;
+        validate_internal_write_target(&key, &bucket, &headers).await?;
+
+        let write = PutObjectWriteRequest {
+            bucket,
+            key,
+            size,
+            quota_operation: QuotaOperation::PutObject,
+            ciphertext_passthrough: false,
+            inbound_replication_put: false,
+            headers: &headers,
+            query: None,
+            trailing_headers: None,
+            version_id: None,
+            sse: PutObjectSseInput {
+                server_side_encryption: None,
+                ssekms_key_id: None,
+                sse_customer_algorithm: None,
+                sse_customer_key: None,
+                sse_customer_key_md5: None,
+            },
+            user_metadata,
+            internal_metadata,
+            content: internal_put_content_input(&headers, tags),
+            object_lock: PutObjectLockInput {
+                legal_hold_status: None,
+                mode: None,
+                retain_until_date: None,
+            },
+            content_md5: expected_md5_hex.map(PutObjectContentMd5::Hex),
+            preserve_etag,
+            origin: PutObjectOrigin::Internal {
+                principal_id,
+                emit_events,
+            },
+        };
+        let committed = self
+            .put_object_core(write, StreamingBlob::wrap(body), start_time)
+            .await
+            .map_err(api_error_from_s3)?;
+
+        let obj_info = committed.obj_info.clone();
+        let result: S3Result<S3Response<()>> = Ok(S3Response::new(()));
+        committed.finish(&result);
+        Ok(obj_info)
+    }
+
+    /// Start a multipart upload for an internal write. The session carries the
+    /// same metadata, bucket default SSE session material, Object Lock
+    /// defaults and replication decision a client-initiated session would.
+    pub(crate) async fn internal_create_multipart_upload(&self, ctx: &InternalPutContext) -> Result<String, ApiError> {
+        let headers = internal_put_headers(&ctx.content_headers)?;
+        validate_internal_write_target(&ctx.key, &ctx.bucket, &headers).await?;
+        let store = self.object_store().ok_or_else(not_initialized)?;
+
+        let mut metadata = ctx.user_metadata.clone();
+        namespace_reserved_user_metadata(&mut metadata);
+        extract_metadata_from_mime(&headers, &mut metadata);
+        if let Some(tags) = ctx.tags.clone() {
+            metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags);
+        }
+
+        let object_lock_config_state = load_bucket_object_lock_config_state(&ctx.bucket)
+            .await
+            .map_err(api_error_from_s3)?;
+        apply_bucket_default_lock_retention(&ctx.bucket, &object_lock_config_state, &mut metadata, false)
+            .map_err(api_error_from_s3)?;
+
+        // Internal callers carry no credential; a bucket default of SSE-KMS is
+        // authorized as an internal write, like every other system write.
+        let prepared_material = sse_prepare_encryption(PrepareEncryptionRequest {
+            bucket: &ctx.bucket,
+            key: &ctx.key,
+            server_side_encryption: None,
+            ssekms_key_id: None,
+            ssekms_context: None,
+            sse_customer_algorithm: None,
+            sse_customer_key: None,
+            sse_customer_key_md5: None,
+            principal: None,
+        })
+        .await?;
+        if let Some(material) = prepared_material {
+            let mut encryption_metadata = encryption_material_to_metadata(&material)?;
+            if material.key_kind == EncryptionKeyKind::Object {
+                mark_encrypted_multipart_metadata(&mut encryption_metadata);
+            }
+            metadata.extend(encryption_metadata);
+        }
+
+        if is_multipart_disk_compression_enabled() && is_disk_compressible(&headers, &ctx.key) {
+            insert_str(
+                &mut metadata,
+                SUFFIX_COMPRESSION,
+                compression_metadata_value(CompressionAlgorithm::default()),
+            );
+        }
+        metadata.extend(ctx.internal_metadata.clone());
+
+        let mt2 = metadata.clone();
+        let mut opts = put_opts_with_replication_authorization(&ctx.bucket, &ctx.key, None, &headers, metadata, false)
+            .await
+            .map_err(ApiError::from)?;
+
+        let dsc = must_replicate_object(
+            &ctx.bucket,
+            &ctx.key,
+            &mt2,
+            "".to_string(),
+            opts.delete_marker_replication_status(),
+            opts.clone(),
+        )
+        .await;
+        if dsc.replicate_any() {
+            insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
+            insert_str(
+                &mut opts.user_defined,
+                SUFFIX_REPLICATION_STATUS,
+                dsc.pending_status().unwrap_or_default(),
+            );
+        }
+
+        let current_opts = get_opts(&ctx.bucket, &ctx.key, opts.version_id.clone(), None, &headers)
+            .await
+            .map_err(ApiError::from)?;
+        match store.get_object_info(&ctx.bucket, &ctx.key, &current_opts).await {
+            Ok(existing_obj_info) => {
+                validate_existing_object_lock_for_write(&object_lock_config_state, &existing_obj_info, &opts)
+                    .map_err(api_error_from_s3)?;
+            }
+            Err(err) => {
+                if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
+                    return Err(ApiError::from(err));
+                }
+            }
+        }
+
+        let upload = store
+            .new_multipart_upload(&ctx.bucket, &ctx.key, &opts)
+            .await
+            .map_err(ApiError::from)?;
+        Ok(upload.upload_id)
+    }
+
+    /// Stage one part of an internal multipart upload. Compression and managed
+    /// SSE follow the session metadata recorded at creation; the staged part
+    /// is verified against `expected_md5_hex` when given.
+    pub(crate) async fn internal_upload_part<B>(
+        &self,
+        ctx: &InternalPutContext,
+        upload_id: &str,
+        part_number: usize,
+        size: u64,
+        expected_md5_hex: Option<String>,
+        body: B,
+    ) -> Result<CompletePart, ApiError>
+    where
+        B: Stream<Item = io::Result<Bytes>> + Send + Sync + Unpin + 'static,
+    {
+        let size =
+            i64::try_from(size).map_err(|_| ApiError::invalid_request("internal part size exceeds the supported range"))?;
+        let bucket = ctx.bucket.as_str();
+        let key = ctx.key.as_str();
+        let store = self.object_store().ok_or_else(not_initialized)?;
+        let mut opts = ObjectOptions::default();
+        let session = store
+            .get_multipart_info(bucket, key, upload_id, &opts)
+            .await
+            .map_err(ApiError::from)?;
+
+        let upload_part_admission = match get_concurrency_manager()
+            .admit_multipart_part(size)
+            .await
+            .map_err(|_| ApiError::other(io::Error::other("foreground write admission closed")))?
+        {
+            ForegroundWriteAdmission::Disabled => None,
+            ForegroundWriteAdmission::Admitted(permit) => {
+                counter!("rustfs.upload_part.foreground_admission.total", "result" => "admitted").increment(1);
+                Some(permit)
+            }
+            ForegroundWriteAdmission::Rejected => {
+                counter!("rustfs.upload_part.foreground_admission.total", "result" => "rejected").increment(1);
+                return Err(ApiError {
+                    code: S3ErrorCode::SlowDown,
+                    message: "foreground write concurrency limit reached, please reduce your request rate".to_string(),
+                    source: None,
+                });
+            }
+        };
+
+        let buffer_size = get_buffer_size_opt_in(size);
+        let body = tokio::io::BufReader::with_capacity(buffer_size, StreamReader::new(body));
+        let actual_size = size;
+        let mut write_plan = WritePlan::new();
+        let mut reader = if rustfs_utils::http::contains_key_str(&session.user_defined, SUFFIX_COMPRESSION) {
+            let hrd = HashReader::from_stream(body, size, actual_size, expected_md5_hex, None, false).map_err(ApiError::from)?;
+            write_plan = write_plan.with_compression(CompressionAlgorithm::default());
+            hrd
+        } else {
+            HashReader::from_stream(body, size, actual_size, expected_md5_hex, None, false).map_err(ApiError::from)?
+        };
+        opts.want_checksum = reader.checksum();
+
+        if session
+            .user_defined
+            .contains_key("x-amz-server-side-encryption-customer-algorithm")
+        {
+            return Err(ApiError::invalid_request("internal multipart writes cannot continue an SSE-C session"));
+        }
+        if session.user_defined.contains_key("x-amz-server-side-encryption") {
+            // Reuses the envelope prepared at creation; the session pins the key.
+            let managed_material = sse_decryption(DecryptionRequest {
+                bucket,
+                key,
+                metadata: &session.user_defined,
+                sse_customer_key: None,
+                sse_customer_key_md5: None,
+                principal: None,
+            })
+            .await?
+            .ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
+            let managed_write = match managed_material.key_kind {
+                EncryptionKeyKind::Object => {
+                    WriteEncryption::multipart_object_key(managed_material.key_bytes, part_number as u32)
+                }
+                EncryptionKeyKind::Direct => {
+                    WriteEncryption::multipart(managed_material.key_bytes, managed_material.base_nonce, part_number)
+                }
+            };
+            write_plan = write_plan.with_encryption(managed_write);
+        }
+
+        reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?;
+        let mut reader = PutObjReader::new(reader);
+
+        let _upload_part_admission = upload_part_admission;
+        let info = store
+            .put_object_part(bucket, key, upload_id, part_number, &mut reader, &opts)
+            .await
+            .map_err(ApiError::from)?;
+        drop(_upload_part_admission);
+
+        Ok(CompletePart {
+            part_num: info.part_num,
+            etag: info.etag,
+            ..Default::default()
+        })
+    }
+
+    /// Complete an internal multipart upload: versioning, quota admission,
+    /// Object Lock validation of the overwritten version, usage accounting,
+    /// immediate ILM transition, replication scheduling and the creation
+    /// event, as the S3 CompleteMultipartUpload handler performs them.
+    pub(crate) async fn internal_complete_multipart_upload(
+        &self,
+        ctx: &InternalPutContext,
+        upload_id: &str,
+        parts: Vec<CompletePart>,
+    ) -> Result<ObjectInfo, ApiError> {
+        let bucket = ctx.bucket.clone();
+        let key = ctx.key.clone();
+        if parts.is_empty() {
+            return Err(ApiError::invalid_request("You must specify at least one part"));
+        }
+        if parts.windows(2).any(|pair| pair[0].part_num >= pair[1].part_num) {
+            return Err(ApiError::invalid_request("multipart parts must be listed in ascending part number order"));
+        }
+        validate_table_catalog_object_mutation(&bucket, &key)
+            .await
+            .map_err(api_error_from_s3)?;
+        let store = self.object_store().ok_or_else(not_initialized)?;
+
+        let headers = HeaderMap::new();
+        let mut opts =
+            get_complete_multipart_upload_opts_with_replication_authorization(&headers, false).map_err(ApiError::from)?;
+        opts.preserve_etag = ctx.preserve_etag.clone();
+        let versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
+        opts.versioned = versioned;
+        opts.version_suspended = BucketVersioningSys::prefix_suspended(&bucket, &key).await;
+        let capacity_scope_token = Uuid::new_v4();
+        opts.capacity_scope_token = Some(capacity_scope_token);
+
+        let current_opts =
+            internal_object_info_lookup_opts(get_opts(&bucket, &key, None, None, &headers).await.map_err(ApiError::from)?);
+        let object_lock_config_state = load_bucket_object_lock_config_state(&bucket)
+            .await
+            .map_err(api_error_from_s3)?;
+        let previous_current_sizes = match store.get_object_info(&bucket, &key, &current_opts).await {
+            Ok(existing_obj_info) => {
+                validate_existing_object_lock_for_write(&object_lock_config_state, &existing_obj_info, &current_opts)
+                    .map_err(api_error_from_s3)?;
+                let physical_size = existing_obj_info.size.max(0) as u64;
+                let logical_size = quota_object_size(&existing_obj_info);
+                Some((physical_size, logical_size))
+            }
+            Err(err) => {
+                if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
+                    return Err(ApiError::from(err));
+                }
+                None
+            }
+        };
+
+        let cache_adapter = self.object_data_cache();
+        let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await;
+
+        let quota_metadata_sys = self.bucket_metadata_sys();
+        let quota_tracking = quota_metadata_sys.is_some();
+        let mut quota_enabled = false;
+        if let Some(metadata_sys) = quota_metadata_sys {
+            let quota_checker = QuotaChecker::new(metadata_sys);
+            let check_result =
+                map_quota_check_outcome(&bucket, quota_checker.check_quota(&bucket, QuotaOperation::PutObject, 0).await)
+                    .map_err(api_error_from_s3)?;
+            quota_enabled = check_result.quota_limit.is_some();
+            apply_quota_admission(&mut opts, &check_result).map_err(api_error_from_s3)?;
+        }
+
+        let previous_current_size = match previous_current_sizes {
+            Some((_, Ok(logical_size))) if quota_enabled => Some(logical_size),
+            Some((_, Err(err))) if quota_enabled => return Err(ApiError::from(err)),
+            Some((physical_size, _)) => Some(physical_size),
+            None => None,
+        };
+
+        let event = ctx.emit_events.then(|| {
+            InternalPutObjectEvent::new(
+                current_notify_interface_for_context(self.context.as_deref()),
+                request_context::RequestContext::fallback(),
+                EventName::ObjectCreatedCompleteMultipartUpload,
+                &bucket,
+                &key,
+                ctx.principal_id,
+            )
+        });
+
+        // The spawned task owns the commit so a cancelled caller cannot leave
+        // the bookkeeping half done.
+        let complete_commit = spawn_traced_join({
+            let store = Arc::clone(&store);
+            let bucket = bucket.clone();
+            let key = key.clone();
+            let upload_id = upload_id.to_string();
+            let opts = opts.clone();
+            async move {
+                let obj_info = store
+                    .clone()
+                    .complete_multipart_upload(&bucket, &key, &upload_id, parts, &opts)
+                    .await
+                    .map_err(ApiError::from)?;
+                let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await;
+                record_capacity_write(Some(capacity_scope_token)).await;
+
+                if quota_tracking {
+                    let committed_size = quota_accounting_object_size(&obj_info, quota_enabled).map_err(api_error_from_s3)?;
+                    if versioned {
+                        record_bucket_object_version_write_memory(&bucket, previous_current_size, committed_size).await;
+                    } else {
+                        record_bucket_object_write_memory(&bucket, previous_current_size, committed_size).await;
+                    }
+                }
+
+                enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await;
+
+                let mt2 = obj_info.user_defined.clone();
+                let dsc = must_replicate_object(
+                    &bucket,
+                    &key,
+                    &mt2,
+                    "".to_string(),
+                    opts.delete_marker_replication_status(),
+                    opts.clone(),
+                )
+                .await;
+                if dsc.replicate_any() {
+                    schedule_object_replication(obj_info.clone(), store, dsc).await;
+                }
+
+                rustfs_scanner::record_dirty_usage_bucket(&bucket);
+                Ok::<_, ApiError>(obj_info)
+            }
+        });
+        let obj_info = complete_commit.await.map_err(|err| {
+            ApiError::other(io::Error::other(format!("complete multipart upload commit owner task failed: {err}")))
+        })??;
+
+        if let Some(event) = event.flatten() {
+            let mut event = event.object(obj_info.clone());
+            if versioned && let Some(version_id) = obj_info.version_id {
+                event = event.version_id(version_id.to_string());
+            }
+            let result: S3Result<S3Response<()>> = Ok(S3Response::new(()));
+            event.complete(&result);
+        }
+        Ok(obj_info)
+    }
+
+    /// Discard an internal multipart upload and its staged parts.
+    pub(crate) async fn internal_abort_multipart_upload(&self, bucket: &str, key: &str, upload_id: &str) -> Result<(), ApiError> {
+        let store = self.object_store().ok_or_else(not_initialized)?;
+        store
+            .abort_multipart_upload(bucket, key, upload_id, &ObjectOptions::default())
+            .await
+            .map_err(ApiError::from)?;
+        rustfs_scanner::record_dirty_usage_bucket(bucket);
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
+    use rustfs_utils::http::{
+        MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, SUFFIX_ODM_PULLED_AT, SUFFIX_ODM_SOURCE, SUFFIX_ODM_SOURCE_ETAG,
+        SUFFIX_ODM_SOURCE_LAST_MODIFIED, SUFFIX_ODM_SOURCE_VERSION_ID, contains_key_str, get_str,
+    };
+
+    const TEST_PRINCIPAL: &str = "rustfs-internal-put-test";
+
+    fn md5_hex(body: &[u8]) -> String {
+        hex_simd::encode_to_string(Md5::digest(body), hex_simd::AsciiCase::Lower)
+    }
+
+    fn body_stream(chunks: Vec<Bytes>) -> impl Stream<Item = io::Result<Bytes>> + Send + Sync + Unpin + 'static {
+        futures::stream::iter(chunks.into_iter().map(Ok))
+    }
+
+    fn provenance_metadata() -> HashMap<String, String> {
+        let mut internal_metadata = HashMap::new();
+        insert_str(&mut internal_metadata, SUFFIX_ODM_SOURCE, "s3:source-bucket".to_string());
+        insert_str(
+            &mut internal_metadata,
+            SUFFIX_ODM_SOURCE_ETAG,
+            "\"0123456789abcdef0123456789abcdef-3\"".to_string(),
+        );
+        insert_str(
+            &mut internal_metadata,
+            SUFFIX_ODM_SOURCE_LAST_MODIFIED,
+            "2026-01-02T03:04:05Z".to_string(),
+        );
+        insert_str(&mut internal_metadata, SUFFIX_ODM_SOURCE_VERSION_ID, String::new());
+        insert_str(&mut internal_metadata, SUFFIX_ODM_PULLED_AT, "2026-09-02T00:00:00Z".to_string());
+        internal_metadata
+    }
+
+    fn internal_context(bucket: &str, key: &str, body: &[u8]) -> InternalPutContext {
+        InternalPutContext {
+            bucket: bucket.to_string(),
+            key: key.to_string(),
+            size: Some(body.len() as u64),
+            expected_md5_hex: Some(md5_hex(body)),
+            preserve_etag: None,
+            content_headers: HashMap::from([
+                ("Content-Type".to_string(), "text/plain".to_string()),
+                ("Cache-Control".to_string(), "max-age=60".to_string()),
+            ]),
+            user_metadata: HashMap::from([("origin".to_string(), "unit-test".to_string())]),
+            tags: Some("team=storage".to_string()),
+            internal_metadata: provenance_metadata(),
+            emit_events: false,
+            principal_id: TEST_PRINCIPAL,
+        }
+    }
+
+    async fn internal_put_test_bucket(prefix: &str) -> (Arc<ECStore>, String) {
+        let store = crate::app::gating_test_env::shared_gating_ecstore().await;
+        crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
+        let bucket = format!("{prefix}-{}", Uuid::new_v4().simple());
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create internal put test bucket");
+        (store, bucket)
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn internal_put_object_writes_through_the_shared_put_path() {
+        let (store, bucket) = internal_put_test_bucket("internal-put").await;
+        let body = b"internal write-back body".to_vec();
+        let ctx = internal_context(&bucket, "dir/object.txt", &body);
+
+        let obj_info = DefaultObjectUsecase::from_global()
+            .internal_put_object(ctx, body_stream(vec![Bytes::from(body.clone())]))
+            .await
+            .expect("internal put must succeed");
+
+        assert_eq!(obj_info.etag.as_deref(), Some(md5_hex(&body).as_str()));
+        assert_eq!(obj_info.size, body.len() as i64);
+
+        let stored = store
+            .get_object_info(&bucket, "dir/object.txt", &ObjectOptions::default())
+            .await
+            .expect("internal put must leave a readable object");
+        let metadata = &stored.user_defined;
+        assert_eq!(metadata.get("content-type").map(String::as_str), Some("text/plain"));
+        assert_eq!(metadata.get("cache-control").map(String::as_str), Some("max-age=60"));
+        assert_eq!(metadata.get("origin").map(String::as_str), Some("unit-test"));
+        assert_eq!(stored.user_tags.as_str(), "team=storage", "tags must be committed as object tags");
+        for suffix in [
+            SUFFIX_ODM_SOURCE,
+            SUFFIX_ODM_SOURCE_ETAG,
+            SUFFIX_ODM_SOURCE_LAST_MODIFIED,
+            SUFFIX_ODM_SOURCE_VERSION_ID,
+            SUFFIX_ODM_PULLED_AT,
+        ] {
+            assert!(
+                metadata.contains_key(&format!("{RUSTFS_INTERNAL_PREFIX}{suffix}")),
+                "missing RustFS provenance key {suffix}"
+            );
+            assert!(
+                metadata.contains_key(&format!("{MINIO_INTERNAL_PREFIX}{suffix}")),
+                "missing MinIO provenance key {suffix}"
+            );
+        }
+        assert_eq!(get_str(metadata, SUFFIX_ODM_SOURCE).as_deref(), Some("s3:source-bucket"));
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn internal_put_object_preserves_the_caller_etag() {
+        let (store, bucket) = internal_put_test_bucket("internal-put-etag").await;
+        let body = b"etag is preserved verbatim".to_vec();
+        let mut ctx = internal_context(&bucket, "preserved.bin", &body);
+        ctx.preserve_etag = Some("0123456789abcdef0123456789abcdef-3".to_string());
+
+        let obj_info = DefaultObjectUsecase::from_global()
+            .internal_put_object(ctx, body_stream(vec![Bytes::from(body.clone())]))
+            .await
+            .expect("internal put with a preserved ETag must succeed");
+        assert_eq!(obj_info.etag.as_deref(), Some("0123456789abcdef0123456789abcdef-3"));
+
+        let stored = store
+            .get_object_info(&bucket, "preserved.bin", &ObjectOptions::default())
+            .await
+            .expect("preserved-ETag object must be readable");
+        assert_eq!(stored.etag.as_deref(), Some("0123456789abcdef0123456789abcdef-3"));
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn internal_put_object_rejects_a_digest_mismatch_without_committing() {
+        let (store, bucket) = internal_put_test_bucket("internal-put-digest").await;
+        let body = b"body whose digest will not match".to_vec();
+        let mut ctx = internal_context(&bucket, "mismatch.bin", &body);
+        ctx.expected_md5_hex = Some(md5_hex(b"a different body"));
+
+        let err = DefaultObjectUsecase::from_global()
+            .internal_put_object(ctx, body_stream(vec![Bytes::from(body)]))
+            .await
+            .expect_err("digest mismatch must fail the internal put");
+        assert_eq!(err.code, S3ErrorCode::BadDigest, "unexpected error: {err}");
+
+        let lookup = store
+            .get_object_info(&bucket, "mismatch.bin", &ObjectOptions::default())
+            .await;
+        assert!(
+            lookup.as_ref().is_err_and(is_err_object_not_found),
+            "a rejected internal put must not leave an object behind"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn internal_put_object_requires_a_known_size() {
+        let (_store, bucket) = internal_put_test_bucket("internal-put-size").await;
+        let body = b"unknown length".to_vec();
+        let mut ctx = internal_context(&bucket, "unknown.bin", &body);
+        ctx.size = None;
+
+        let err = DefaultObjectUsecase::from_global()
+            .internal_put_object(ctx, body_stream(vec![Bytes::from(body)]))
+            .await
+            .expect_err("an unknown size must be rejected before the body is read");
+        assert_eq!(err.code, S3ErrorCode::InvalidRequest);
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn internal_multipart_roundtrip_completes_and_abort_leaves_nothing() {
+        const FIRST_PART_SIZE: usize = 5 * 1024 * 1024;
+        let (store, bucket) = internal_put_test_bucket("internal-mpu").await;
+        let usecase = DefaultObjectUsecase::from_global();
+        let first_part = vec![0x41u8; FIRST_PART_SIZE];
+        let last_part = b"tail of the multipart object".to_vec();
+        let mut ctx = internal_context(&bucket, "multipart/object.bin", &[]);
+        ctx.size = None;
+        ctx.expected_md5_hex = None;
+        ctx.preserve_etag = Some("0123456789abcdef0123456789abcdef-2".to_string());
+
+        let upload_id = usecase
+            .internal_create_multipart_upload(&ctx)
+            .await
+            .expect("internal multipart create must succeed");
+        let part_one = usecase
+            .internal_upload_part(
+                &ctx,
+                &upload_id,
+                1,
+                first_part.len() as u64,
+                Some(md5_hex(&first_part)),
+                body_stream(vec![Bytes::from(first_part.clone())]),
+            )
+            .await
+            .expect("first internal part must stage");
+        let part_two = usecase
+            .internal_upload_part(
+                &ctx,
+                &upload_id,
+                2,
+                last_part.len() as u64,
+                Some(md5_hex(&last_part)),
+                body_stream(vec![Bytes::from(last_part.clone())]),
+            )
+            .await
+            .expect("last internal part must stage");
+        assert_eq!(part_one.part_num, 1);
+        assert_eq!(part_two.part_num, 2);
+
+        let obj_info = usecase
+            .internal_complete_multipart_upload(&ctx, &upload_id, vec![part_one, part_two])
+            .await
+            .expect("internal multipart complete must succeed");
+        assert_eq!(obj_info.size, (first_part.len() + last_part.len()) as i64);
+        assert_eq!(obj_info.parts.len(), 2);
+        assert_eq!(obj_info.etag.as_deref(), Some("0123456789abcdef0123456789abcdef-2"));
+        let stored = store
+            .get_object_info(&bucket, &ctx.key, &ObjectOptions::default())
+            .await
+            .expect("completed multipart object must be readable");
+        assert_eq!(stored.user_defined.get("content-type").map(String::as_str), Some("text/plain"));
+        assert_eq!(stored.user_defined.get("origin").map(String::as_str), Some("unit-test"));
+        assert!(contains_key_str(&stored.user_defined, SUFFIX_ODM_SOURCE));
+
+        let aborted_upload_id = usecase
+            .internal_create_multipart_upload(&ctx)
+            .await
+            .expect("second internal multipart create must succeed");
+        usecase
+            .internal_upload_part(
+                &ctx,
+                &aborted_upload_id,
+                1,
+                last_part.len() as u64,
+                None,
+                body_stream(vec![Bytes::from(last_part.clone())]),
+            )
+            .await
+            .expect("part of the aborted upload must stage");
+        usecase
+            .internal_abort_multipart_upload(&bucket, &ctx.key, &aborted_upload_id)
+            .await
+            .expect("internal abort must succeed");
+        let uploads = store
+            .list_multipart_uploads(&bucket, &ctx.key, None, None, None, 100)
+            .await
+            .expect("list multipart uploads after abort");
+        assert!(
+            uploads.uploads.iter().all(|upload| upload.upload_id != aborted_upload_id),
+            "aborted internal upload must not linger: {:?}",
+            uploads.uploads
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn internal_complete_multipart_upload_rejects_unordered_parts() {
+        let (_store, bucket) = internal_put_test_bucket("internal-mpu-order").await;
+        let ctx = internal_context(&bucket, "unordered.bin", &[]);
+        let parts = vec![
+            CompletePart {
+                part_num: 2,
+                ..Default::default()
+            },
+            CompletePart {
+                part_num: 1,
+                ..Default::default()
+            },
+        ];
+        let err = DefaultObjectUsecase::from_global()
+            .internal_complete_multipart_upload(&ctx, "upload", parts)
+            .await
+            .expect_err("unordered parts must be rejected before touching the store");
+        assert_eq!(err.code, S3ErrorCode::InvalidRequest);
+    }
+
+    #[test]
+    fn internal_put_event_names_the_principal_and_creation_event() {
+        let event_args = InternalPutObjectEvent::builder(EventName::ObjectCreatedPut, "bucket", "key", TEST_PRINCIPAL).build();
+        assert_eq!(event_args.event_name, EventName::ObjectCreatedPut);
+        assert_eq!(event_args.bucket_name, "bucket");
+        assert_eq!(event_args.req_params.get("principalId").map(String::as_str), Some(TEST_PRINCIPAL));
+        assert!(!event_args.is_replication_request());
+    }
+
+    #[test]
+    fn internal_put_headers_normalize_names_and_reject_invalid_values() {
+        let headers = internal_put_headers(&HashMap::from([
+            ("Content-Type".to_string(), "application/json".to_string()),
+            ("Cache-Control".to_string(), "no-cache".to_string()),
+        ]))
+        .expect("valid content headers must build");
+        assert_eq!(headers.get("content-type").and_then(|v| v.to_str().ok()), Some("application/json"));
+        let content = internal_put_content_input(&headers, Some("a=b".to_string()));
+        assert_eq!(content.content_type.as_deref(), Some("application/json"));
+        assert_eq!(content.cache_control.as_deref(), Some("no-cache"));
+        assert_eq!(content.tagging.as_deref(), Some("a=b"));
+        assert!(content.storage_class.is_none());
+
+        let err = internal_put_headers(&HashMap::from([("Content-Type".to_string(), "bad\nvalue".to_string())]))
+            .expect_err("a header value with a control character must be rejected");
+        assert_eq!(err.code, S3ErrorCode::InvalidRequest);
+    }
+}

rustfs/src/app/object/mod.rs

@@ -191,6 +191,10 @@ mod delete;
 mod extract;
 mod get;
 mod head;
+// Consumed by the on-demand migration write-back (rustfs/backlog#2153); until
+// that lands only tests construct the internal entry points.
+#[cfg_attr(not(test), expect(dead_code, reason = "wired by the on-demand migration write-back"))]
+mod internal_put;
 mod put;
 mod restore;
 mod shared;
@@ -202,6 +206,7 @@ pub(crate) use self::copy::*;
 pub(crate) use self::delete::*;
 pub(crate) use self::extract::*;
 pub(crate) use self::get::*;
+pub(crate) use self::internal_put::*;
 use self::put::*;
 pub(crate) use self::put::{guard_put_object_body_read_timeout, put_object_body_read_timeout};
 pub(crate) use self::shared::*;

rustfs/src/app/object/put.rs

@@ -895,6 +895,226 @@ fn is_post_object_sse_kms_requested(input: &PutObjectInput, headers: &HeaderMap)
     is_sse_kms_requested(input, headers)
 }
 
+/// Standard content headers and the tagging / storage-class values of a PUT
+/// that become object metadata through [`apply_put_request_metadata`].
+pub(super) struct PutObjectContentInput {
+    pub(super) cache_control: Option<CacheControl>,
+    pub(super) content_disposition: Option<ContentDisposition>,
+    pub(super) content_encoding: Option<ContentEncoding>,
+    pub(super) content_language: Option<ContentLanguage>,
+    pub(super) content_type: Option<ContentType>,
+    pub(super) expires: Option<String>,
+    pub(super) website_redirect_location: Option<WebsiteRedirectLocation>,
+    pub(super) tagging: Option<TaggingHeader>,
+    pub(super) storage_class: Option<StorageClass>,
+}
+
+/// Encryption inputs of a PUT after the S3 input/header merge.
+pub(super) struct PutObjectSseInput {
+    pub(super) server_side_encryption: Option<ServerSideEncryption>,
+    pub(super) ssekms_key_id: Option<SSEKMSKeyId>,
+    pub(super) sse_customer_algorithm: Option<SSECustomerAlgorithm>,
+    pub(super) sse_customer_key: Option<s3s::dto::SSECustomerKey>,
+    pub(super) sse_customer_key_md5: Option<SSECustomerKeyMD5>,
+}
+
+/// Explicit Object Lock values of a PUT; all `None` means the bucket default
+/// retention decides.
+pub(super) struct PutObjectLockInput {
+    pub(super) legal_hold_status: Option<ObjectLockLegalHoldStatus>,
+    pub(super) mode: Option<ObjectLockMode>,
+    pub(super) retain_until_date: Option<Timestamp>,
+}
+
+/// Expected body MD5 as the caller carries it. It is decoded at the same
+/// point of the write path for every origin so an invalid digest keeps its
+/// precedence relative to quota, admission and Object Lock errors.
+pub(super) enum PutObjectContentMd5 {
+    /// `Content-MD5` request header value.
+    Base64(String),
+    /// Lowercase hex digest, as an internal caller already holds it.
+    #[cfg_attr(not(test), expect(dead_code, reason = "constructed by the internal put entry point"))]
+    Hex(String),
+}
+
+/// Where a single-object write originates.
+pub(super) enum PutObjectOrigin<'a> {
+    /// The S3 PutObject/PostObject handler: request-bound identity, the
+    /// audit/notification chain and the bucket-generation guard installed by
+    /// the access layer all come from the request.
+    S3 {
+        req: &'a S3Request<PutObjectInput>,
+        event_name: EventName,
+    },
+    /// A trusted in-process caller writing on the server's behalf. There is no
+    /// request and no credential: managed-SSE authorization treats the write
+    /// as internal, and the creation event, when requested, names
+    /// `principal_id` instead of an access key.
+    #[cfg_attr(not(test), expect(dead_code, reason = "constructed by the internal put entry point"))]
+    Internal { principal_id: &'static str, emit_events: bool },
+}
+
+impl PutObjectOrigin<'_> {
+    fn replication_request_authorized(&self) -> bool {
+        match self {
+            Self::S3 { req, .. } => replication_request_authorized(req),
+            Self::Internal { .. } => false,
+        }
+    }
+
+    fn apply_bucket_generation_guard(&self, bucket: &str, opts: &mut ObjectOptions) -> S3Result<()> {
+        match self {
+            Self::S3 { req, .. } => apply_bucket_generation_guard(req, bucket, opts),
+            Self::Internal { .. } => Ok(()),
+        }
+    }
+
+    fn sse_principal(&self) -> Option<SseKmsPrincipal> {
+        match self {
+            Self::S3 { req, .. } => SseKmsPrincipal::from_request(req),
+            Self::Internal { .. } => None,
+        }
+    }
+}
+
+/// Every input the shared single-object write path needs, independent of
+/// whether an S3 request or an internal caller produced it.
+pub(super) struct PutObjectWriteRequest<'a> {
+    pub(super) bucket: String,
+    pub(super) key: String,
+    /// Authoritative plaintext length; never negative.
+    pub(super) size: i64,
+    pub(super) quota_operation: QuotaOperation,
+    /// Authorized SSE-C replication body that is already ciphertext.
+    pub(super) ciphertext_passthrough: bool,
+    pub(super) inbound_replication_put: bool,
+    /// Request headers, or the object's content headers for an internal write.
+    pub(super) headers: &'a HeaderMap,
+    pub(super) query: Option<&'a str>,
+    pub(super) trailing_headers: Option<s3s::TrailingHeaders>,
+    pub(super) version_id: Option<String>,
+    pub(super) sse: PutObjectSseInput,
+    /// User metadata keyed the way s3s delivers it (`x-amz-meta-` stripped).
+    pub(super) user_metadata: HashMap<String, String>,
+    /// Internal `x-rustfs-internal-*` / `x-minio-internal-*` keys written
+    /// verbatim onto the object; empty for S3 requests.
+    pub(super) internal_metadata: HashMap<String, String>,
+    pub(super) content: PutObjectContentInput,
+    pub(super) object_lock: PutObjectLockInput,
+    pub(super) content_md5: Option<PutObjectContentMd5>,
+    /// ETag to store instead of the computed one; `None` keeps the computed
+    /// (or replication-header-derived) value.
+    pub(super) preserve_etag: Option<String>,
+    pub(super) origin: PutObjectOrigin<'a>,
+}
+
+/// Audit/notification completion of a write, per origin.
+pub(super) enum PutObjectCompletion {
+    S3(OperationHelper),
+    Internal(Option<Box<InternalPutObjectEvent>>),
+}
+
+impl PutObjectCompletion {
+    fn object(self, obj_info: ObjectInfo) -> Self {
+        match self {
+            Self::S3(helper) => Self::S3(helper.object(obj_info)),
+            Self::Internal(event) => Self::Internal(event.map(|event| Box::new(event.object(obj_info)))),
+        }
+    }
+
+    fn version_id(self, version_id: String) -> Self {
+        match self {
+            Self::S3(helper) => Self::S3(helper.version_id(version_id)),
+            Self::Internal(event) => Self::Internal(event.map(|event| Box::new(event.version_id(version_id)))),
+        }
+    }
+
+    fn complete<T>(self, result: &S3Result<S3Response<T>>) -> Self {
+        match self {
+            Self::S3(helper) => Self::S3(helper.complete(result)),
+            Self::Internal(event) => {
+                if let Some(event) = event {
+                    event.complete(result);
+                }
+                Self::Internal(None)
+            }
+        }
+    }
+}
+
+/// Record the failed completion and hand the error back to the caller.
+fn fail_put_object(completion: PutObjectCompletion, err: S3Error) -> S3Error {
+    let result: S3Result<S3Response<()>> = Err(err);
+    let _ = completion.complete(&result);
+    match result {
+        Err(err) => err,
+        Ok(_) => unreachable!("failed PutObject completion carries an error"),
+    }
+}
+
+/// A committed single-object write awaiting its response-side completion.
+pub(super) struct PutObjectCommitted {
+    pub(super) obj_info: ObjectInfo,
+    pub(super) put_versioned: bool,
+    pub(super) effective_sse: Option<ServerSideEncryption>,
+    pub(super) effective_kms_key_id: Option<SSEKMSKeyId>,
+    pub(super) sse_customer_algorithm: Option<SSECustomerAlgorithm>,
+    pub(super) sse_customer_key_md5: Option<SSECustomerKeyMD5>,
+    pub(super) put_extra_checksum_headers: Vec<(&'static str, String)>,
+    completion: PutObjectCompletion,
+    put_request_guard: PutObjectGuard,
+    bucket: String,
+    key: String,
+    start_time: Instant,
+    size: i64,
+    use_zero_copy_eager_put_path: bool,
+    concurrent_put_requests: usize,
+    buffer_size: usize,
+}
+
+impl PutObjectCommitted {
+    /// Publish the audit entry / creation event for `result`, record the
+    /// request-level PutObject metrics and release the request guard.
+    pub(super) fn finish<T>(self, result: &S3Result<S3Response<T>>) {
+        let Self {
+            completion,
+            mut put_request_guard,
+            bucket,
+            key,
+            start_time,
+            size,
+            use_zero_copy_eager_put_path,
+            concurrent_put_requests,
+            buffer_size,
+            ..
+        } = self;
+        let _ = completion.complete(result);
+
+        // Record PutObject metrics via zero-copy-metrics
+        {
+            let duration_ms = start_time.elapsed().as_millis() as f64;
+            rustfs_io_metrics::record_put_object(
+                duration_ms,
+                size,
+                use_zero_copy_eager_put_path, // Track if zero-copy was enabled
+            );
+        }
+
+        debug!(
+            target: "rustfs::app::object_usecase",
+            component = "app",
+            subsystem = "object",
+            bucket = %bucket,
+            key = %key,
+            concurrent_put_requests,
+            buffer_size,
+            "PutObject request completed"
+        );
+
+        put_request_guard.finish_ok();
+    }
+}
+
 impl DefaultObjectUsecase {
     fn should_use_large_put_concurrency_tuning(size: i64) -> bool {
         size >= DEFAULT_PUT_LARGE_CONCURRENCY_TUNING_MIN_SIZE_BYTES
@@ -1055,7 +1275,7 @@ impl DefaultObjectUsecase {
         };
 
         // Resolve the authoritative decoded/plain object length (rejecting negative/unknown) before anything else consumes it.
-        let mut size = resolve_put_object_authoritative_size(&req.headers, content_length)?;
+        let size = resolve_put_object_authoritative_size(&req.headers, content_length)?;
 
         if let Some(limit) = max_content_length
             && u64::try_from(size).is_ok_and(|size| size > limit)
@@ -1063,6 +1283,140 @@ impl DefaultObjectUsecase {
             return Err(S3Error::new(S3ErrorCode::EntityTooLarge));
         }
 
+        let write = PutObjectWriteRequest {
+            bucket: bucket.clone(),
+            key,
+            size,
+            quota_operation,
+            ciphertext_passthrough,
+            inbound_replication_put,
+            headers: &req.headers,
+            query: req.uri.query(),
+            trailing_headers: req.trailing_headers.clone(),
+            version_id,
+            sse: PutObjectSseInput {
+                server_side_encryption,
+                ssekms_key_id,
+                sse_customer_algorithm,
+                sse_customer_key,
+                sse_customer_key_md5,
+            },
+            user_metadata: metadata.unwrap_or_default(),
+            internal_metadata: HashMap::new(),
+            content: PutObjectContentInput {
+                cache_control,
+                content_disposition,
+                content_encoding,
+                content_language,
+                content_type,
+                expires,
+                website_redirect_location,
+                tagging,
+                storage_class,
+            },
+            object_lock: PutObjectLockInput {
+                legal_hold_status: object_lock_legal_hold_status,
+                mode: object_lock_mode,
+                retain_until_date: object_lock_retain_until_date,
+            },
+            content_md5: content_md5.map(PutObjectContentMd5::Base64),
+            preserve_etag: None,
+            origin: PutObjectOrigin::S3 { req: &req, event_name },
+        };
+        let committed = self.put_object_core(write, body, start_time).await?;
+
+        let raw_version = committed.obj_info.version_id.map(|v| v.to_string());
+        let put_version = if committed.put_versioned { raw_version } else { None };
+
+        let e_tag = committed.obj_info.etag.clone().map(|etag| to_s3s_etag(&etag));
+
+        let expiration = resolve_put_object_expiration(&bucket, &committed.obj_info).await;
+
+        let mut checksums = PutObjectChecksums {
+            crc32: input.checksum_crc32,
+            crc32c: input.checksum_crc32c,
+            sha1: input.checksum_sha1,
+            sha256: input.checksum_sha256,
+            crc64nvme: input.checksum_crc64nvme,
+        };
+        apply_trailing_checksums(
+            input.checksum_algorithm.as_ref().map(|a| a.as_str()),
+            &req.trailing_headers,
+            &mut checksums,
+        );
+
+        let output = PutObjectOutput {
+            e_tag,
+            server_side_encryption: committed.effective_sse.clone(),
+            sse_customer_algorithm: committed.sse_customer_algorithm.clone(),
+            sse_customer_key_md5: committed.sse_customer_key_md5.clone(),
+            ssekms_key_id: committed.effective_kms_key_id.clone(),
+            expiration,
+            checksum_crc32: checksums.crc32,
+            checksum_crc32c: checksums.crc32c,
+            checksum_sha1: checksums.sha1,
+            checksum_sha256: checksums.sha256,
+            checksum_crc64nvme: checksums.crc64nvme,
+            version_id: put_version,
+            ..Default::default()
+        };
+
+        // For browser-based POST uploads (multipart/form-data), response status/body handling
+        // is decided by s3s PostObject serializer (success_action_status / redirect semantics).
+
+        let response_build_stage_start = put_stage_metrics_enabled.then(Instant::now);
+        let mut response = S3Response::new(output);
+        // Echo XXHash3/64/128 / SHA-512 checksums that s3s PutObjectOutput has no typed
+        // field for (#1256).
+        inject_additional_checksum_headers(&mut response.headers, &committed.put_extra_checksum_headers);
+        rustfs_io_metrics::record_put_object_stage_duration_from("app_response_build", response_build_stage_start);
+        let result = Ok(response);
+        committed.finish(&result);
+
+        result
+    }
+
+    /// The single-object write path shared by the S3 handler and internal
+    /// callers: quota admission, foreground write admission, bucket default
+    /// SSE, Object Lock defaults, put options, the hashing/compressing/
+    /// encrypting reader, the owned store commit, usage accounting,
+    /// replication scheduling and the creation-event setup. The caller shapes
+    /// the request before and builds its response after.
+    pub(super) async fn put_object_core(
+        &self,
+        write: PutObjectWriteRequest<'_>,
+        body: StreamingBlob,
+        start_time: Instant,
+    ) -> S3Result<PutObjectCommitted> {
+        let put_stage_metrics_enabled = rustfs_io_metrics::put_stage_metrics_enabled();
+        let PutObjectWriteRequest {
+            bucket,
+            key,
+            mut size,
+            quota_operation,
+            ciphertext_passthrough,
+            inbound_replication_put,
+            headers,
+            query,
+            trailing_headers,
+            version_id,
+            sse,
+            user_metadata,
+            internal_metadata,
+            content,
+            object_lock,
+            content_md5,
+            preserve_etag,
+            origin,
+        } = write;
+        let PutObjectSseInput {
+            server_side_encryption,
+            ssekms_key_id,
+            sse_customer_algorithm,
+            sse_customer_key,
+            sse_customer_key_md5,
+        } = sse;
+
         // The app check preserves the existing S3 error contract; the storage
         // commit path reserves the exact net logical growth under its locks.
         let quota_stage_start = put_stage_metrics_enabled.then(Instant::now);
@@ -1084,7 +1438,7 @@ impl DefaultObjectUsecase {
 
         let ingress_stage_start = put_stage_metrics_enabled.then(Instant::now);
         let should_compress =
-            is_disk_compressible(&req.headers, &key) && size > MIN_DISK_COMPRESSIBLE_SIZE as i64 && !ciphertext_passthrough;
+            is_disk_compressible(headers, &key) && size > MIN_DISK_COMPRESSIBLE_SIZE as i64 && !ciphertext_passthrough;
 
         // Resolve the store through the request-bound server context
         // (backlog#1052 S6), not the process-global handle, so an embedded
@@ -1175,7 +1529,7 @@ impl DefaultObjectUsecase {
         let (put_path, zero_copy_eager_put_path_status, use_zero_copy_eager_put_path, use_empty_or_small_eager_put_path) =
             select_put_path_with_concurrency(
                 size,
-                &req.headers,
+                headers,
                 server_side_encryption_requested,
                 should_compress,
                 false,
@@ -1200,33 +1554,33 @@ impl DefaultObjectUsecase {
         validate_sse_headers_for_write(
             effective_sse.as_ref(),
             effective_kms_key_id.as_ref(),
-            extract_ssekms_context_from_headers(&req.headers)?.as_ref(),
+            extract_ssekms_context_from_headers(headers)?.as_ref(),
             sse_customer_algorithm.as_ref(),
             sse_customer_key.as_ref(),
             sse_customer_key_md5.as_ref(),
             true, // PutObject requires all three: algorithm, key, key_md5
         )?;
 
-        let mut metadata = metadata.unwrap_or_default();
-        let has_explicit_object_lock_retention = object_lock_mode.is_some()
-            || object_lock_retain_until_date.is_some()
-            || has_replication_retention_update(&req.headers, inbound_replication_put);
+        let mut metadata = user_metadata;
+        let has_explicit_object_lock_retention = object_lock.mode.is_some()
+            || object_lock.retain_until_date.is_some()
+            || has_replication_retention_update(headers, inbound_replication_put);
         let object_lock_config_stage_start = put_stage_metrics_enabled.then(Instant::now);
         let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?;
         rustfs_io_metrics::record_put_object_stage_duration_from("app_object_lock_config_lookup", object_lock_config_stage_start);
         apply_put_request_metadata(
             &mut metadata,
-            &req.headers,
+            headers,
             &key,
-            cache_control,
-            content_disposition,
-            content_encoding,
-            content_language,
-            content_type,
-            expires,
-            website_redirect_location,
-            tagging,
-            storage_class.clone(),
+            content.cache_control,
+            content.content_disposition,
+            content.content_encoding,
+            content.content_language,
+            content.content_type,
+            content.expires,
+            content.website_redirect_location,
+            content.tagging,
+            content.storage_class,
         )?;
         apply_bucket_default_lock_retention(
             &bucket,
@@ -1234,29 +1588,33 @@ impl DefaultObjectUsecase {
             &mut metadata,
             has_explicit_object_lock_retention,
         )?;
+        metadata.extend(internal_metadata);
 
         let put_opts_stage_start = put_stage_metrics_enabled.then(Instant::now);
         let mut opts: ObjectOptions = put_opts_with_replication_authorization(
             &bucket,
             &key,
             version_id.clone(),
-            &req.headers,
+            headers,
             metadata.clone(),
-            replication_request_authorized(&req),
+            origin.replication_request_authorized(),
         )
         .await
         .map_err(ApiError::from)?;
+        if let Some(etag) = preserve_etag {
+            opts.preserve_etag = Some(etag);
+        }
         if let Some(quota_check) = quota_check.as_ref() {
             apply_quota_admission(&mut opts, quota_check)?;
         }
         rustfs_io_metrics::record_put_object_stage_duration_from("app_put_opts_build", put_opts_stage_start);
-        apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
+        origin.apply_bucket_generation_guard(&bucket, &mut opts)?;
         apply_put_request_object_lock_opts(
             &bucket,
             &object_lock_config_state,
-            object_lock_legal_hold_status,
-            object_lock_mode,
-            object_lock_retain_until_date,
+            object_lock.legal_hold_status,
+            object_lock.mode,
+            object_lock.retain_until_date,
             &mut opts,
         )?;
         let eager_put_commit_cancellation =
@@ -1278,7 +1636,7 @@ impl DefaultObjectUsecase {
         let prelookup_stage_start = (prelookup_required && put_stage_metrics_enabled).then(Instant::now);
         let prelookup_previous_current_size: Option<Option<u64>> = if prelookup_required {
             let current_opts: ObjectOptions = internal_object_info_lookup_opts(
-                get_opts(&bucket, &key, version_id.clone(), None, &req.headers)
+                get_opts(&bucket, &key, version_id.clone(), None, headers)
                     .await
                     .map_err(ApiError::from)?,
             );
@@ -1315,16 +1673,18 @@ impl DefaultObjectUsecase {
             )?;
         }
 
-        let mut md5hex = if let Some(base64_md5) = content_md5 {
-            let md5 = base64_simd::STANDARD
-                .decode_to_vec(base64_md5.as_bytes())
-                .map_err(|e| ApiError::from(StorageError::other(format!("Invalid content MD5: {e}"))))?;
-            Some(hex_simd::encode_to_string(&md5, hex_simd::AsciiCase::Lower))
-        } else {
-            None
+        let mut md5hex = match content_md5 {
+            Some(PutObjectContentMd5::Base64(base64_md5)) => {
+                let md5 = base64_simd::STANDARD
+                    .decode_to_vec(base64_md5.as_bytes())
+                    .map_err(|e| ApiError::from(StorageError::other(format!("Invalid content MD5: {e}"))))?;
+                Some(hex_simd::encode_to_string(&md5, hex_simd::AsciiCase::Lower))
+            }
+            Some(PutObjectContentMd5::Hex(md5hex)) => Some(md5hex),
+            None => None,
         };
 
-        let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query());
+        let mut sha256hex = get_content_sha256_with_query(headers, query);
 
         let mut write_plan = WritePlan::new();
         // Additional-checksum (XXHash3/64/128, SHA-512) values to echo on the PutObject
@@ -1342,7 +1702,7 @@ impl DefaultObjectUsecase {
             let mut hrd =
                 HashReader::from_stream(body, size, size, md5hex.take(), sha256hex.take(), false).map_err(ApiError::from)?;
 
-            if let Err(err) = hrd.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), false) {
+            if let Err(err) = hrd.add_checksum_from_s3s(headers, trailing_headers.clone(), false) {
                 return Err(ApiError::from(err).into());
             }
 
@@ -1392,7 +1752,7 @@ impl DefaultObjectUsecase {
         };
 
         if size >= 0 {
-            if let Err(err) = reader.add_checksum_from_s3s(&req.headers, req.trailing_headers.clone(), false) {
+            if let Err(err) = reader.add_checksum_from_s3s(headers, trailing_headers.clone(), false) {
                 return Err(ApiError::from(err).into());
             }
 
@@ -1402,12 +1762,35 @@ impl DefaultObjectUsecase {
         rustfs_io_metrics::record_put_object_path(put_path);
         rustfs_io_metrics::record_put_object_stage_duration_from("ingress_prepare", ingress_stage_start);
 
-        let mut helper = OperationHelper::new(&req, event_name, S3Operation::PutObject);
-        let ssekms_context = extract_ssekms_context_from_headers(&req.headers)?;
+        let (mut completion, request_context) = match &origin {
+            PutObjectOrigin::S3 { req, event_name } => (
+                PutObjectCompletion::S3(OperationHelper::new(req, *event_name, S3Operation::PutObject)),
+                req.extensions.get::<request_context::RequestContext>().cloned(),
+            ),
+            PutObjectOrigin::Internal {
+                principal_id,
+                emit_events,
+            } => {
+                let principal_id = *principal_id;
+                let request_context = request_context::RequestContext::fallback();
+                let event = emit_events.then(|| {
+                    InternalPutObjectEvent::new(
+                        current_notify_interface_for_context(self.context.as_deref()),
+                        request_context.clone(),
+                        EventName::ObjectCreatedPut,
+                        &bucket,
+                        &key,
+                        principal_id,
+                    )
+                });
+                (PutObjectCompletion::Internal(event.flatten().map(Box::new)), Some(request_context))
+            }
+        };
+        let ssekms_context = extract_ssekms_context_from_headers(headers)?;
 
         // Apply encryption using unified SSE API.
         let encryption_stage_start = put_stage_metrics_enabled.then(Instant::now);
-        let write_principal = SseKmsPrincipal::from_request(&req);
+        let write_principal = origin.sse_principal();
         let encryption_request = EncryptionRequest {
             bucket: &bucket,
             key: &key,
@@ -1430,11 +1813,7 @@ impl DefaultObjectUsecase {
         } else {
             match sse_encryption(encryption_request).await {
                 Ok(material) => material,
-                Err(err) => {
-                    let result = Err(err.into());
-                    let _ = helper.complete(&result);
-                    return result;
-                }
+                Err(err) => return Err(fail_put_object(completion, err.into())),
             }
         };
 
@@ -1460,7 +1839,6 @@ impl DefaultObjectUsecase {
 
         let mt2 = metadata.clone();
         opts.user_defined.extend(metadata);
-        let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
         let request_id = request_context
             .as_ref()
             .map(|ctx| ctx.request_id.clone())
@@ -1660,100 +2038,41 @@ impl DefaultObjectUsecase {
         let PutObjectCommitResult { obj_info, put_versioned } = match put_commit_result {
             Ok(Ok(result)) => result,
             Ok(Err(err)) => {
-                let result: S3Result<S3Response<PutObjectOutput>> = Err(err);
                 put_request_guard.finish_err();
-                let _ = helper.complete(&result);
-                return result;
+                return Err(fail_put_object(completion, err));
             }
             Err(err) => {
-                let result: S3Result<S3Response<PutObjectOutput>> = Err(S3Error::with_message(
-                    S3ErrorCode::InternalError,
-                    format!("put object commit owner task failed: {err}"),
-                ));
                 put_request_guard.finish_err();
-                let _ = helper.complete(&result);
-                return result;
+                return Err(fail_put_object(
+                    completion,
+                    S3Error::with_message(S3ErrorCode::InternalError, format!("put object commit owner task failed: {err}")),
+                ));
             }
         };
 
-        let raw_version = obj_info.version_id.map(|v| v.to_string());
-
-        helper = helper.object(obj_info.clone());
-        if let Some(version_id) = &raw_version {
-            helper = helper.version_id(version_id.clone());
+        completion = completion.object(obj_info.clone());
+        if let Some(version_id) = obj_info.version_id {
+            completion = completion.version_id(version_id.to_string());
         }
 
-        let put_version = if put_versioned { raw_version } else { None };
-
-        let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag));
-
-        let expiration = resolve_put_object_expiration(&bucket, &obj_info).await;
-
-        let mut checksums = PutObjectChecksums {
-            crc32: input.checksum_crc32,
-            crc32c: input.checksum_crc32c,
-            sha1: input.checksum_sha1,
-            sha256: input.checksum_sha256,
-            crc64nvme: input.checksum_crc64nvme,
-        };
-        apply_trailing_checksums(
-            input.checksum_algorithm.as_ref().map(|a| a.as_str()),
-            &req.trailing_headers,
-            &mut checksums,
-        );
-
-        let output = PutObjectOutput {
-            e_tag,
-            server_side_encryption: effective_sse,
-            sse_customer_algorithm: sse_customer_algorithm.clone(),
-            sse_customer_key_md5: sse_customer_key_md5.clone(),
-            ssekms_key_id: effective_kms_key_id,
-            expiration,
-            checksum_crc32: checksums.crc32,
-            checksum_crc32c: checksums.crc32c,
-            checksum_sha1: checksums.sha1,
-            checksum_sha256: checksums.sha256,
-            checksum_crc64nvme: checksums.crc64nvme,
-            version_id: put_version,
-            ..Default::default()
-        };
-
-        // For browser-based POST uploads (multipart/form-data), response status/body handling
-        // is decided by s3s PostObject serializer (success_action_status / redirect semantics).
-
-        let response_build_stage_start = put_stage_metrics_enabled.then(Instant::now);
-        let mut response = S3Response::new(output);
-        // Echo XXHash3/64/128 / SHA-512 checksums that s3s PutObjectOutput has no typed
-        // field for (#1256).
-        inject_additional_checksum_headers(&mut response.headers, &put_extra_checksum_headers);
-        rustfs_io_metrics::record_put_object_stage_duration_from("app_response_build", response_build_stage_start);
-        let result = Ok(response);
-        let _ = helper.complete(&result);
-
-        // Record PutObject metrics via zero-copy-metrics
-        {
-            let duration_ms = start_time.elapsed().as_millis() as f64;
-            rustfs_io_metrics::record_put_object(
-                duration_ms,
-                size,
-                use_zero_copy_eager_put_path, // Track if zero-copy was enabled
-            );
-        }
-
-        debug!(
-            target: "rustfs::app::object_usecase",
-            component = "app",
-            subsystem = "object",
-            bucket = %bucket,
-            key = %key,
+        Ok(PutObjectCommitted {
+            obj_info,
+            put_versioned,
+            effective_sse,
+            effective_kms_key_id,
+            sse_customer_algorithm,
+            sse_customer_key_md5,
+            put_extra_checksum_headers,
+            completion,
+            put_request_guard,
+            bucket,
+            key,
+            start_time,
+            size,
+            use_zero_copy_eager_put_path,
             concurrent_put_requests,
             buffer_size,
-            "PutObject request completed"
-        );
-
-        put_request_guard.finish_ok();
-
-        result
+        })
     }
 }
 

rustfs/src/module_switches.rs

@@ -35,14 +35,9 @@ pub(crate) const ENV_HEAL_ENABLED: &str = "RUSTFS_HEAL_ENABLED";
 pub(crate) const ENV_HEAL_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_HEAL";
 pub(crate) const ENV_BITROT_SELFTEST_ENABLE: &str = "RUSTFS_BITROT_SELFTEST_ENABLE";
 pub(crate) const ENV_BITROT_SELFTEST_STRICT: &str = "RUSTFS_BITROT_SELFTEST_STRICT";
-/// On-demand migration module switch (rustfs/backlog#2152). Off until GA
-/// (rustfs/backlog#2163) so every intermediate PR ships dark.
-pub(crate) const ENV_ON_DEMAND_MIGRATION_ENABLED: &str = "RUSTFS_ON_DEMAND_MIGRATION_ENABLED";
-pub(crate) const DEFAULT_ON_DEMAND_MIGRATION_ENABLED: bool = false;
 
 static AUDIT_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_AUDIT_ENABLE);
 static NOTIFY_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_NOTIFY_ENABLE);
-static ON_DEMAND_MIGRATION_MODULE_ENABLED: AtomicBool = AtomicBool::new(DEFAULT_ON_DEMAND_MIGRATION_ENABLED);
 
 /// Whether the data scanner is enabled, defaulting to on.
 pub(crate) fn scanner_enabled_from_env() -> bool {
@@ -85,51 +80,3 @@ pub fn is_notify_module_enabled() -> bool {
 pub(crate) fn set_notify_module_enabled(enabled: bool) {
     NOTIFY_MODULE_ENABLED.store(enabled, Ordering::Relaxed);
 }
-
-/// Whether the on-demand migration module is enabled, defaulting to off.
-/// Read once at startup by `startup_bucket_metadata` and published below.
-pub(crate) fn on_demand_migration_enabled_from_env() -> bool {
-    rustfs_utils::get_env_bool(ENV_ON_DEMAND_MIGRATION_ENABLED, DEFAULT_ON_DEMAND_MIGRATION_ENABLED)
-}
-
-/// Last published on-demand migration module state.
-pub fn is_on_demand_migration_module_enabled() -> bool {
-    ON_DEMAND_MIGRATION_MODULE_ENABLED.load(Ordering::Relaxed)
-}
-
-/// Publish the on-demand migration module state resolved at startup. The
-/// ecstore runtime receives the same value through
-/// `OnDemandMigrationSys::set_module_enabled`, since ecstore cannot read
-/// this crate.
-pub(crate) fn set_on_demand_migration_module_enabled(enabled: bool) {
-    ON_DEMAND_MIGRATION_MODULE_ENABLED.store(enabled, Ordering::Relaxed);
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn on_demand_migration_switch_defaults_off_and_follows_env() {
-        temp_env::with_var(ENV_ON_DEMAND_MIGRATION_ENABLED, None::<&str>, || {
-            assert!(!on_demand_migration_enabled_from_env());
-        });
-        temp_env::with_var(ENV_ON_DEMAND_MIGRATION_ENABLED, Some("true"), || {
-            assert!(on_demand_migration_enabled_from_env());
-        });
-        temp_env::with_var(ENV_ON_DEMAND_MIGRATION_ENABLED, Some("not-a-bool"), || {
-            assert!(!on_demand_migration_enabled_from_env(), "unparsable values keep the default");
-        });
-    }
-
-    #[test]
-    fn on_demand_migration_switch_publishes_to_the_cell() {
-        // The cell is process-global; restore it so sibling tests observe the default.
-        let before = is_on_demand_migration_module_enabled();
-        set_on_demand_migration_module_enabled(true);
-        assert!(is_on_demand_migration_module_enabled());
-        set_on_demand_migration_module_enabled(false);
-        assert!(!is_on_demand_migration_module_enabled());
-        set_on_demand_migration_module_enabled(before);
-    }
-}

rustfs/src/protocols/client.rs

@@ -163,8 +163,7 @@ fn build_object_uri(bucket: &str, key: &str, query: &[(&str, Option<&str>)]) ->
 struct RequestParams<'a> {
     bucket: Option<String>,
     object: Option<String>,
-    access_key: &'a str,
-    secret_key: &'a str,
+    credentials: &'a rustfs_credentials::Credentials,
 }
 
 /// Protocol storage client that implements the StorageBackend trait
@@ -181,39 +180,22 @@ impl ProtocolStorageClient {
     }
 
     /// Create a proper S3Request with ReqInfo extension for authorization
-    async fn create_request<T>(
-        &self,
-        input: T,
-        method: Method,
-        uri: http::Uri,
-        params: RequestParams<'_>,
-    ) -> S3Result<S3Request<T>> {
+    fn create_request<T>(input: T, method: Method, uri: http::Uri, params: RequestParams<'_>) -> S3Result<S3Request<T>> {
         let mut extensions = http::Extensions::default();
 
         let is_owner = if let Some(global_cred) = current_action_credentials() {
-            params.access_key == global_cred.access_key
+            params.credentials.access_key == global_cred.access_key
         } else {
             false
         };
 
         let credentials = Some(s3s::auth::Credentials {
-            access_key: params.access_key.to_string(),
-            secret_key: params.secret_key.to_string().into(),
+            access_key: params.credentials.access_key.clone(),
+            secret_key: params.credentials.secret_key.clone().into(),
         });
 
         extensions.insert(ReqInfo {
-            cred: Some(rustfs_credentials::Credentials {
-                access_key: params.access_key.to_string(),
-                secret_key: params.secret_key.to_string(),
-                session_token: String::new(),
-                expiration: None,
-                status: String::new(),
-                parent_user: String::new(),
-                groups: None,
-                claims: None,
-                name: None,
-                description: None,
-            }),
+            cred: Some(params.credentials.clone()),
             is_owner,
             bucket: params.bucket,
             object: params.object,
@@ -247,8 +229,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         &self,
         bucket: &str,
         key: &str,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &rustfs_credentials::Credentials,
         start_pos: Option<u64>,
     ) -> Result<GetObjectOutput, Self::Error> {
         trace_protocol_request("get_object", Some(bucket), Some(key));
@@ -279,19 +260,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         })?;
 
         let uri = build_object_uri(bucket, key, &[])?;
-        let req = self
-            .create_request(
-                input,
-                Method::GET,
-                uri,
-                RequestParams {
-                    bucket: Some(bucket.to_string()),
-                    object: Some(key.to_string()),
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::GET,
+            uri,
+            RequestParams {
+                bucket: Some(bucket.to_string()),
+                object: Some(key.to_string()),
+                credentials,
+            },
+        )?;
 
         match self.fs.get_object(req).await {
             Ok(response) => Ok(response.output),
@@ -302,8 +280,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn put_object(
         &self,
         input: PutObjectInput,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &rustfs_credentials::Credentials,
     ) -> Result<PutObjectOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
@@ -330,19 +307,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
             }
         }
 
-        let req = self
-            .create_request(
-                input,
-                Method::PUT,
-                uri,
-                RequestParams {
-                    bucket: Some(bucket),
-                    object: Some(key),
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::PUT,
+            uri,
+            RequestParams {
+                bucket: Some(bucket),
+                object: Some(key),
+                credentials,
+            },
+        )?;
         let req = S3Request { headers, ..req };
 
         match self.fs.put_object(req).await {
@@ -355,8 +329,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         &self,
         bucket: &str,
         key: &str,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &rustfs_credentials::Credentials,
     ) -> Result<DeleteObjectOutput, Self::Error> {
         trace_protocol_request("delete_object", Some(bucket), Some(key));
 
@@ -369,19 +342,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
             })?;
 
         let uri = build_object_uri(bucket, key, &[])?;
-        let req = self
-            .create_request(
-                input,
-                Method::DELETE,
-                uri,
-                RequestParams {
-                    bucket: Some(bucket.to_string()),
-                    object: Some(key.to_string()),
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::DELETE,
+            uri,
+            RequestParams {
+                bucket: Some(bucket.to_string()),
+                object: Some(key.to_string()),
+                credentials,
+            },
+        )?;
 
         match self.fs.delete_object(req).await {
             Ok(response) => Ok(response.output),
@@ -393,8 +363,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         &self,
         bucket: &str,
         key: &str,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &rustfs_credentials::Credentials,
     ) -> Result<HeadObjectOutput, Self::Error> {
         trace_protocol_request("head_object", Some(bucket), Some(key));
 
@@ -407,19 +376,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
             })?;
 
         let uri = build_object_uri(bucket, key, &[])?;
-        let req = self
-            .create_request(
-                input,
-                Method::HEAD,
-                uri,
-                RequestParams {
-                    bucket: Some(bucket.to_string()),
-                    object: Some(key.to_string()),
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::HEAD,
+            uri,
+            RequestParams {
+                bucket: Some(bucket.to_string()),
+                object: Some(key.to_string()),
+                credentials,
+            },
+        )?;
 
         match self.fs.head_object(req).await {
             Ok(response) => Ok(response.output),
@@ -427,7 +393,11 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         }
     }
 
-    async fn head_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<HeadBucketOutput, Self::Error> {
+    async fn head_bucket(
+        &self,
+        bucket: &str,
+        credentials: &rustfs_credentials::Credentials,
+    ) -> Result<HeadBucketOutput, Self::Error> {
         trace_protocol_request("head_bucket", Some(bucket), None);
 
         let input = HeadBucketInput::builder().bucket(bucket.to_string()).build().map_err(|e| {
@@ -435,19 +405,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         })?;
 
         let uri = build_bucket_uri(bucket, &[])?;
-        let req = self
-            .create_request(
-                input,
-                Method::HEAD,
-                uri,
-                RequestParams {
-                    bucket: Some(bucket.to_string()),
-                    object: None,
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::HEAD,
+            uri,
+            RequestParams {
+                bucket: Some(bucket.to_string()),
+                object: None,
+                credentials,
+            },
+        )?;
 
         match self.fs.head_bucket(req).await {
             Ok(response) => Ok(response.output),
@@ -458,26 +425,22 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn list_objects_v2(
         &self,
         input: ListObjectsV2Input,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &rustfs_credentials::Credentials,
     ) -> Result<ListObjectsV2Output, Self::Error> {
         trace_protocol_request("list_objects_v2", Some(&input.bucket), None);
 
         let bucket = input.bucket.clone();
         let uri = build_bucket_uri(&bucket, &[("list-type", Some("2"))])?;
-        let req = self
-            .create_request(
-                input,
-                Method::GET,
-                uri,
-                RequestParams {
-                    bucket: Some(bucket),
-                    object: None,
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::GET,
+            uri,
+            RequestParams {
+                bucket: Some(bucket),
+                object: None,
+                credentials,
+            },
+        )?;
 
         match self.fs.list_objects_v2(req).await {
             Ok(response) => Ok(response.output),
@@ -485,13 +448,13 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         }
     }
 
-    async fn list_buckets(&self, access_key: &str, secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
+    async fn list_buckets(&self, credentials: &rustfs_credentials::Credentials) -> Result<ListBucketsOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
             component = LOG_COMPONENT_PROTOCOLS,
             subsystem = LOG_SUBSYSTEM_STORAGE_CLIENT,
             operation = "list_buckets",
-            access_key = %MaskedAccessKey(access_key),
+            access_key = %MaskedAccessKey(&credentials.access_key),
             "Protocol storage client request"
         );
 
@@ -499,19 +462,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
             s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build ListBucketsInput: {}", e))
         })?;
 
-        let req = self
-            .create_request(
-                input,
-                Method::GET,
-                http::Uri::from_static("/"),
-                RequestParams {
-                    bucket: None,
-                    object: None,
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::GET,
+            http::Uri::from_static("/"),
+            RequestParams {
+                bucket: None,
+                object: None,
+                credentials,
+            },
+        )?;
 
         match self.fs.list_buckets(req).await {
             Ok(response) => Ok(response.output),
@@ -542,7 +502,11 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         self.fs.list_buckets(request).await.map(|response| response.output)
     }
 
-    async fn create_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<CreateBucketOutput, Self::Error> {
+    async fn create_bucket(
+        &self,
+        bucket: &str,
+        credentials: &rustfs_credentials::Credentials,
+    ) -> Result<CreateBucketOutput, Self::Error> {
         trace_protocol_request("create_bucket", Some(bucket), None);
 
         let input = CreateBucketInput::builder().bucket(bucket.to_string()).build().map_err(|e| {
@@ -550,19 +514,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         })?;
 
         let uri = build_bucket_uri(bucket, &[])?;
-        let req = self
-            .create_request(
-                input,
-                Method::PUT,
-                uri,
-                RequestParams {
-                    bucket: Some(bucket.to_string()),
-                    object: None,
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::PUT,
+            uri,
+            RequestParams {
+                bucket: Some(bucket.to_string()),
+                object: None,
+                credentials,
+            },
+        )?;
 
         match self.fs.create_bucket(req).await {
             Ok(response) => Ok(response.output),
@@ -574,8 +535,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         &self,
         bucket: &str,
         key: &str,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &rustfs_credentials::Credentials,
         start_pos: u64,
         length: u64,
     ) -> Result<GetObjectOutput, Self::Error> {
@@ -607,19 +567,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
             })?;
 
         let uri = build_object_uri(bucket, key, &[])?;
-        let req = self
-            .create_request(
-                input,
-                Method::GET,
-                uri,
-                RequestParams {
-                    bucket: Some(bucket.to_string()),
-                    object: Some(key.to_string()),
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::GET,
+            uri,
+            RequestParams {
+                bucket: Some(bucket.to_string()),
+                object: Some(key.to_string()),
+                credentials,
+            },
+        )?;
 
         match self.fs.get_object(req).await {
             Ok(response) => Ok(response.output),
@@ -630,8 +587,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn copy_object(
         &self,
         input: CopyObjectInput,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &rustfs_credentials::Credentials,
     ) -> Result<CopyObjectOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
@@ -647,19 +603,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         let key = input.key.clone();
         let uri = build_object_uri(&bucket, &key, &[])?;
 
-        let req = self
-            .create_request(
-                input,
-                Method::PUT,
-                uri,
-                RequestParams {
-                    bucket: Some(bucket),
-                    object: Some(key),
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::PUT,
+            uri,
+            RequestParams {
+                bucket: Some(bucket),
+                object: Some(key),
+                credentials,
+            },
+        )?;
 
         match self.fs.copy_object(req).await {
             Ok(response) => Ok(response.output),
@@ -667,7 +620,11 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         }
     }
 
-    async fn delete_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<DeleteBucketOutput, Self::Error> {
+    async fn delete_bucket(
+        &self,
+        bucket: &str,
+        credentials: &rustfs_credentials::Credentials,
+    ) -> Result<DeleteBucketOutput, Self::Error> {
         trace_protocol_request("delete_bucket", Some(bucket), None);
 
         let input = DeleteBucketInput::builder().bucket(bucket.to_string()).build().map_err(|e| {
@@ -675,19 +632,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         })?;
 
         let uri = build_bucket_uri(bucket, &[])?;
-        let req = self
-            .create_request(
-                input,
-                Method::DELETE,
-                uri,
-                RequestParams {
-                    bucket: Some(bucket.to_string()),
-                    object: None,
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::DELETE,
+            uri,
+            RequestParams {
+                bucket: Some(bucket.to_string()),
+                object: None,
+                credentials,
+            },
+        )?;
 
         match self.fs.delete_bucket(req).await {
             Ok(response) => Ok(response.output),
@@ -698,8 +652,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn create_multipart_upload(
         &self,
         input: CreateMultipartUploadInput,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &rustfs_credentials::Credentials,
     ) -> Result<CreateMultipartUploadOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
@@ -715,19 +668,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         let key = input.key.clone();
         let uri = build_object_uri(&bucket, &key, &[("uploads", None)])?;
 
-        let req = self
-            .create_request(
-                input,
-                Method::POST,
-                uri,
-                RequestParams {
-                    bucket: Some(bucket),
-                    object: Some(key),
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::POST,
+            uri,
+            RequestParams {
+                bucket: Some(bucket),
+                object: Some(key),
+                credentials,
+            },
+        )?;
 
         match self.fs.create_multipart_upload(req).await {
             Ok(response) => Ok(response.output),
@@ -738,8 +688,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn upload_part(
         &self,
         input: UploadPartInput,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &rustfs_credentials::Credentials,
     ) -> Result<UploadPartOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
@@ -786,19 +735,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
             }
         }
 
-        let req = self
-            .create_request(
-                input,
-                Method::PUT,
-                uri,
-                RequestParams {
-                    bucket: Some(bucket),
-                    object: Some(key),
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::PUT,
+            uri,
+            RequestParams {
+                bucket: Some(bucket),
+                object: Some(key),
+                credentials,
+            },
+        )?;
         let req = S3Request { headers, ..req };
 
         match self.fs.upload_part(req).await {
@@ -810,8 +756,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn complete_multipart_upload(
         &self,
         input: CompleteMultipartUploadInput,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &rustfs_credentials::Credentials,
     ) -> Result<CompleteMultipartUploadOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
@@ -828,19 +773,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         let upload_id = input.upload_id.clone();
         let uri = build_object_uri(&bucket, &key, &[("uploadId", Some(upload_id.as_str()))])?;
 
-        let req = self
-            .create_request(
-                input,
-                Method::POST,
-                uri,
-                RequestParams {
-                    bucket: Some(bucket),
-                    object: Some(key),
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::POST,
+            uri,
+            RequestParams {
+                bucket: Some(bucket),
+                object: Some(key),
+                credentials,
+            },
+        )?;
 
         match self.fs.complete_multipart_upload(req).await {
             Ok(response) => Ok(response.output),
@@ -851,8 +793,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn abort_multipart_upload(
         &self,
         input: AbortMultipartUploadInput,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &rustfs_credentials::Credentials,
     ) -> Result<AbortMultipartUploadOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
@@ -870,19 +811,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
         let upload_id = input.upload_id.clone();
         let uri = build_object_uri(&bucket, &key, &[("uploadId", Some(upload_id.as_str()))])?;
 
-        let req = self
-            .create_request(
-                input,
-                Method::DELETE,
-                uri,
-                RequestParams {
-                    bucket: Some(bucket),
-                    object: Some(key),
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::DELETE,
+            uri,
+            RequestParams {
+                bucket: Some(bucket),
+                object: Some(key),
+                credentials,
+            },
+        )?;
 
         match self.fs.abort_multipart_upload(req).await {
             Ok(response) => Ok(response.output),
@@ -893,8 +831,7 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
     async fn upload_part_copy(
         &self,
         input: UploadPartCopyInput,
-        access_key: &str,
-        secret_key: &str,
+        credentials: &rustfs_credentials::Credentials,
     ) -> Result<UploadPartCopyOutput, Self::Error> {
         trace!(
             event = EVENT_PROTOCOL_STORAGE_CLIENT_REQUEST,
@@ -921,19 +858,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
             ],
         )?;
 
-        let req = self
-            .create_request(
-                input,
-                Method::PUT,
-                uri,
-                RequestParams {
-                    bucket: Some(bucket),
-                    object: Some(key),
-                    access_key,
-                    secret_key,
-                },
-            )
-            .await?;
+        let req = Self::create_request(
+            input,
+            Method::PUT,
+            uri,
+            RequestParams {
+                bucket: Some(bucket),
+                object: Some(key),
+                credentials,
+            },
+        )?;
 
         match self.fs.upload_part_copy(req).await {
             Ok(response) => Ok(response.output),
@@ -945,6 +879,47 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
 #[cfg(test)]
 mod tests {
     use super::*;
+    use rustfs_credentials::{IAM_POLICY_CLAIM_NAME_SA, INHERITED_POLICY_TYPE};
+
+    #[test]
+    fn create_request_preserves_authenticated_service_account_identity() {
+        let claims = std::collections::HashMap::from([
+            ("parent".to_string(), serde_json::json!("alice")),
+            (IAM_POLICY_CLAIM_NAME_SA.to_string(), serde_json::json!(INHERITED_POLICY_TYPE)),
+        ]);
+        let credentials = rustfs_credentials::Credentials {
+            access_key: "service-account".to_string(),
+            secret_key: "secret".to_string(),
+            session_token: "signed-service-account-token".to_string(),
+            parent_user: "alice".to_string(),
+            groups: Some(vec!["developers".to_string()]),
+            claims: Some(claims.clone()),
+            ..Default::default()
+        };
+
+        let request = ProtocolStorageClient::create_request(
+            ListObjectsV2Input::default(),
+            Method::GET,
+            http::Uri::from_static("/bucket?list-type=2"),
+            RequestParams {
+                bucket: Some("bucket".to_string()),
+                object: None,
+                credentials: &credentials,
+            },
+        )
+        .expect("request should build");
+        let request_info = request.extensions.get::<ReqInfo>().expect("request info should be present");
+        let copied = request_info.cred.as_ref().expect("credentials should be present");
+
+        assert_eq!(copied.access_key, credentials.access_key);
+        assert_eq!(copied.secret_key, credentials.secret_key);
+        assert_eq!(copied.session_token, credentials.session_token);
+        assert_eq!(copied.parent_user, credentials.parent_user);
+        assert_eq!(copied.groups, credentials.groups);
+        assert_eq!(copied.claims, Some(claims));
+        assert!(copied.is_service_account());
+    }
+
     #[cfg(feature = "webdav")]
     #[test]
     fn request_extensions_preserve_authenticated_identity_and_source_ip() {

rustfs/src/startup_bucket_metadata.rs

@@ -12,11 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::module_switches::{on_demand_migration_enabled_from_env, set_on_demand_migration_module_enabled};
 use crate::storage_api::startup::bucket_metadata::contract::bucket::{BucketOperations, BucketOptions};
 use crate::storage_api::startup::bucket_metadata::{
-    ECStore, Error as StorageError, OnDemandMigrationSys, Result as StorageResult, get_global_replication_pool,
-    init_bucket_metadata_sys, reconcile_bucket_resync_target_intents, try_migrate_bucket_metadata, try_migrate_iam_config,
+    ECStore, Error as StorageError, Result as StorageResult, get_global_replication_pool, init_bucket_metadata_sys,
+    reconcile_bucket_resync_target_intents, try_migrate_bucket_metadata, try_migrate_iam_config,
 };
 use std::{
     io::{Error as IoError, Result as IoResult},
@@ -25,13 +24,11 @@ use std::{
 };
 use tokio_util::sync::CancellationToken;
 
-const EVENT_ON_DEMAND_MIGRATION_RUNTIME_INITIALIZED: &str = "on_demand_migration_runtime_initialized";
 const EVENT_REPLICATION_RESYNC_STARTUP_BACKGROUND_CANCELED: &str = "replication_resync_startup_background_canceled";
 const EVENT_REPLICATION_RESYNC_STARTUP_BACKGROUND_COMPLETED: &str = "replication_resync_startup_background_completed";
 const EVENT_REPLICATION_RESYNC_STARTUP_BACKGROUND_FAILED: &str = "replication_resync_startup_background_failed";
 const EVENT_REPLICATION_RESYNC_STARTUP_BACKGROUND_STARTED: &str = "replication_resync_startup_background_started";
 const LOG_COMPONENT_STARTUP_BUCKET_METADATA: &str = "startup_bucket_metadata";
-const LOG_SUBSYSTEM_ON_DEMAND_MIGRATION: &str = "on_demand_migration";
 const LOG_SUBSYSTEM_REPLICATION: &str = "replication";
 const METRIC_REPLICATION_RESYNC_STARTUP_BACKGROUND_DURATION_SECONDS: &str =
     "rustfs_replication_resync_startup_background_duration_seconds";
@@ -61,7 +58,6 @@ pub(crate) async fn init_embedded_bucket_metadata_runtime(store: Arc<ECStore>, c
     let buckets: Vec<String> = buckets_list.into_iter().map(|v| v.name).collect();
 
     try_migrate_bucket_metadata(store.clone()).await;
-    init_on_demand_migration_runtime();
     init_bucket_metadata_sys(store.clone(), buckets.clone()).await;
     try_migrate_iam_config(store).await;
     spawn_bucket_resync_startup_reconcile(buckets.clone(), ctx.clone(), false);
@@ -83,33 +79,12 @@ pub(crate) async fn init_bucket_metadata_runtime(store: Arc<ECStore>, ctx: Cance
     try_migrate_bucket_metadata(store.clone()).await;
 
     try_migrate_iam_config(store.clone()).await;
-    init_on_demand_migration_runtime();
     init_bucket_metadata_sys(store, buckets.clone()).await;
     spawn_bucket_resync_startup_reconcile(buckets.clone(), ctx, true);
 
     Ok(buckets)
 }
 
-/// Publishes the on-demand migration module switch and registers the
-/// runtime's config hook before bucket metadata is loaded, so every cache
-/// install path (initial load included) reaches `OnDemandMigrationSys`
-/// (rustfs/backlog#2152). Idempotent across embedded and server startups.
-fn init_on_demand_migration_runtime() {
-    let enabled = on_demand_migration_enabled_from_env();
-    set_on_demand_migration_module_enabled(enabled);
-    let sys = OnDemandMigrationSys::get();
-    sys.set_module_enabled(enabled);
-    let hook_registered = sys.register_config_hook();
-    tracing::info!(
-        event = EVENT_ON_DEMAND_MIGRATION_RUNTIME_INITIALIZED,
-        component = LOG_COMPONENT_STARTUP_BUCKET_METADATA,
-        subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION,
-        state = if enabled { "enabled" } else { "disabled" },
-        hook_registered,
-        "On-demand migration runtime initialized"
-    );
-}
-
 fn spawn_bucket_resync_startup_reconcile(buckets: Vec<String>, ctx: CancellationToken, init_resync_after_reconcile: bool) {
     tokio::spawn(async move {
         describe_bucket_resync_startup_background_metrics();

rustfs/src/storage/access.rs

@@ -4361,10 +4361,15 @@ mod tests {
         apply_bucket_generation_guard(&req, &bucket, &mut opts).expect("apply the RestoreObject authorization guard");
         assert_eq!(opts.expected_bucket_incarnation_id, Some(authorized_incarnation_id));
 
-        let err = crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(app_context))
-            .execute_restore_object(req)
-            .await
-            .expect_err("the old RestoreObject authorization must not reach the recreated bucket");
+        // The RestoreObject usecase future is large enough that, inlined into
+        // this test body, the test thread's 2 MiB stack sits within a few KiB
+        // of overflowing on Linux; heap-pin it so unrelated growth in bucket
+        // metadata futures cannot tip the test over.
+        let err = Box::pin(
+            crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(app_context)).execute_restore_object(req),
+        )
+        .await
+        .expect_err("the old RestoreObject authorization must not reach the recreated bucket");
         assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
         store
             .delete_bucket(&bucket, &DeleteBucketOptions::default())

rustfs/src/storage/storage_api.rs

@@ -407,8 +407,8 @@ pub(crate) mod ecstore_bucket {
     #[cfg(test)]
     pub(crate) use rustfs_ecstore::api::bucket::lifecycle::tier_delete_journal::test_util::install_all_v6_fleet_capability_proof;
     pub(crate) use rustfs_ecstore::api::bucket::{
-        bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, migration, object_lock, on_demand_migration,
-        policy_sys, replication, tagging, target, utils,
+        bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys,
+        replication, tagging, target, utils,
     };
     pub(crate) use rustfs_ecstore::api::bucket::{quota, versioning, versioning_sys};
 }

rustfs/src/storage_api.rs

@@ -290,7 +290,6 @@ pub(crate) mod startup {
             }
         }
 
-        pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::OnDemandMigrationSys;
         pub(crate) use crate::storage::storage_api::{
             ECStore, Error, Result, get_global_replication_pool, init_bucket_metadata_sys,
             reconcile_bucket_resync_target_intents, try_migrate_bucket_metadata, try_migrate_iam_config,