test(e2e): activate conditional write regressions (#6454 )

fix(ci): restore main branch checks (#6465 )
* fix(protocols): add missing test dependency * style(ecstore): fix decommission formatting
2026-08-23 23:34:50 +08:00 · 2026-08-23 23:34:46 +08:00 · 2026-08-23 23:22:38 +08:00 · 2026-08-23 23:22:03 +08:00 · 2026-08-23 23:17:14 +08:00 · 2026-08-23 23:04:35 +08:00
273 changed files with 45484 additions and 4431 deletions
@@ -0,0 +1,20 @@
+# Report-only calibration baseline from https://github.com/rustfs/rustfs/actions/runs/29394996173.
+# Update counts only with a linked coverage run and a reviewed explanation.
+phase = "report-only"
+allowed_drop_percentage_points = 1.0
+
+[crates."crates/iam"]
+covered = 5149
+count = 8131
+
+[crates."crates/kms"]
+covered = 2950
+count = 4200
+
+[crates."crates/policy"]
+covered = 4636
+count = 5464
+
+[crates."crates/crypto"]
+covered = 469
+count = 494
@@ -1,2 +1,2 @@
-sha256-darwin=b4ae71aa894e5c7795ae3eb8116f1777a7601d0f5db3898be2e48faf3329bd9b
-sha256-linux=433debd9d9defa832986269abdf0f1d131597b2d7a417ce930e17c1fd47d85ba
+sha256-darwin=ae412c27e2e43f3fda3e0c21ef616b48a6403354faab8db8d4154202e3c4eec5
+sha256-linux=ef3be856bd3257c2c369428f66a48ad073dad8187229fe95840a600a81edf22b
@@ -36,6 +36,8 @@ script-tests: ## Run shell script tests
 	./scripts/test_manual_transition_runbooks.sh
 	./scripts/check_embedded_secrets.sh --self-test
 	python3 ./scripts/check_test_wiring.py --self-test
+	python3 ./scripts/check_security_coverage.py --self-test
+	python3 ./scripts/check_scheduled_validation_freshness.py --self-test
 	python3 ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
@@ -78,6 +78,12 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'

+# The durable ILM decommission regressions build isolated multi-pool stores and
+# deliberately take source or target disks offline while checking fencing.
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
+test-group = 'ecstore-serial-flaky'
+
 # Serialize the bucket-incarnation / lifecycle-fence tests. They drive
 # init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
 # OnceLock state that serial_test's #[serial] cannot protect across nextest's
@@ -190,6 +196,10 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'

+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
+test-group = 'ecstore-serial-flaky'
+
 # Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
 # too (see the matching default-profile override near the top). No retries.
 [[profile.ci.overrides]]
@@ -305,7 +315,7 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
 #     the target incl. multipart and the resync path, SSE-C and
 #     target-without-KMS stay fail-closed), and one guards event/history
 #     observers.
-#   * 12 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
+#   * 13 `_real_dual_node` site-replication tests — each spawns TWO full rustfs
 #     servers and drives the cross-process site-replication control plane.
 #   * 1 `_real_three_node` site-replication test.
 #   * 1 `_real_single_node` service-account round-trip test.
@@ -325,8 +335,8 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
 #
 # Wired by .github/workflows/e2e-replication-nightly.yml (schedule +
 # workflow_dispatch), which builds the rustfs binary once, installs awscurl so
-# the STS dual-node test actually exercises its path (it skips gracefully with
-# a visible log line when awscurl is absent), and routes scheduled failures
+# the STS dual-node test actually exercises its path (the test fails when
+# awscurl is absent), and routes scheduled failures
 # through .github/actions/schedule-failure-issue (ci-8). Explicit division of
 # labor with e2e-full: these tests run only in the consolidated nightly
 # workflow, not in the merge/main lane.
@@ -396,7 +406,7 @@ path = "junit.xml"
 #     object_lambda) — too heavy for the merge budget; they run in the
 #     e2e-nightly serial cluster-fault lane.
 #   * replication_extension_test — repl-1 already splits it into the PR
-#     `e2e-smoke` (20 fast) and `e2e-repl-nightly` (55 slow) lanes and reserves
+#     `e2e-smoke` (20 fast) and `e2e-repl-nightly` (56 slow) lanes and reserves
 #     it for those, so e2e-full does not double-run it.
 #   * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
 #     manual-localhost:9000 reliant/policy tests are ci-13's migration.
@@ -14,9 +14,10 @@

 name: "Schedule Failure Issue"
 description: >-
-  Open (or update) a tracking issue when a scheduled workflow run fails.
+  Open (or update) a tracking issue when a scheduled workflow run fails or
+  does not complete normally.
  Dedupes by workflow name: if an open issue titled
-  "[scheduled-failure] <workflow name>" already exists, the failure is
+  "[scheduled-failure] <workflow name>" already exists, the result is
  appended as a comment; otherwise a new issue is created. This is the
  single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8).

@@ -38,6 +39,30 @@ inputs:
      Set to an empty string to skip labeling.
    required: false
    default: "infrastructure"
+  source-run-id:
+    description: "Run ID to report. Defaults to the current workflow run."
+    required: false
+    default: ${{ github.run_id }}
+  source-run-attempt:
+    description: "Run attempt to report. Defaults to the current attempt."
+    required: false
+    default: ${{ github.run_attempt }}
+  source-event:
+    description: "Trigger event of the run being reported."
+    required: false
+    default: ${{ github.event_name }}
+  source-ref-name:
+    description: "Ref name of the run being reported."
+    required: false
+    default: ${{ github.ref_name }}
+  source-sha:
+    description: "Commit SHA of the run being reported."
+    required: false
+    default: ${{ github.sha }}
+  details-file:
+    description: "Optional Markdown file appended to the issue body."
+    required: false
+    default: ""

 runs:
  using: "composite"
@@ -48,17 +73,22 @@ runs:
        GH_TOKEN: ${{ inputs.github-token }}
        WORKFLOW_NAME: ${{ inputs.workflow-name }}
        ISSUE_LABEL: ${{ inputs.label }}
+        SOURCE_RUN_ID: ${{ inputs.source-run-id }}
+        SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }}
+        SOURCE_EVENT: ${{ inputs.source-event }}
+        SOURCE_REF_NAME: ${{ inputs.source-ref-name }}
+        SOURCE_SHA: ${{ inputs.source-sha }}
+        DETAILS_FILE: ${{ inputs.details-file }}
      run: |
        set -euo pipefail

        title="[scheduled-failure] ${WORKFLOW_NAME}"
-        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}"

-        # Failed job names for this run attempt. The alert job runs while the
-        # run as a whole is still in progress, so inspect the jobs that have
-        # already completed with a non-success conclusion.
+        # Inspect the reported run attempt. It can be the current in-workflow
+        # failure or a completed run observed by the external watchdog.
        failed_jobs="$(gh api \
-          "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \
+          "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \
          --paginate \
          --jq '.jobs[]
                | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled")
@@ -67,15 +97,26 @@ runs:
          failed_jobs="- (failed job not recorded yet — see the run page)"
        fi

+        details=""
+        if [ -n "${DETAILS_FILE}" ]; then
+          if [ -f "${DETAILS_FILE}" ]; then
+            details="$(cat "${DETAILS_FILE}")"
+          else
+            details="Details file was not available: \`${DETAILS_FILE}\`"
+          fi
+        fi
+
        body="$(cat <<EOF
-        Scheduled run of **${WORKFLOW_NAME}** failed.
+        Run of **${WORKFLOW_NAME}** did not complete successfully.

-        - Run: ${run_url} (attempt ${GITHUB_RUN_ATTEMPT})
-        - Event: \`${GITHUB_EVENT_NAME}\`
-        - Ref: \`${GITHUB_REF_NAME}\` @ \`${GITHUB_SHA}\`
+        - Run: ${run_url} (attempt ${SOURCE_RUN_ATTEMPT})
+        - Event: \`${SOURCE_EVENT}\`
+        - Ref: \`${SOURCE_REF_NAME}\` @ \`${SOURCE_SHA}\`

-        Failed jobs:
+        Non-success jobs:
        ${failed_jobs}
+
+        ${details}
        EOF
        )"

@@ -0,0 +1,14 @@
+[
+  { "workflow": ".github/workflows/audit.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/build.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/minio-interop.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/runner-hygiene.yml", "max_age_hours": 792 }
+]
@@ -46,7 +46,7 @@ on:
    # advisory could sit unnoticed for seven days. The check list is unchanged —
    # splitting it into a light daily advisories-only run and a weekly full run
    # would create runs where sources/bans/licenses go unverified.
-    - cron: '0 3 * * *' # Daily 03:00 UTC (staggered after the midnight ci/build crons)
+    - cron: '23 3 * * *' # Daily 03:23 UTC
  workflow_dispatch:

 permissions:
@@ -52,7 +52,7 @@ on:
      - ".dockerignore"
      - "flake.lock"
  schedule:
-    - cron: "0 1 * * 0" # Weekly on Sunday 01:00 UTC (staggered after the ci.yml midnight cron)
+    - cron: "13 1 * * 0" # Weekly on Sunday 01:13 UTC
  workflow_dispatch:
    inputs:
      build_docker:
@@ -1032,3 +1032,23 @@ jobs:

          echo "🎉 Released $TAG successfully!"
          echo "📄 Release URL: ${{ needs.create-release.outputs.release_url }}"
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [build-check, prepare-platform-matrix, build-rustfs, build-summary]
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -94,6 +94,9 @@ concurrency:

 env:
  CARGO_TERM_COLOR: always
+  # Swatinem/rust-cache hashes every RUST* variable. Keep this aligned with
+  # ci.yml or the writer and readers use disjoint cache keys.
+  RUST_BACKTRACE: 1

 jobs:
  # Readers: test-and-lint, test-ilm-integration-serial, build-rustfs-debug-binary,
@@ -101,7 +104,7 @@ jobs:
  warm-ci-dev:
    name: Warm ci-dev
    runs-on: sm-standard-4
-    timeout-minutes: 90
+    timeout-minutes: 120
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
@@ -191,7 +194,7 @@ jobs:
  warm-ci-feat-rio:
    name: Warm ci-feat-rio
    runs-on: sm-standard-4
-    timeout-minutes: 90
+    timeout-minutes: 120
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
@@ -219,7 +222,7 @@ jobs:
  warm-ci-feat-proto:
    name: Warm ci-feat-proto
    runs-on: sm-standard-4
-    timeout-minutes: 90
+    timeout-minutes: 120
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
@@ -126,7 +126,10 @@ jobs:
        run: ./scripts/check_embedded_secrets.sh

      - name: Check test wiring
-        run: python3 ./scripts/check_test_wiring.py
+        run: |
+          python3 ./scripts/check_test_wiring.py --self-test
+          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
+          python3 ./scripts/check_test_wiring.py

      - name: Check no planning docs committed
        run: ./scripts/check_no_planning_docs.sh
@@ -59,7 +59,7 @@ on:
  merge_group:
    types: [ checks_requested ]
  schedule:
-    - cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
+    - cron: "11 0 * * 0" # Weekly on Sunday 00:11 UTC
  workflow_dispatch:

 permissions:
@@ -161,7 +161,10 @@ jobs:
        run: ./scripts/check_embedded_secrets.sh

      - name: Check test wiring
-        run: python3 ./scripts/check_test_wiring.py
+        run: |
+          python3 ./scripts/check_test_wiring.py --self-test
+          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
+          python3 ./scripts/check_test_wiring.py

      - name: Check no planning docs committed
        run: ./scripts/check_no_planning_docs.sh
@@ -678,6 +681,19 @@ jobs:
          cache-save-if: 'false'
          install-build-packaging-tools: 'false'

+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.12"
+
+      - name: Install awscurl
+        run: |
+          python3 -m pip install --user --upgrade pip "awscurl==0.44"
+          echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
+
+      - name: Verify awscurl
+        run: test -x "$AWSCURL_PATH"
+
      # Download after the cache restore so the freshly built binary from the
      # build job always wins over anything restored into target/debug.
      - name: Download debug binary
@@ -800,6 +816,20 @@ jobs:
      - name: Verify awscurl
        run: test -x "$AWSCURL_PATH"

+      - name: Install mc
+        env:
+          MC_VERSION: RELEASE.2025-08-13T08-35-41Z
+          MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
+        run: |
+          MC_BINARY="mc.linux-amd64.${MC_VERSION}"
+          curl -fsSLo "$RUNNER_TEMP/mc" "https://github.com/minio/mc/releases/download/${MC_VERSION}/${MC_BINARY}"
+          echo "${MC_SHA256}  $RUNNER_TEMP/mc" | sha256sum --check --status
+          chmod +x "$RUNNER_TEMP/mc"
+          echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
+
+      - name: Verify mc
+        run: mc --version
+
      - name: Install Vault
        run: |
          VAULT_VERSION="1.17.6"
@@ -1032,3 +1062,37 @@ jobs:
          path: artifacts/s3tests-single/**
          if-no-files-found: ignore
          retention-days: 3
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs:
+      - typos
+      - quick-checks
+      - test-and-lint
+      - test-ilm-integration-serial
+      - test-and-lint-rio-v2
+      - test-and-lint-protocols
+      - build-rustfs-debug-binary
+      - build-rustfs-debug-binary-rio-v2
+      - uring-integration
+      - e2e-tests
+      - e2e-full
+      - e2e-tests-rio-v2
+      - s3-implemented-tests
+      - s3-lifecycle-behavior-tests
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -12,14 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-# Weekly workspace line-coverage baseline (backlog#1153 infra-5).
+# Workspace line-coverage baseline and security-crate calibration
+# (backlog#1153 infra-5/infra-6).
 #
-# NON-BLOCKING by design: this workflow only runs on schedule and manual
-# dispatch, so it never attaches a status to a PR and must never be made a
-# required check. It exists to give coverage a visible baseline and trend
-# (per-crate table in the job summary, lcov artifact kept 90 days) — the
-# per-crate ratchet for the security-critical crates builds on it later
-# (backlog#1153 infra-6, report-only first per the ci-11 ladder).
+# NON-BLOCKING by design: the weekly job gives coverage a visible baseline and
+# trend, while relevant pull requests run a report-only security-crate
+# comparison. Neither job is a required check during calibration.
 #
 # Measurement scope matches the PR test gate (ci.yml "Run tests"):
 # `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are
@@ -31,13 +29,28 @@
 name: coverage

 on:
+  pull_request:
+    branches: [main]
+    paths:
+      - "crates/iam/**"
+      - "crates/kms/**"
+      - "crates/policy/**"
+      - "crates/crypto/**"
+      - ".config/coverage-baselines.toml"
+      - "scripts/coverage_per_crate.py"
+      - "scripts/check_security_coverage.py"
+      - ".github/workflows/coverage.yml"
  workflow_dispatch:
  schedule:
    # 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
    # build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
    # (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
    # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
-    - cron: "0 7 * * 0"
+    - cron: "43 7 * * 0"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'schedule' }}

 # Only alert-on-failure needs more than read access; it declares its own
 # job-level `issues: write`.
@@ -46,12 +59,14 @@ permissions:

 jobs:
  coverage:
-    name: Workspace coverage (weekly)
+    name: Workspace line coverage
    runs-on: sm-standard-4
    # The instrumented build cannot reuse the regular CI cache (different
-    # RUSTFLAGS), so a cold week rebuilds the workspace before running the
-    # full suite; give it double the test job's 60-minute budget.
-    timeout-minutes: 120
+    # RUSTFLAGS), so a cold run rebuilds the workspace before running the
+    # full suite. Two later exact-head runs exhausted 150 minutes before the
+    # report steps, so allow one additional 90-minute cold-run margin while
+    # keeping the calibration job bounded.
+    timeout-minutes: 240
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
      # Match the PR gate's nextest semantics (ci.yml runs `--profile ci`):
@@ -91,7 +106,9 @@ jobs:
          cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json

      - name: Write per-crate summary
-        run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
+        run: |
+          python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
+          python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"

      - name: Upload coverage artifact
        if: always()
@@ -40,7 +40,7 @@ on:
  schedule:
    # 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00),
    # stale (01:30) and performance-ab (06:00).
-    - cron: "0 4 * * *"
+    - cron: "29 4 * * *"

 # Only alert-on-failure needs more than read access; it declares its own
 # job-level `issues: write`.
@@ -75,11 +75,7 @@ jobs:
          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
          install-build-packaging-tools: 'false'

-      # awscurl lets the STS dual-node test actually exercise its path. Without
-      # it the test skips gracefully with a visible log line
-      # (`awscurl_available()` in crates/e2e_test/src/common.rs), so the lane
-      # still passes — installing it just upgrades that one test from skip to
-      # real coverage.
+      # The STS dual-node test requires awscurl and fails if it is unavailable.
      - name: Set up Python
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
        with:
@@ -87,7 +83,7 @@ jobs:

      - name: Install awscurl
        run: |
-          python3 -m pip install --user --upgrade pip awscurl
+          python3 -m pip install --user --upgrade pip "awscurl==0.44"
          echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"

      - name: Verify awscurl
@@ -196,6 +192,12 @@ jobs:
          cache-save-if: 'false'
          install-build-packaging-tools: 'false'

+      - name: Install and verify protocol socket oracle
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq iproute2
+          ss -tn state CLOSE-WAIT >/dev/null
+
      # The suite owns fixed protocol ports and serializes its internal cases.
      - name: Verify protocol e2e membership
        env:
@@ -21,6 +21,9 @@
 #     suite and reports promotion candidates. Regressions, unclassified tests,
 #     incomplete execution, and infrastructure errors fail the job; classified
 #     failures for not-yet-implemented features remain informational.
+#   - Non-blocking upstream HEAD canary: collects current upstream node IDs and
+#     reports new, removed, duplicate, or overlapping classifications without
+#     making upstream drift a release gate.
 #   - Manual runs (workflow_dispatch): same, with configurable mode/scope.
 #
 # All test execution is delegated to scripts/s3-tests/run.sh (single source of
@@ -97,7 +100,7 @@ on:
  schedule:
    # Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the
    # single-node and the 4-node distributed topologies (matrix below).
-    - cron: "0 2 * * 0"
+    - cron: "19 2 * * 0"

 env:
  # main user
@@ -178,9 +181,14 @@ jobs:

      - name: Install Python tools
        run: |
-          python3 -m pip install --user --upgrade pip awscurl tox
+          python3 -m pip install --user --upgrade pip "awscurl==0.44" "tox==4.60.0"
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"

+      - name: Verify Python tools
+        run: |
+          test "$(python3 -c 'import importlib.metadata as m; print(m.version("awscurl"))')" = "0.44"
+          test "$(python3 -c 'import importlib.metadata as m; print(m.version("tox"))')" = "4.60.0"
+
      - name: Enable buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3

@@ -287,6 +295,7 @@ jobs:

          frontend fe_s3
            bind *:9000
+            option http-buffer-request
            default_backend be_s3

          backend be_s3
@@ -353,6 +362,85 @@ jobs:
          name: s3tests-${{ env.TEST_MODE }}-shard-${{ matrix.shard-index }}
          path: artifacts/**

+  upstream-head-canary:
+    name: Upstream HEAD classification canary
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.12"
+
+      - name: Install collection tool
+        run: |
+          python3 -m pip install --user "tox==4.60.0"
+          python3 - <<'PY'
+          from importlib.metadata import version
+
+          assert version("tox") == "4.60.0"
+          PY
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Compare upstream HEAD classifications
+        id: upstream-compare
+        run: |
+          ARTIFACT_DIR="artifacts/s3tests-upstream-head"
+          UPSTREAM_DIR="${RUNNER_TEMP}/s3-tests-upstream"
+          mkdir -p "${ARTIFACT_DIR}"
+          git clone --depth 1 https://github.com/ceph/s3-tests.git "${UPSTREAM_DIR}"
+          git -C "${UPSTREAM_DIR}" rev-parse HEAD > "${ARTIFACT_DIR}/upstream-sha.txt"
+          cp "${UPSTREAM_DIR}/s3tests.conf.SAMPLE" "${UPSTREAM_DIR}/s3tests.conf"
+          (
+            cd "${UPSTREAM_DIR}"
+            S3TEST_CONF="${UPSTREAM_DIR}/s3tests.conf" tox -- \
+              -q --collect-only s3tests/functional/test_s3.py \
+              -m "not rustfs_never_marker"
+          ) 2>&1 | tee "${ARTIFACT_DIR}/collect.log"
+          grep -E '^s3tests/functional/test_s3\.py::' \
+            "${ARTIFACT_DIR}/collect.log" > "${ARTIFACT_DIR}/collected-nodeids.txt"
+          python3 scripts/s3-tests/report_compat.py \
+            --lists-dir scripts/s3-tests \
+            --collected-nodeids "${ARTIFACT_DIR}/collected-nodeids.txt" \
+            --check-classifications-only 2>&1 | tee "${ARTIFACT_DIR}/classification-drift.txt"
+
+      - name: Publish canary report
+        if: always()
+        env:
+          CANARY_OUTCOME: ${{ steps.upstream-compare.outcome }}
+        run: |
+          {
+            echo "## ceph/s3-tests upstream HEAD canary"
+            echo
+            if [ -f artifacts/s3tests-upstream-head/upstream-sha.txt ]; then
+              echo "Upstream HEAD: $(cat artifacts/s3tests-upstream-head/upstream-sha.txt)"
+            fi
+            echo
+            echo '```text'
+            if [ -s artifacts/s3tests-upstream-head/classification-drift.txt ]; then
+              cat artifacts/s3tests-upstream-head/classification-drift.txt
+            elif [ "${CANARY_OUTCOME}" != "success" ]; then
+              echo "Canary did not complete; inspect the collection log artifact."
+            else
+              echo "No classification drift detected."
+            fi
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload canary artifacts
+        if: always() && env.ACT != 'true'
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: s3tests-upstream-head
+          path: artifacts/s3tests-upstream-head/**
+          retention-days: 14
+
  alert-on-failure:
    name: Alert on scheduled failure
    needs: [s3tests]
@@ -30,7 +30,7 @@ on:
      - "Cargo.lock"
      - ".github/workflows/fuzz.yml"
  schedule:
-    - cron: "0 2 * * *"
+    - cron: "17 2 * * *"
  workflow_dispatch:
    inputs:
      profile:
@@ -121,3 +121,21 @@ jobs:
          cargo nextest run --run-ignored ignored-only --no-tests=fail \
            -p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
            -E "$INTEROP_FILTER"
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [minio-interop]
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -45,13 +45,6 @@
 # docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
 # has fewer cores and reintroduces fleet-state risk for no reliability gain.

-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: mint

 on:
@@ -70,13 +63,13 @@ on:
          - core
          - full
      mint-image:
-        description: "Mint image reference"
+        description: "Mint image reference (empty = pinned default)"
        required: false
-        default: "minio/mint:edge"
+        default: ""
  schedule:
    # Weekly, after the Sunday s3-tests full sweep (starts 02:00 UTC, up to
    # 3h) has finished, so the two never contend for the same runner pool.
-    - cron: "0 6 * * 0"
+    - cron: "41 6 * * 0"

 env:
  S3_ACCESS_KEY: rustfsadmin-ci
@@ -16,7 +16,7 @@ name: Nightly GNU Build

 on:
  schedule:
-    - cron: "0 0 * * *"
+    - cron: "7 0 * * *"
      timezone: "Asia/Shanghai"
  workflow_dispatch:

@@ -39,11 +39,10 @@ jobs:
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
-      - name: Checkout main branch
+      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          persist-credentials: false
-          ref: main

      - name: Setup Rust environment
        uses: ./.github/actions/setup
@@ -89,11 +88,10 @@ jobs:
      # either casing.
      NO_PROXY: 127.0.0.1,localhost
    steps:
-      - name: Checkout main branch
+      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          persist-credentials: false
-          ref: main

      - name: Setup Rust environment
        uses: ./.github/actions/setup
@@ -178,11 +176,10 @@ jobs:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
      NO_PROXY: 127.0.0.1,localhost
    steps:
-      - name: Checkout main branch
+      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          persist-credentials: false
-          ref: main

      - name: Setup Rust environment
        uses: ./.github/actions/setup
@@ -194,3 +191,23 @@ jobs:

      - name: Run HA leader failover live checks (three-node Raft cluster in Docker)
        run: bash scripts/test/vault_ha_kms_live.sh
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [build, kms-vault-lane, kms-vault-ha-failover]
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -22,22 +22,15 @@
 # correctness cost (e.g. the #4221 fsync durability fix) is recorded, not
 # blocked (rustfs/backlog#935 correction 1).

-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: Performance A/B

 on:
  schedule:
-    - cron: "0 6 * * *" # 06:00 UTC nightly, against main
+    - cron: "31 6 * * *" # 06:31 UTC nightly, against main
  workflow_dispatch:
    inputs:
      duration:
-        description: "warp duration per round (short by default to fit the double-build budget)"
+        description: "warp duration per round"
        required: false
        default: "12s"
        type: string
@@ -46,12 +39,8 @@ on:
        required: false
        default: false
        type: boolean
-  push:
-    # Every main commit pre-builds and caches its release binary (perf-3) so the
-    # nightly A/B restores a ready baseline instead of paying the double build.
-    branches: [main]
-
 permissions:
+  actions: read
  contents: read

 env:
@@ -59,83 +48,19 @@ env:
  RUST_BACKTRACE: 1

 jobs:
-  # perf-3: on every push to main, build the release binary once and cache it
-  # keyed by commit SHA (rustfs-baseline-<sha>). The warp-ab measurements
-  # restore this instead of paying the ~32min-per-side source
-  # build. That double build is what pushed the expanded 24-cell nightly past its
-  # ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental
-  # builds off the shared cargo cache keep each push cheap, and building on the
-  # same sm-standard-2 runner the A/B measures on guarantees the cached binary is
-  # ABI-identical. Do NOT source this from build.yml's per-merge artifact: those
-  # are cancelled ~7/8 of the time and are not a reliable baseline.
-  build-baseline-cache:
-    name: Build + cache baseline binary
-    if: github.event_name == 'push'
-    runs-on: sm-standard-2
-    # Latest-wins: consumers only ever restore the binary for the *current*
-    # origin/main tip, so when pushes land faster than the ~65min build, a
-    # superseded build's output is dead weight — cancel it instead of stacking
-    # hour-long jobs on the shared runner pool. A skipped intermediate SHA at
-    # most costs one same-commit self-heal in the A/B job.
-    concurrency:
-      group: perf-baseline-build-main
-      cancel-in-progress: true
-    # #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a
-    # single release build past 60min on this runner — every cache build on
-    # 2026-07-15 died on the old 60min ceiling ("exceeded the maximum execution
-    # time of 1h0m0s") and the cache never populated. The measured binary must
-    # keep the production profile, so the budget absorbs the build instead.
-    timeout-minutes: 100
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      - name: Build release rustfs
-        run: cargo build --release --bin rustfs
-
-      - name: Stage binary for cache
-        run: |
-          set -euo pipefail
-          mkdir -p baseline-bin
-          cp target/release/rustfs baseline-bin/rustfs
-
-      - name: Cache baseline binary by SHA
-        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
-        with:
-          path: baseline-bin/rustfs
-          key: rustfs-baseline-${{ github.sha }}
-
  warp-ab:
    name: Warp A/B budget gate
-    # Always run on schedule / manual dispatch. Never on push — that event only
-    # feeds build-baseline-cache above.
-    if: >-
-      github.event_name == 'schedule' ||
-      github.event_name == 'workflow_dispatch'
    runs-on: sm-standard-2
-    # With perf-3's cached baseline binary the common (cache-hit) nightly is
-    # measurement-only and finishes well under 50min. This ceiling stays
-    # generous only to absorb the same-commit cache-miss self-heal (~65min
-    # single build with the post-#4806 LTO profile + measurement). A timeout
-    # surfaces via the alert-on-failure job (it fires on cancelled/timed-out,
-    # not just failure). perf-6 recalibrates the budget once the noise study
-    # lands.
-    timeout-minutes: 120
+    # A normal nightly restores the last successful binary and builds only the
+    # candidate; daily access keeps that cache warm. A cache miss may build both
+    # and needs room for the A/B run plus artifact and cache publication.
+    timeout-minutes: 180
    steps:
      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          persist-credentials: false
-          fetch-depth: 0 # baseline is built from origin/main
+          fetch-depth: 0 # baseline may be an earlier successful scheduled head

      - name: Setup Rust environment
        uses: ./.github/actions/setup
@@ -163,24 +88,55 @@ jobs:
          fi
          echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"

-      # perf-3: resolve the commits so the cache can be keyed by SHA. The
-      # baseline is origin/main; the candidate is the checked-out ref. On the
-      # nightly (checkout == main) they are the same commit, so one cached binary
-      # serves both phases and the run does zero source builds.
+      # A failed regression run must keep comparing against the last known-good
+      # scheduled head. Otherwise the next nightly would absorb the regression
+      # into its baseline and turn green without a fix.
+      - name: Find last successful scheduled baseline
+        id: scheduled_baseline
+        if: github.event_name == 'schedule'
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
+        with:
+          result-encoding: string
+          script: |
+            const { data } = await github.rest.actions.listWorkflowRuns({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: "performance-ab.yml",
+              event: "schedule",
+              status: "success",
+              per_page: 1,
+            });
+            return data.workflow_runs[0]?.head_sha ?? "";
+
+      # Manual runs compare a selected ref with current main. Scheduled runs
+      # compare current main with the last successful scheduled head. With no
+      # history, the first run measures the candidate against itself and seeds
+      # that head only if the complete rig succeeds.
      - name: Resolve baseline / candidate commits
        id: commits
+        env:
+          SCHEDULED_BASELINE_SHA: ${{ steps.scheduled_baseline.outputs.result }}
        run: |
          set -euo pipefail
-          baseline_sha="$(git rev-parse origin/main)"
          candidate_sha="$(git rev-parse HEAD)"
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}"
+          else
+            baseline_sha="$(git rev-parse origin/main)"
+          fi
+          git cat-file -e "${baseline_sha}^{commit}"
+          if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
+            echo "::error::baseline $baseline_sha is not an ancestor of candidate $candidate_sha; update the selected ref before comparing" >&2
+            exit 1
+          fi
          echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
          echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
          echo "baseline commit: $baseline_sha"
          echo "candidate commit: $candidate_sha"

-      # Exact-key restore of the baseline binary built by build-baseline-cache
-      # when origin/main last landed. A miss (binary evicted or not built yet)
-      # leaves cache-hit unset and the rig falls back to a source build.
+      # Exact-key restore of the candidate binary saved by its successful
+      # scheduled run. A miss leaves cache-hit unset and falls back to a source
+      # build of that known-good head.
      - name: Restore cached baseline binary
        id: baseline_cache
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
@@ -270,11 +226,11 @@ jobs:
            elif [[ "$selfheal_built" == "true" ]]; then
              base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
            else
-              base_src="isolated origin/main source build (saved as rustfs-baseline-$baseline_sha)"
+              base_src="isolated baseline source build (saved as rustfs-baseline-$baseline_sha)"
            fi
            if [[ "$candidate_sha" == "$baseline_sha" ]]; then
-              # Nightly on main: the candidate is the same commit as the baseline,
-              # so reuse the one binary for both phases and skip all builds.
+              # No commits landed since the last successful baseline, so reuse
+              # the one binary for both phases and measure only rig drift.
              args+=(--candidate-bin "$base_bin")
              cand_src="same binary as baseline (same commit)"
            elif [[ "$candidate_built" == "true" ]]; then
@@ -362,6 +318,23 @@ jobs:
            fi
          } >> "$GITHUB_STEP_SUMMARY"

+      - name: Stage successful candidate baseline
+        if: >-
+          steps.ab.outputs.status == '0' &&
+          steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
+        run: |
+          set -euo pipefail
+          cp candidate-bin/rustfs baseline-bin/rustfs
+
+      - name: Cache successful candidate baseline
+        if: >-
+          steps.ab.outputs.status == '0' &&
+          steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: baseline-bin/rustfs
+          key: rustfs-baseline-${{ steps.commits.outputs.candidate_sha }}
+
      # Scheduled failure alerting is handled by the alert-on-failure job below
      # (perf-2 consuming ci-8's schedule-failure-issue composite action).

@@ -369,6 +342,10 @@ jobs:
        if: always()
        run: |
          status="${{ steps.ab.outputs.status }}"
+          if [[ -z "$status" ]]; then
+            echo "::error::warp A/B setup failed before the rig ran. Check the first failed workflow step." >&2
+            exit 1
+          fi
          if [[ "$status" != "0" ]]; then
            echo "::error::warp A/B budget gate failed (exit $status). See the step summary / gate.md artifact." >&2
            exit "$status"
@@ -30,7 +30,7 @@ name: Runner Hygiene

 on:
  schedule:
-    - cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron)
+    - cron: "37 6 1 * *" # Monthly, 1st at 06:37 UTC
  workflow_dispatch:

 permissions:
@@ -0,0 +1,57 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Scheduled Validation Freshness
+
+on:
+  schedule:
+    - cron: "47 23 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: scheduled-validation-freshness
+  cancel-in-progress: false
+
+jobs:
+  check-freshness:
+    name: Check scheduled validation freshness
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Check latest scheduled runs
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set +e
+          python3 scripts/check_scheduled_validation_freshness.py \
+            --report "${RUNNER_TEMP}/scheduled-validation-freshness.md"
+          status=$?
+          cat "${RUNNER_TEMP}/scheduled-validation-freshness.md" >> "${GITHUB_STEP_SUMMARY}"
+          exit "${status}"
+      - name: Open or update freshness issue
+        if: failure()
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          details-file: ${{ runner.temp }}/scheduled-validation-freshness.md
@@ -0,0 +1,63 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Scheduled Validation Watchdog
+
+on:
+  workflow_run:
+    workflows:
+      - "Security Audit"
+      - "Build and Release"
+      - "Continuous Integration"
+      - "coverage"
+      - "e2e-nightly"
+      - "e2e-s3tests"
+      - "Fuzz"
+      - "mint"
+      - "minio-interop"
+      - "Nightly GNU Build"
+      - "Performance A/B"
+      - "Runner Hygiene"
+    types: [completed]
+
+permissions:
+  contents: read
+
+jobs:
+  alert-on-incomplete-run:
+    name: Alert on incomplete scheduled run
+    if: >-
+      github.event.workflow_run.event == 'schedule' &&
+      github.event.workflow_run.conclusion != 'success' &&
+      github.event.workflow_run.conclusion != 'failure'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update incomplete-run issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          workflow-name: ${{ github.event.workflow_run.name }}
+          source-run-id: ${{ github.event.workflow_run.id }}
+          source-run-attempt: ${{ github.event.workflow_run.run_attempt }}
+          source-event: ${{ github.event.workflow_run.event }}
+          source-ref-name: ${{ github.event.workflow_run.head_branch }}
+          source-sha: ${{ github.event.workflow_run.head_sha }}
@@ -30,7 +30,8 @@ make build-docker BUILD_OS=ubuntu22.04
 - Crate membership: `Cargo.toml` `[workspace].members`
 - Architecture, layering, crate map: [ARCHITECTURE.md](ARCHITECTURE.md)
 - Migration guardrails & readiness contracts: [docs/architecture/](docs/architecture/README.md)
- CI gates: `.github/workflows/ci.yml` (source of truth; never copy its steps into docs)
+- CI workflow steps: `.github/workflows/`; event, timeout, and required-status
+  matrix: [docs/testing/ci-gates.md](docs/testing/ci-gates.md)
 - Test-layer taxonomy, per-layer entry commands, serial/nextest rules, flake
  policy: [docs/testing/README.md](docs/testing/README.md)
 - Tier/ILM transition debugging (xl.meta inspection, versionId tracing):
@@ -70,6 +70,8 @@ make pre-pr

 > For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md).

+> For the event, timeout, required-status, and local reproduction matrix, see [docs/testing/ci-gates.md](docs/testing/ci-gates.md).
+
 ### 🔒 Automated Pre-commit Hooks
 #### What `make pre-commit` and `make pre-pr` actually run

@@ -68,16 +68,16 @@ dependencies = [

 [[package]]
 name = "aes-gcm"
-version = "0.11.0"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028"
+checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f"
 dependencies = [
 "aead",
 "aes 0.9.2",
 "cipher 0.5.2",
 "ctr",
+ "ctutils",
 "ghash",
- "subtle",
 "zeroize",
 ]

@@ -333,6 +333,12 @@ dependencies = [
 "password-hash",
 ]

+[[package]]
+name = "array-init"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc"
+
 [[package]]
 name = "arrayvec"
 version = "0.7.8"
@@ -813,11 +819,12 @@ dependencies = [

 [[package]]
 name = "async_zip"
-version = "0.0.18"
+version = "0.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d8c50d65ce1b0e0cb65a785ff615f78860d7754290647d3b983208daa4f85e6"
+checksum = "fb7f5f40e1eb30949a266fc900d37fd3267c7baf50c3705ac09c5d8ced5def63"
 dependencies = [
 "async-compression",
+ "binrw",
 "crc32fast",
 "futures-lite",
 "pin-project",
@@ -869,9 +876,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"

 [[package]]
 name = "aws-config"
-version = "1.10.1"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b180a3c8b55960db3426d8964b8745e652466a1a49fe1a2eda828046d30b5e4"
+checksum = "a767267da9e2c2e189b2f9df8b5657e850ecf5352644734ba130d4a57095cf1b"
 dependencies = [
 "aws-credential-types",
 "aws-runtime",
@@ -964,9 +971,9 @@ dependencies = [

 [[package]]
 name = "aws-sdk-kms"
-version = "1.115.0"
+version = "1.116.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5b034f8b7ceadb873d0bc607c30bb4b0be68e09a84c837174e7c2c6878ff882"
+checksum = "484ecdbea2a1cfc0e6eea69ce0a665f93913671b303ba40b2361b1d826544e7e"
 dependencies = [
 "arc-swap",
 "aws-credential-types",
@@ -990,9 +997,9 @@ dependencies = [

 [[package]]
 name = "aws-sdk-s3"
-version = "1.142.0"
+version = "1.143.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9e15a5c55e05f4b0b7e483160b3c85cccdf77cff02c95504f3e71d460855cd2"
+checksum = "a0ade5433c9561daac7c0c6bc910f1240b4f8ec0d6148b0b463aac0691d747c9"
 dependencies = [
 "arc-swap",
 "aws-credential-types",
@@ -1027,9 +1034,9 @@ dependencies = [

 [[package]]
 name = "aws-sdk-sso"
-version = "1.106.0"
+version = "1.107.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d0efcee834347b6705eca3eea2defd88242f43774f55d7326604222e3c86260"
+checksum = "769b0abd0f89cfe11da5099986dd493e4f94347ce9a4562cb86ddecfe926b6c0"
 dependencies = [
 "arc-swap",
 "aws-credential-types",
@@ -1053,9 +1060,9 @@ dependencies = [

 [[package]]
 name = "aws-sdk-ssooidc"
-version = "1.108.0"
+version = "1.109.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a59312a04cf19c962cfee32b64ecfee758f8786407ff6da5b30fff46ae96f201"
+checksum = "f4075b8a2c8cda4076a3dcc43b9d6dabd93e0c2502abeaaf7e14aaead9bb312b"
 dependencies = [
 "arc-swap",
 "aws-credential-types",
@@ -1079,9 +1086,9 @@ dependencies = [

 [[package]]
 name = "aws-sdk-sts"
-version = "1.111.0"
+version = "1.112.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "120e7eb63457a9e547f9986fe3b273f77c43679da4d04f46359fa881c5e19b6e"
+checksum = "3f582002918346a3e685be1b391c7bea155073088cea6bd4e4b7663df9e43b6c"
 dependencies = [
 "arc-swap",
 "aws-credential-types",
@@ -1544,6 +1551,30 @@ dependencies = [
 "serde",
 ]

+[[package]]
+name = "binrw"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ad120d555272286c1017d25165ab8bd74806f13fc85b258484ec7e4ce75458f"
+dependencies = [
+ "array-init",
+ "binrw_derive",
+ "bytemuck",
+]
+
+[[package]]
+name = "binrw_derive"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6df92e0e9baae4dc82c7bad7715ca40c0a5c71539057bf2ea04a5c29c980410b"
+dependencies = [
+ "either",
+ "owo-colors",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
 [[package]]
 name = "bit-set"
 version = "0.8.0"
@@ -1631,9 +1662,9 @@ dependencies = [

 [[package]]
 name = "blocking"
-version = "1.6.2"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21"
+checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa"
 dependencies = [
 "async-channel",
 "async-task",
@@ -1722,6 +1753,12 @@ version = "3.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"

+[[package]]
+name = "bytemuck"
+version = "1.25.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
+
 [[package]]
 name = "byteorder"
 version = "1.5.0"
@@ -1858,9 +1895,9 @@ dependencies = [

 [[package]]
 name = "cc"
-version = "1.4.3"
+version = "1.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d"
+checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273"
 dependencies = [
 "find-msvc-tools",
 "jobserver",
@@ -2325,9 +2362,9 @@ dependencies = [

 [[package]]
 name = "crc32fast"
-version = "1.5.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
+checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550"
 dependencies = [
 "cfg-if",
 ]
@@ -2522,12 +2559,6 @@ dependencies = [
 "subtle",
 ]

-[[package]]
-name = "cty"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
-
 [[package]]
 name = "curve25519-dalek"
 version = "4.1.3"
@@ -3940,9 +3971,9 @@ dependencies = [

 [[package]]
 name = "either"
-version = "1.17.0"
+version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d"
+checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"

 [[package]]
 name = "elliptic-curve"
@@ -4476,6 +4507,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5"
 dependencies = [
 "polyval",
+ "zeroize",
 ]

 [[package]]
@@ -5058,9 +5090,9 @@ dependencies = [

 [[package]]
 name = "hotpath"
-version = "0.23.3"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dce755d457a63bdd0c95e4c91511daad1b58b33209543b7f38027b676f387e5e"
+checksum = "e2645642a23d4061ec15a4a6e74f851a3145c3125356846cfc7772ff9c6f2737"
 dependencies = [
 "arc-swap",
 "async-channel",
@@ -5092,9 +5124,9 @@ dependencies = [

 [[package]]
 name = "hotpath-macros"
-version = "0.23.3"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a903af89a8429cb07790c3818bc15270b394f80af1bc254e5ccf9c7de2961770"
+checksum = "89a3d3cdf9b0d4d3d4f6d4a29798f3b9170401ba500eaa58dd8f890f926af0f1"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5103,15 +5135,15 @@ dependencies = [

 [[package]]
 name = "hotpath-macros-meta"
-version = "0.23.3"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bcc0ab94ffbb2ee77f4a897df02b5a137a10cf24d69bda936e59aff4dd456e61"
+checksum = "e84cd2417fa60938241cf1cd6c03e09953f5c821122dc5da9b8f27975d136c5b"

 [[package]]
 name = "hotpath-meta"
-version = "0.23.3"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "053481f6cec8f775a3276c7f6e2f21123111d28261e4edc15ea7421c445964bb"
+checksum = "4d2c145b67b1a4e7bcefa918995e212c30a49a85f05cc5962fe1f717878d560b"
 dependencies = [
 "hotpath-macros-meta",
 ]
@@ -5378,9 +5410,9 @@ checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"

 [[package]]
 name = "icu_provider"
-version = "2.3.0"
+version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428"
+checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73"
 dependencies = [
 "displaydoc",
 "icu_locale_core",
@@ -5767,9 +5799,9 @@ dependencies = [

 [[package]]
 name = "keccak"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffd9697dc4a9a62e2da93389f34400b77a28f0287711263cabb203b3ccb9c0e4"
+checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598"
 dependencies = [
 "cfg-if",
 "cpufeatures 0.3.0",
@@ -5988,15 +6020,6 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"

-[[package]]
-name = "libmimalloc-sys"
-version = "0.1.49"
-source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
-dependencies = [
- "cc",
- "cty",
-]
-
 [[package]]
 name = "libredox"
 version = "0.1.20"
@@ -6090,9 +6113,9 @@ dependencies = [

 [[package]]
 name = "log"
-version = "0.4.33"
+version = "0.4.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
+checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"

 [[package]]
 name = "lru"
@@ -6282,7 +6305,7 @@ dependencies = [
 "hashbrown 0.16.1",
 "indexmap 2.14.0",
 "metrics",
- "ordered-float 5.3.0",
+ "ordered-float 5.5.0",
 "quanta",
 "radix_trie",
 "rand 0.9.5",
@@ -6397,14 +6420,6 @@ dependencies = [
 "synstructure 0.13.2",
 ]

-[[package]]
-name = "mimalloc"
-version = "0.1.52"
-source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11#6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11"
-dependencies = [
- "libmimalloc-sys",
-]
-
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -7238,9 +7253,9 @@ dependencies = [

 [[package]]
 name = "ordered-float"
-version = "5.3.0"
+version = "5.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e"
+checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0"
 dependencies = [
 "num-traits",
 ]
@@ -7275,6 +7290,12 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"

+[[package]]
+name = "owo-colors"
+version = "4.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d"
+
 [[package]]
 name = "p12-keystore"
 version = "0.2.1"
@@ -7817,6 +7838,7 @@ dependencies = [
 "cpubits",
 "cpufeatures 0.3.0",
 "universal-hash",
+ "zeroize",
 ]

 [[package]]
@@ -8287,6 +8309,16 @@ name = "quick-xml"
 version = "0.41.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
+[[package]]
+name = "quick-xml"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b"
 dependencies = [
 "encoding_rs",
 "memchr",
@@ -8951,9 +8983,9 @@ dependencies = [

 [[package]]
 name = "russh"
-version = "0.62.7"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9decb68e4e44e1079700e54f17c8f23806ec53d7e0db73ab1c71d9dabc666812"
+checksum = "00cf00190c315093734a8d405225bd8773a219bc86538a9b73bfc51145b33995"
 dependencies = [
 "aes 0.9.2",
 "aws-lc-rs",
@@ -9162,13 +9194,11 @@ dependencies = [
 "insta",
 "jiff",
 "libc",
- "libmimalloc-sys",
 "libsystemd",
 "matchit 0.9.2",
 "md-5 0.11.0",
 "metrics",
 "metrics-util",
- "mimalloc",
 "mime_guess",
 "opentelemetry",
 "opentelemetry_sdk",
@@ -9177,7 +9207,7 @@ dependencies = [
 "percent-encoding",
 "pin-project-lite",
 "proptest",
- "quick-xml",
+ "quick-xml 0.42.0",
 "rand 0.10.2",
 "rcgen",
 "regex",
@@ -9204,6 +9234,8 @@ dependencies = [
 "rustfs-lock",
 "rustfs-log-analyzer",
 "rustfs-madmin",
+ "rustfs-mimalloc",
+ "rustfs-mimalloc-sys",
 "rustfs-notify",
 "rustfs-object-capacity",
 "rustfs-object-data-cache",
@@ -9441,7 +9473,7 @@ dependencies = [
 "path-absolutize",
 "pin-project-lite",
 "proptest",
- "quick-xml",
+ "quick-xml 0.42.0",
 "rand 0.10.2",
 "ratelimit",
 "rcgen",
@@ -9587,6 +9619,7 @@ dependencies = [
 "serde",
 "serde_json",
 "serial_test",
+ "sha2 0.11.0",
 "temp-env",
 "tempfile",
 "thiserror 2.0.20",
@@ -9875,6 +9908,24 @@ dependencies = [
 "tokio",
 ]

+[[package]]
+name = "rustfs-mimalloc"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a406f4aa07084301d485beec873af6dccc8e3f8762da244743df92038b1db1a6"
+dependencies = [
+ "rustfs-mimalloc-sys",
+]
+
+[[package]]
+name = "rustfs-mimalloc-sys"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3051b819175f58445d4c369a72f0ab88149f3885ba8bea2aff3be01f53fe7cd"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "rustfs-notify"
 version = "1.0.0-rc.3"
@@ -9889,7 +9940,7 @@ dependencies = [
 "jiff",
 "metrics",
 "percent-encoding",
- "quick-xml",
+ "quick-xml 0.42.0",
 "rayon",
 "rustc-hash",
 "rustfs-config",
@@ -10662,9 +10713,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"

 [[package]]
 name = "rustls-webpki"
-version = "0.103.14"
+version = "0.103.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a"
+checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
 dependencies = [
 "aws-lc-rs",
 "ring",
@@ -10708,7 +10759,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 [[package]]
 name = "s3s"
 version = "0.15.0-alpha.1"
-source = "git+https://github.com/rustfs/s3s.git?rev=ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a#ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a"
+source = "git+https://github.com/rustfs/s3s.git?rev=e080e38c56a3b43acbacce55710d765a5ce9003d#e080e38c56a3b43acbacce55710d765a5ce9003d"
 dependencies = [
 "arc-swap",
 "arrayvec",
@@ -10735,7 +10786,7 @@ dependencies = [
 "nom 8.0.0",
 "numeric_cast",
 "pin-project-lite",
- "quick-xml",
+ "quick-xml 0.41.0",
 "regex",
 "serde",
 "serde_json",
@@ -12680,14 +12731,15 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"

 [[package]]
 name = "uuid"
-version = "1.24.1"
+version = "1.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9"
+checksum = "f053576934f05a761a402421fbbe3d425d9366f75f978806a037b3ca481abecc"
 dependencies = [
 "getrandom 0.4.3",
 "js-sys",
 "rand 0.10.2",
 "serde_core",
+ "sha1_smol",
 "wasm-bindgen",
 ]

@@ -13401,9 +13453,9 @@ dependencies = [

 [[package]]
 name = "zerovec-derive"
-version = "0.11.5"
+version = "0.11.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9"
+checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -135,7 +135,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.3" }

 # Async Runtime and Networking
 async-channel = "2.5.0"
-async_zip = { default-features = false, version = "0.0.18" }
+async_zip = { default-features = false, version = "0.0.19" }
 mysql_async = { default-features = false, version = "0.37" }
 async-compression = { version = "0.4.43" }
 async-recursion = "1.1.1"
@@ -178,7 +178,7 @@ byteorder = "1.5.0"
 flatbuffers = "25.12.19"
 form_urlencoded = "1.2.2"
 prost = "0.14.4"
-quick-xml = "0.41.0"
+quick-xml = "0.42.0"
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.229" }
@@ -191,7 +191,7 @@ serde_urlencoded = "0.7.1"
 # matching stable releases are not available yet, while previous stable lines
 # have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
 # releases.
-aes-gcm = { version = "=0.11.0" }
+aes-gcm = { version = "=0.11.1" }
 argon2 = { version = "=0.6.0-rc.8" }
 blake2 = "=0.11.0-rc.6"
 chacha20poly1305 = { version = "=0.11.0" }
@@ -227,11 +227,11 @@ arc-swap = "1.9.2"
 astral-tokio-tar = "0.6.4"
 atoi = "3.1.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.10.1" }
+aws-config = { version = "1.11.0" }
 aws-credential-types = { version = "1.3.0" }
-aws-sdk-kms = { default-features = false, version = "1.115.0" }
-aws-sdk-s3 = { default-features = false, version = "1.142.0" }
-aws-sdk-sts = { default-features = false, version = "1.111.0" }
+aws-sdk-kms = { default-features = false, version = "1.116.0" }
+aws-sdk-s3 = { default-features = false, version = "1.143.0" }
+aws-sdk-sts = { default-features = false, version = "1.112.0" }
 aws-smithy-http-client = { default-features = false, version = "1.4.0" }
 aws-smithy-runtime-api = { version = "1.15.0" }
 aws-smithy-types = { version = "1.6.2" }
@@ -291,7 +291,7 @@ rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.4" }
 rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
-s3s = { git = "https://github.com/rustfs/s3s.git", rev = "ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a" }
+s3s = { git = "https://github.com/rustfs/s3s.git", rev = "e080e38c56a3b43acbacce55710d765a5ce9003d" }
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.3"
@@ -314,7 +314,7 @@ tracing-subscriber = { version = "0.3.23" }
 transform-stream = "0.3.1"
 url = "2.5.8"
 urlencoding = "2.1.3"
-uuid = { version = "1.24.1" }
+uuid = { version = "1.25.0" }
 vaultrs = { version = "0.8.0" }
 tar = "0.4.46"
 walkdir = "2.5.0"
@@ -343,16 +343,16 @@ libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "10.0.2" }
 rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.62.7" }
+russh = { version = "0.63.0" }
 russh-sftp = "2.4.0"

 # WebDAV
 dav-server = "0.11.0"

 # Performance Analysis and Memory Profiling
-mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11" }
-libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "6d4c41bb10c6d9da1d1b6f07b38c4cc051667f11", features = ["extended"] }
-hotpath = { version = "0.23.3", default-features = false }
+rustfs-mimalloc = { version = "0.5.0" }
+rustfs-mimalloc-sys = { version = "0.5.0" }
+hotpath = { version = "0.24.0", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }

@@ -901,6 +901,10 @@ pub struct Metrics {
    scanner_cycle_max_duration_millis: AtomicU64,
    scanner_cycle_max_objects: AtomicU64,
    scanner_cycle_max_directories: AtomicU64,
+    scanner_cycle_timeout_total: AtomicU64,
+    scanner_cycle_recovery_required_total: AtomicU64,
+    scanner_cycle_last_progress_age_seconds: AtomicU64,
+    scanner_leader_lease_without_progress: AtomicBool,
    scanner_bitrot_cycle_enabled: AtomicBool,
    scanner_bitrot_cycle_millis: AtomicU64,
    scanner_checkpoint: Mutex<Option<ScannerCheckpointReport>>,
@@ -914,7 +918,15 @@ pub struct Metrics {
    scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64,
    scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64,
    scanner_usage_last_save_unix_secs: AtomicU64,
+    scanner_usage_last_durable_success_unix_secs: AtomicU64,
+    scanner_usage_last_publication_unix_secs: AtomicU64,
+    scanner_usage_last_publication_state: Mutex<String>,
+    scanner_usage_last_publication_reason: Mutex<String>,
    scanner_usage_last_save_result: AtomicU8,
+    scanner_usage_deferred_pending: AtomicBool,
+    scanner_usage_deferred_total: AtomicU64,
+    scanner_usage_last_deferred_unix_secs: AtomicU64,
+    scanner_usage_last_deferred_reason: Mutex<String>,
    scanner_source_work: Vec<ScannerSourceWorkCounters>,
    current_scan_cycle_source_work_start: Vec<ScannerSourceWorkCounters>,
    last_scan_cycle_source_work: Vec<ScannerSourceWorkCounters>,
@@ -1212,6 +1224,22 @@ pub struct ScannerUsageFreshnessSnapshot {
    pub last_usage_save_unix_secs: u64,
    pub last_usage_save_result: String,
    pub last_usage_save_result_code: u64,
+    #[serde(default)]
+    pub last_durable_success_unix_secs: u64,
+    #[serde(default)]
+    pub last_publication_unix_secs: u64,
+    #[serde(default)]
+    pub last_publication_state: String,
+    #[serde(default)]
+    pub last_publication_reason: String,
+    #[serde(default)]
+    pub deferred_pending: bool,
+    #[serde(default)]
+    pub deferred_total: u64,
+    #[serde(default)]
+    pub last_deferred_unix_secs: u64,
+    #[serde(default)]
+    pub last_deferred_reason: String,
 }

 #[derive(Clone, Debug, Default, Serialize, Deserialize)]
@@ -1370,6 +1398,14 @@ pub struct ScannerMetricsReport {
    #[serde(default)]
    pub cycle_max_directories: u64,
    #[serde(default)]
+    pub cycle_timeout_total: u64,
+    #[serde(default)]
+    pub cycle_recovery_required_total: u64,
+    #[serde(default)]
+    pub cycle_last_progress_age: u64,
+    #[serde(default)]
+    pub leader_lease_without_progress: bool,
+    #[serde(default)]
    pub bitrot_cycle_enabled: bool,
    #[serde(default)]
    pub bitrot_cycle_seconds: f64,
@@ -1430,6 +1466,9 @@ const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total
 const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
 const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
 const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
+const OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL: &str = "rustfs_scanner_cycle_timeout_total";
+const OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE: &str = "rustfs_scanner_cycle_last_progress_age";
+const OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS: &str = "rustfs_scanner_leader_lease_without_progress";

 fn scan_cycle_result_label(result: u8) -> &'static str {
    match result {
@@ -1913,6 +1952,10 @@ impl Metrics {
            scanner_cycle_max_duration_millis: AtomicU64::new(0),
            scanner_cycle_max_objects: AtomicU64::new(0),
            scanner_cycle_max_directories: AtomicU64::new(0),
+            scanner_cycle_timeout_total: AtomicU64::new(0),
+            scanner_cycle_recovery_required_total: AtomicU64::new(0),
+            scanner_cycle_last_progress_age_seconds: AtomicU64::new(0),
+            scanner_leader_lease_without_progress: AtomicBool::new(false),
            scanner_bitrot_cycle_enabled: AtomicBool::new(false),
            scanner_bitrot_cycle_millis: AtomicU64::new(0),
            scanner_checkpoint: Mutex::new(None),
@@ -1926,7 +1969,15 @@ impl Metrics {
            scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64::new(0),
            scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64::new(0),
            scanner_usage_last_save_unix_secs: AtomicU64::new(0),
+            scanner_usage_last_durable_success_unix_secs: AtomicU64::new(0),
+            scanner_usage_last_publication_unix_secs: AtomicU64::new(0),
+            scanner_usage_last_publication_state: Mutex::new(String::new()),
+            scanner_usage_last_publication_reason: Mutex::new(String::new()),
            scanner_usage_last_save_result: AtomicU8::new(ScannerUsageSaveResult::Unknown as u8),
+            scanner_usage_deferred_pending: AtomicBool::new(false),
+            scanner_usage_deferred_total: AtomicU64::new(0),
+            scanner_usage_last_deferred_unix_secs: AtomicU64::new(0),
+            scanner_usage_last_deferred_reason: Mutex::new(String::new()),
            scanner_source_work: ScannerWorkSource::all()
                .iter()
                .map(|_| ScannerSourceWorkCounters::default())
@@ -2251,6 +2302,44 @@ impl Metrics {
            .store(unix_now_secs(), Ordering::Relaxed);
    }

+    /// Record an intentional retryable usage publication deferral separately
+    /// from the last durable save result.
+    pub fn record_scanner_usage_deferred(&self, reason: impl Into<String>) {
+        let reason = reason.into();
+        self.record_scanner_usage_publication("deferred", reason.clone());
+        self.scanner_usage_deferred_pending.store(true, Ordering::Release);
+        self.scanner_usage_deferred_total.fetch_add(1, Ordering::Relaxed);
+        self.scanner_usage_last_deferred_unix_secs
+            .store(unix_now_secs(), Ordering::Relaxed);
+        let mut last_reason = match self.scanner_usage_last_deferred_reason.lock() {
+            Ok(guard) => guard,
+            Err(poisoned) => poisoned.into_inner(),
+        };
+        *last_reason = reason;
+    }
+
+    pub fn record_scanner_usage_durable_success(&self) {
+        self.record_scanner_usage_publication("success", "");
+        self.scanner_usage_last_durable_success_unix_secs
+            .store(unix_now_secs(), Ordering::Relaxed);
+        self.scanner_usage_deferred_pending.store(false, Ordering::Release);
+    }
+
+    pub fn record_scanner_usage_publication(&self, state: &str, reason: impl Into<String>) {
+        self.scanner_usage_last_publication_unix_secs
+            .store(unix_now_secs(), Ordering::Relaxed);
+        let mut publication_state = match self.scanner_usage_last_publication_state.lock() {
+            Ok(guard) => guard,
+            Err(poisoned) => poisoned.into_inner(),
+        };
+        *publication_state = state.to_string();
+        let mut publication_reason = match self.scanner_usage_last_publication_reason.lock() {
+            Ok(guard) => guard,
+            Err(poisoned) => poisoned.into_inner(),
+        };
+        *publication_reason = reason.into();
+    }
+
    pub fn record_scanner_source_work(&self, source: ScannerWorkSource, work: ScannerSourceWorkUpdate) {
        if let Some(counters) = self.scanner_source_work.get(source.index()) {
            counters.add(work);
@@ -2412,12 +2501,29 @@ impl Metrics {
            .store(cycle_max_objects.unwrap_or_default(), Ordering::Relaxed);
        self.scanner_cycle_max_directories
            .store(cycle_max_directories.unwrap_or_default(), Ordering::Relaxed);
+        self.scanner_leader_lease_without_progress.store(false, Ordering::Relaxed);
+        self.scanner_cycle_last_progress_age_seconds.store(0, Ordering::Relaxed);
+        metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(0.0);
+        metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(0.0);
        self.scanner_bitrot_cycle_enabled
            .store(bitrot_cycle.is_some(), Ordering::Relaxed);
        self.scanner_bitrot_cycle_millis
            .store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
    }

+    pub fn record_scanner_cycle_timeout(&self, recovery_required: bool, progress_age: Duration) {
+        self.scanner_cycle_timeout_total.fetch_add(1, Ordering::Relaxed);
+        if recovery_required {
+            self.scanner_cycle_recovery_required_total.fetch_add(1, Ordering::Relaxed);
+        }
+        self.scanner_cycle_last_progress_age_seconds
+            .store(progress_age.as_secs(), Ordering::Relaxed);
+        self.scanner_leader_lease_without_progress.store(true, Ordering::Relaxed);
+        metrics::counter!(OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL).increment(1);
+        metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(progress_age.as_secs_f64());
+        metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(1.0);
+    }
+
    pub fn record_scanner_set_scan_state(&self, concurrency_limit: Option<usize>, queued: Option<usize>, active: Option<usize>) {
        if let Some(concurrency_limit) = concurrency_limit {
            self.scanner_set_scan_concurrency_limit
@@ -3256,6 +3362,23 @@ impl Metrics {
            last_usage_save_unix_secs: self.scanner_usage_last_save_unix_secs.load(Ordering::Relaxed),
            last_usage_save_result: usage_save_result.as_str().to_string(),
            last_usage_save_result_code: usage_save_result as u8 as u64,
+            last_durable_success_unix_secs: self.scanner_usage_last_durable_success_unix_secs.load(Ordering::Relaxed),
+            last_publication_unix_secs: self.scanner_usage_last_publication_unix_secs.load(Ordering::Relaxed),
+            last_publication_state: match self.scanner_usage_last_publication_state.lock() {
+                Ok(state) => state.clone(),
+                Err(poisoned) => poisoned.into_inner().clone(),
+            },
+            last_publication_reason: match self.scanner_usage_last_publication_reason.lock() {
+                Ok(reason) => reason.clone(),
+                Err(poisoned) => poisoned.into_inner().clone(),
+            },
+            deferred_pending: self.scanner_usage_deferred_pending.load(Ordering::Acquire),
+            deferred_total: self.scanner_usage_deferred_total.load(Ordering::Relaxed),
+            last_deferred_unix_secs: self.scanner_usage_last_deferred_unix_secs.load(Ordering::Relaxed),
+            last_deferred_reason: match self.scanner_usage_last_deferred_reason.lock() {
+                Ok(reason) => reason.clone(),
+                Err(poisoned) => poisoned.into_inner().clone(),
+            },
        };
        m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed);
        m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0;
@@ -3265,6 +3388,10 @@ impl Metrics {
        m.cycle_max_duration_seconds = self.scanner_cycle_max_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
        m.cycle_max_objects = self.scanner_cycle_max_objects.load(Ordering::Relaxed);
        m.cycle_max_directories = self.scanner_cycle_max_directories.load(Ordering::Relaxed);
+        m.cycle_timeout_total = self.scanner_cycle_timeout_total.load(Ordering::Relaxed);
+        m.cycle_recovery_required_total = self.scanner_cycle_recovery_required_total.load(Ordering::Relaxed);
+        m.cycle_last_progress_age = self.scanner_cycle_last_progress_age_seconds.load(Ordering::Relaxed);
+        m.leader_lease_without_progress = self.scanner_leader_lease_without_progress.load(Ordering::Relaxed);
        m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed);
        m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0;
        m.scan_checkpoint = match self.scanner_checkpoint.lock() {
@@ -4623,6 +4750,7 @@ mod tests {
        metrics.record_scanner_dirty_usage_cycle_snapshot(1);
        metrics.record_scanner_dirty_usage_cycle_clear(1, 1);
        metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
+        metrics.record_scanner_usage_deferred("data_movement");

        let report = metrics.report().await;

@@ -4634,6 +4762,22 @@ mod tests {
        assert!(report.usage_freshness.last_usage_save_unix_secs > 0);
        assert_eq!(report.usage_freshness.last_usage_save_result, "success");
        assert_eq!(report.usage_freshness.last_usage_save_result_code, 1);
+        assert!(report.usage_freshness.deferred_pending);
+        assert_eq!(report.usage_freshness.deferred_total, 1);
+        assert!(report.usage_freshness.last_deferred_unix_secs > 0);
+        assert_eq!(report.usage_freshness.last_deferred_reason, "data_movement");
+
+        metrics.record_scanner_usage_durable_success();
+        let report = metrics.report().await;
+        assert!(!report.usage_freshness.deferred_pending);
+        assert_eq!(report.usage_freshness.deferred_total, 1);
+        assert!(report.usage_freshness.last_durable_success_unix_secs > 0);
+        assert_eq!(report.usage_freshness.last_publication_state, "success");
+
+        metrics.record_scanner_usage_publication("no_update", "no_update");
+        let report = metrics.report().await;
+        assert_eq!(report.usage_freshness.last_publication_state, "no_update");
+        assert_eq!(report.usage_freshness.last_publication_reason, "no_update");
    }

    #[tokio::test]
@@ -4926,4 +5070,20 @@ mod tests {
        assert!(!report.bitrot_cycle_enabled);
        assert_eq!(report.bitrot_cycle_seconds, 0.0);
    }
+
+    #[tokio::test]
+    async fn scanner_cycle_timeout_metrics_reset_for_a_new_cycle() {
+        let metrics = Metrics::new();
+        metrics.record_scanner_cycle_timeout(true, Duration::from_secs(17));
+        let timed_out = metrics.report().await;
+        assert_eq!(timed_out.cycle_timeout_total, 1);
+        assert_eq!(timed_out.cycle_last_progress_age, 17);
+        assert!(timed_out.leader_lease_without_progress);
+
+        metrics.record_scanner_cycle_config(Duration::from_secs(60), None, Some(Duration::from_secs(1)), None, None);
+        let current = metrics.report().await;
+        assert_eq!(current.cycle_timeout_total, 1);
+        assert_eq!(current.cycle_last_progress_age, 0);
+        assert!(!current.leader_lease_without_progress);
+    }
 }
@@ -23,21 +23,32 @@
 //! unconsumed intents is the consumer's job (see `rustfs-heal`
 //! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`.

+use std::collections::HashMap;
+use std::collections::hash_map::RandomState;
+use std::hash::{BuildHasher, Hash};
+use std::sync::atomic::AtomicU64;
+use std::sync::atomic::AtomicUsize;
 use std::sync::{
-    Arc, OnceLock,
+    Arc, Mutex, OnceLock,
    atomic::{AtomicBool, Ordering},
 };
+use std::time::{Duration, Instant};
 use tokio::sync::mpsc;
 use uuid::Uuid;

 /// Bounded capacity of the global MRF channel. Backpressure is resolved by
 /// dropping (and counting) intents, never by blocking the producer.
 const MRF_CHANNEL_CAPACITY: usize = 8192;
+const MRF_COALESCER_SHARDS: usize = 16;
+const MRF_COALESCER_MAX_KEYS: usize = 8192;
+const MRF_COALESCER_MAX_BYTES: usize = 16 * 1024 * 1024;
+const MRF_COALESCER_TTL: Duration = Duration::from_secs(60);
+const MRF_MAX_IDENTITY_COMPONENT: usize = 1024;

 /// Why an intent was produced. Drives the heal priority mapping on the
 /// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High,
 /// PartialWrite -> Normal).
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub enum MrfKind {
    /// Erasure decode failed while serving a read (read path).
    DecodeFailure,
@@ -67,12 +78,52 @@ pub struct MrfIntent {
    /// Version the intent targets, as raw UUID bytes.
    pub version_id: Option<[u8; 16]>,
    pub kind: MrfKind,
+    /// Stable erasure-set scope when the producer has it. Kept optional so
+    /// metadata corruption and legacy producers do not invent a scope.
+    pub scope: Option<MrfScope>,
+    /// Generation of the node-local ingress lease. It is not persisted in
+    /// the journal; replayed records acquire a fresh lease when re-enqueued.
+    pub lease: Option<MrfIngressLease>,
    pub enqueued_at_ms: u64,
    /// Times this intent has already been offered to the heal manager.
    /// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`.
    pub attempts: u8,
 }

+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct MrfScope {
+    pub pool_index: u32,
+    pub set_index: u32,
+}
+
+/// Opaque generation used to release exactly the admission that created an
+/// ingress entry. A generation prevents a late terminal callback from
+/// deleting a newer retry for the same identity (ABA).
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct MrfIngressLease(u64);
+
+impl MrfIngressLease {
+    const fn new(value: u64) -> Self {
+        Self(value)
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum MrfDropReason {
+    Disabled,
+    Uninitialized,
+    Full,
+    OversizedIdentity,
+    CoalescerFull,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum MrfIngressResult {
+    Enqueued,
+    Coalesced,
+    Dropped(MrfDropReason),
+}
+
 /// Consumer-side retry ceiling before an intent is given up on.
 pub const MRF_MAX_ATTEMPTS: u8 = 3;

@@ -87,6 +138,159 @@ impl MrfIntent {

 static GLOBAL_MRF_SENDER: OnceLock<mpsc::Sender<MrfIntent>> = OnceLock::new();

+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+struct MrfIdentityKey {
+    kind: MrfKind,
+    bucket: Arc<str>,
+    object: Arc<str>,
+    version_id: Option<[u8; 16]>,
+    scope: Option<MrfScope>,
+}
+
+#[derive(Debug)]
+struct IngressEntry {
+    lease: MrfIngressLease,
+    expires_at: Instant,
+    bytes: usize,
+}
+
+type MrfCoalescerShard = Mutex<HashMap<MrfIdentityKey, IngressEntry>>;
+type MrfCoalescer = Box<[MrfCoalescerShard]>;
+
+static MRF_COALESCER: OnceLock<MrfCoalescer> = OnceLock::new();
+static NEXT_MRF_LEASE: AtomicU64 = AtomicU64::new(1);
+static MRF_COALESCER_COUNT: AtomicUsize = AtomicUsize::new(0);
+static MRF_COALESCER_BYTES: AtomicUsize = AtomicUsize::new(0);
+static MRF_HASH_STATE: OnceLock<RandomState> = OnceLock::new();
+
+fn coalescer() -> &'static [MrfCoalescerShard] {
+    MRF_COALESCER.get_or_init(|| {
+        (0..MRF_COALESCER_SHARDS)
+            .map(|_| Mutex::new(HashMap::new()))
+            .collect::<Vec<_>>()
+            .into_boxed_slice()
+    })
+}
+
+fn key_shard(key: &MrfIdentityKey) -> usize {
+    let hash = MRF_HASH_STATE.get_or_init(RandomState::new).hash_one(key);
+    usize::try_from(hash).unwrap_or(0) % MRF_COALESCER_SHARDS
+}
+
+fn canonical_version(version_id: Option<Uuid>) -> Option<[u8; 16]> {
+    version_id
+        .filter(|version| !version.is_nil())
+        .map(|version| *version.as_bytes())
+}
+
+fn canonical_identity(
+    kind: MrfKind,
+    version_id: Option<[u8; 16]>,
+    scope: Option<MrfScope>,
+) -> (Option<[u8; 16]>, Option<MrfScope>) {
+    let version_id = version_id.filter(|bytes| *bytes != [0; 16]);
+    match kind {
+        MrfKind::MetadataCorruption => (None, None),
+        MrfKind::DecodeFailure | MrfKind::PartialWrite => (version_id, scope),
+    }
+}
+
+fn identity_estimated_bytes(key: &MrfIdentityKey) -> usize {
+    64usize
+        .saturating_add(key.bucket.len())
+        .saturating_add(key.object.len())
+        .saturating_add(key.version_id.map_or(0, |_| 16))
+        .saturating_add(key.scope.map_or(0, |_| 8))
+}
+
+fn reserve(counter: &AtomicUsize, limit: usize, amount: usize) -> bool {
+    let mut current = counter.load(Ordering::Relaxed);
+    loop {
+        let Some(next) = current.checked_add(amount) else {
+            return false;
+        };
+        if next > limit {
+            return false;
+        }
+        match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) {
+            Ok(_) => return true,
+            Err(observed) => current = observed,
+        }
+    }
+}
+
+fn coalescer_admit(key: MrfIdentityKey) -> Result<MrfIngressLease, MrfIngressResult> {
+    let shard = key_shard(&key);
+    let mut entries = coalescer()[shard]
+        .lock()
+        .map_err(|_| MrfIngressResult::Dropped(MrfDropReason::CoalescerFull))?;
+    let now = Instant::now();
+    let before = entries.len();
+    let mut expired_bytes = 0usize;
+    entries.retain(|_, entry| {
+        if entry.expires_at > now {
+            true
+        } else {
+            expired_bytes = expired_bytes.saturating_add(entry.bytes);
+            false
+        }
+    });
+    let evicted = before.saturating_sub(entries.len());
+    if evicted > 0 {
+        MRF_COALESCER_COUNT.fetch_sub(evicted, Ordering::Relaxed);
+        MRF_COALESCER_BYTES.fetch_sub(expired_bytes, Ordering::Relaxed);
+        let evicted = u64::try_from(evicted).unwrap_or(u64::MAX);
+        metrics::counter!("rustfs_heal_mrf_coalescer_expired_total").increment(evicted);
+        metrics::counter!("rustfs_heal_mrf_coalescer_evictions_total").increment(evicted);
+    }
+    if entries.contains_key(&key) {
+        metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
+        return Err(MrfIngressResult::Coalesced);
+    }
+    let bytes = identity_estimated_bytes(&key);
+    let count_reserved = reserve(&MRF_COALESCER_COUNT, MRF_COALESCER_MAX_KEYS, 1);
+    let bytes_reserved = count_reserved && reserve(&MRF_COALESCER_BYTES, MRF_COALESCER_MAX_BYTES, bytes);
+    if !count_reserved || !bytes_reserved {
+        if count_reserved {
+            MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
+        }
+        metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "coalescer_full").increment(1);
+        return Err(MrfIngressResult::Dropped(MrfDropReason::CoalescerFull));
+    }
+    let lease = MrfIngressLease::new(NEXT_MRF_LEASE.fetch_add(1, Ordering::Relaxed));
+    if entries
+        .insert(
+            key,
+            IngressEntry {
+                lease,
+                expires_at: now + MRF_COALESCER_TTL,
+                bytes,
+            },
+        )
+        .is_some()
+    {
+        MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
+        MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
+        metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1);
+        return Err(MrfIngressResult::Coalesced);
+    }
+    Ok(lease)
+}
+
+fn coalescer_release(key: &MrfIdentityKey, lease: Option<MrfIngressLease>) {
+    let Some(lease) = lease else {
+        return;
+    };
+    if let Ok(mut entries) = coalescer()[key_shard(key)].lock() {
+        let should_remove = entries.get(key).is_some_and(|entry| entry.lease == lease);
+        if should_remove {
+            let bytes = entries.remove(key).map(|entry| entry.bytes).unwrap_or(0);
+            MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed);
+            MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed);
+        }
+    }
+}
+
 /// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check
 /// this before touching the channel so the disabled path stays allocation- and
 /// sync-free.
@@ -122,21 +326,90 @@ pub fn init_mrf_channel() -> Result<mpsc::Receiver<MrfIntent>, &'static str> {
 /// This runs on IO error paths, so it stays synchronous and cheap: one
 /// bounded allocation for the two `Arc<str>` handles plus the channel slot.
 pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option<Uuid>) -> bool {
+    matches!(
+        try_send_mrf_intent_typed(kind, bucket, object, version_id, None),
+        MrfIngressResult::Enqueued
+    )
+}
+
+/// Typed ingress result. `Coalesced` means an equivalent in-flight channel
+/// intent already exists; it is not a second executable or durable admission.
+pub fn try_send_mrf_intent_typed(
+    kind: MrfKind,
+    bucket: &str,
+    object: &str,
+    version_id: Option<Uuid>,
+    scope: Option<MrfScope>,
+) -> MrfIngressResult {
    if !mrf_delivery_enabled() {
-        return false;
+        return MrfIngressResult::Dropped(MrfDropReason::Disabled);
    }
    let Some(sender) = GLOBAL_MRF_SENDER.get() else {
-        return false;
+        return MrfIngressResult::Dropped(MrfDropReason::Uninitialized);
    };
-    let intent = MrfIntent {
+    if bucket.len() > MRF_MAX_IDENTITY_COMPONENT || object.len() > MRF_MAX_IDENTITY_COMPONENT {
+        return MrfIngressResult::Dropped(MrfDropReason::OversizedIdentity);
+    }
+    let (version_id, scope) = canonical_identity(kind, canonical_version(version_id), scope);
+    let key = MrfIdentityKey {
+        kind,
        bucket: Arc::from(bucket),
        object: Arc::from(object),
-        version_id: version_id.map(|vid| *vid.as_bytes()),
+        version_id,
+        scope,
+    };
+    let lease = match coalescer_admit(key.clone()) {
+        Ok(lease) => lease,
+        Err(result) => return result,
+    };
+    let intent = MrfIntent {
+        bucket: key.bucket.clone(),
+        object: key.object.clone(),
+        version_id: key.version_id,
        kind,
+        scope,
+        lease: Some(lease),
        enqueued_at_ms: unix_now_ms(),
        attempts: 0,
    };
-    sender.try_send(intent).is_ok()
+    match sender.try_send(intent) {
+        Ok(()) => MrfIngressResult::Enqueued,
+        Err(mpsc::error::TrySendError::Full(_)) => {
+            coalescer_release(&key, Some(lease));
+            metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "channel_full").increment(1);
+            MrfIngressResult::Dropped(MrfDropReason::Full)
+        }
+        Err(mpsc::error::TrySendError::Closed(_)) => {
+            coalescer_release(&key, Some(lease));
+            MrfIngressResult::Dropped(MrfDropReason::Uninitialized)
+        }
+    }
+}
+
+/// Release the ingress key once the consumer owns the intent.
+pub fn release_mrf_intent(intent: &MrfIntent) {
+    release_mrf_identity(intent.kind, &intent.bucket, &intent.object, intent.version_id, intent.scope, intent.lease);
+}
+
+pub fn release_mrf_identity(
+    kind: MrfKind,
+    bucket: &str,
+    object: &str,
+    version_id: Option<[u8; 16]>,
+    scope: Option<MrfScope>,
+    lease: Option<MrfIngressLease>,
+) {
+    let (version_id, scope) = canonical_identity(kind, version_id, scope);
+    coalescer_release(
+        &MrfIdentityKey {
+            kind,
+            bucket: Arc::from(bucket),
+            object: Arc::from(object),
+            version_id,
+            scope,
+        },
+        lease,
+    );
 }

 fn unix_now_ms() -> u64 {
@@ -144,7 +417,8 @@ fn unix_now_ms() -> u64 {
    // failure would be a bug rather than something to handle here.
    std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
-        .map(|d| d.as_millis() as u64)
+        .ok()
+        .and_then(|d| u64::try_from(d.as_millis()).ok())
        .unwrap_or(0)
 }

@@ -215,12 +489,60 @@ mod tests {
            object: Arc::from("object"),
            version_id: Some([0u8; 16]),
            kind: MrfKind::DecodeFailure,
+            scope: None,
+            lease: None,
            enqueued_at_ms: 0,
            attempts: 0,
        };
        assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len());
    }

+    #[test]
+    fn ingress_duplicate_identity_coalesces_and_releases_for_retry() {
+        let key = MrfIdentityKey {
+            kind: MrfKind::DecodeFailure,
+            bucket: Arc::from("ingress-test-bucket"),
+            object: Arc::from("ingress-test-object"),
+            version_id: Some([9; 16]),
+            scope: Some(MrfScope {
+                pool_index: 3,
+                set_index: 4,
+            }),
+        };
+        let lease = coalescer_admit(key.clone()).expect("first identity should be admitted");
+        for _ in 0..999 {
+            assert_eq!(coalescer_admit(key.clone()), Err(MrfIngressResult::Coalesced));
+        }
+        coalescer_release(&key, Some(lease));
+        let retry_lease = coalescer_admit(key.clone()).expect("released identity must admit a retry");
+        coalescer_release(&key, Some(retry_lease));
+    }
+
+    #[test]
+    fn ingress_identity_preserves_kind_scope_and_version_boundaries() {
+        let (nil_version, nil_scope) = canonical_identity(
+            MrfKind::DecodeFailure,
+            Some([0; 16]),
+            Some(MrfScope {
+                pool_index: 1,
+                set_index: 2,
+            }),
+        );
+        assert_eq!(nil_version, None, "nil UUID is the unversioned identity");
+        assert!(nil_scope.is_some());
+
+        let (metadata_version, metadata_scope) = canonical_identity(
+            MrfKind::MetadataCorruption,
+            Some([7; 16]),
+            Some(MrfScope {
+                pool_index: 1,
+                set_index: 2,
+            }),
+        );
+        assert_eq!(metadata_version, None);
+        assert_eq!(metadata_scope, None);
+    }
+
    #[tokio::test]
    async fn try_send_delivers_and_respects_capacity() {
        let mut receiver = init_mrf_channel().expect("first initialization should succeed");
@@ -230,6 +552,7 @@ mod tests {
        let intent = receiver.recv().await.expect("intent should arrive");
        assert_eq!(intent.kind, MrfKind::DecodeFailure);
        assert_eq!(intent.bucket.as_ref(), "b");
+        release_mrf_intent(&intent);

        // Disable delivery: producers become no-ops.
        set_mrf_delivery_enabled(false);
@@ -239,8 +562,8 @@ mod tests {
        // Fill the bounded channel past capacity: excess intents are dropped,
        // never blocking.
        let mut accepted = 0;
-        for _ in 0..(MRF_CHANNEL_CAPACITY + 64) {
-            if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) {
+        for index in 0..(MRF_CHANNEL_CAPACITY + 64) {
+            if try_send_mrf_intent(MrfKind::PartialWrite, "b", &format!("o-{index}"), None) {
                accepted += 1;
            }
        }
@@ -84,6 +84,12 @@ Current guidance:
 - `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
 - `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)

+Scanner cycle budget controls:
+
+- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
+- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
+- A timed-out cycle cancels cooperative scanner work, then fences its leader epoch before releasing the lease. An uncooperative I/O operation is dropped after the bounded shutdown window; its cursor is not claimed to be durable and the scanner reports `recovery-required` when the worker cannot stop cooperatively, the cycle state was not confirmed durable, or epoch fencing cannot be persisted.
+
 ## Mmap read environment aliases

 - `RUSTFS_OBJECT_MMAP_READ_ENABLE` (canonical)
@@ -60,9 +60,9 @@ pub const DEFAULT_RNG_SEED: Option<u64> = None; // None means random
 /// Dedicated blocking thread pool for fsync/fdatasync operations.
 /// When > 1, fsync operations are isolated from the main blocking pool to
 /// prevent device-bound fsync from starving read operations (pread/stat/open).
-/// Default 0 means auto (no isolation, use main runtime).
+/// Default 64 isolates fsync from the main blocking pool to prevent device-bound fsync from starving read I/O.
 pub const ENV_FSYNC_BLOCKING_THREADS: &str = "RUSTFS_RUNTIME_FSYNC_BLOCKING_THREADS";
-pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 0;
+pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 64;

 // Dial9 Tokio Telemetry Default values
 pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default
@@ -143,9 +143,12 @@ pub const ENV_SCANNER_MAX_WAIT_SECS: &str = "RUSTFS_SCANNER_MAX_WAIT_SECS";
 /// Default scanner speed preset.
 pub const DEFAULT_SCANNER_SPEED: &str = "default";

-/// Default scanner cycle runtime budget.
-/// `0` keeps the existing unbounded per-cycle behavior.
-pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 0;
+/// Default scanner cycle runtime budget when no override is configured.
+///
+/// An explicit `0` remains the compatibility escape hatch for an unbounded
+/// cycle. Keeping the unset default finite prevents a stalled scanner I/O
+/// operation from holding the leader lease forever.
+pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 30 * 60;

 /// Default scanner per-cycle object budget.
 /// `0` keeps the existing unbounded per-cycle behavior.
@@ -123,7 +123,7 @@ via `create_s3_client(idx)` / `create_all_clients()`. See
 | `find_available_port` | Random free port (isolation primitive) |
 | `rustfs_binary_path` / `_with_features` | Locate/build the binary; honors `RUSTFS_BUILD_FEATURES` |
 | `requested_rustfs_build_features` / `rustfs_build_feature_enabled` | Feature-gate a test to what the binary was built with |
-| `awscurl_available` + `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl` (skip gracefully when absent) |
+| `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl`; missing binaries are test failures |
 | `replication_fast_env` | Env vars that shrink replication timers (from repl-4); pass to `start_rustfs_server_with_env` |
 | `local_http_client` / `init_logging` | Loopback HTTP client; idempotent tracing init |
 | `RustFSTestClusterEnvironment` (`new`/`start`/`start_node`/`stop_node`/`create_all_clients`) | Multi-node harness |
@@ -189,7 +189,7 @@ cargo nextest run --profile e2e-smoke -p e2e_test
 cargo nextest run --profile e2e-full -p e2e_test
 # Cluster fault nightly lane
 cargo nextest run --profile e2e-nightly -p e2e_test
-# Replication nightly lane; install awscurl so STS paths do not skip
+# Replication nightly lane; awscurl is required for STS paths
 cargo nextest run --profile e2e-repl-nightly -p e2e_test
 # Fixed-port protocol nightly lane
 RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
@@ -221,9 +221,8 @@ The `s3s-e2e` CI job selects a random `RUSTFS_TEST_PORT` (see the `e2e-tests`
 job) to dodge this; local single-node tests already use random ports, so a
 lingering orphan is usually the cause of a spurious bind failure.

-**`awscurl` not found.** `awscurl`-dependent tests skip gracefully with a
-visible log line (`awscurl_available()`); install `awscurl` to actually run
-them.
+**`awscurl` not found.** `awscurl`-dependent tests fail closed with a process
+spawn error. Install the pinned CI version before running their profiles.

 ## Related

@@ -258,10 +257,9 @@ A test module may join the smoke filter only if every test in it is:
 2. **Single-node** — spawns its own server via
   `RustFSTestEnvironment`/`start_rustfs_server` on a random port with an
   isolated temp dir. No `RustFSTestClusterEnvironment`, no fixed ports.
-3. **Dependency-free** — no pre-started server at `localhost:9000`, no Vault,
-   no fixed protocol ports. Tools that may be absent on the runner (e.g.
-   `awscurl`) are acceptable only when the test skips gracefully with a
-   visible log line (see `bucket_policy_check_test.rs`).
+3. **Hermetic dependencies** — no pre-started server at `localhost:9000`, no
+   Vault, and no fixed protocol ports. Any required CLI must be pinned and
+   installed by the workflow; a missing CLI must fail the test.
 4. **Not `#[ignore]`** — ignored tests are activation work (backlog#1149
   ci-13 / backlog#1148 ilm-3), not smoke candidates.

@@ -278,4 +276,9 @@ listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or
 moving e2e tests so acceptance numbers in the test-strategy issues
 (backlog#1147–#1155) stay auditable. When a profile membership change is
 intentional, review its JSON listing before updating the matching
-`.config/e2e-*-selection.txt` test-ID digest.
+`.config/e2e-*-selection.txt` test-ID digest. Update only the platform that
+produced the listing:
+
+```bash
+python3 scripts/check_test_wiring.py --update-profile e2e-full /path/to/listing.json linux
+```
@@ -52,10 +52,6 @@ fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key:
 #[tokio::test]
 async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    if !crate::common::awscurl_available() {
-        info!("Skipping test_bucket_policy_authenticated_user because awscurl is not available");
-        return Ok(());
-    }
    info!("Starting test_bucket_policy_authenticated_user...");

    let mut env = RustFSTestEnvironment::new().await?;
@@ -24,10 +24,9 @@ use tracing::{info, warn};
 const BUCKET: &str = "conditional-put-race-bucket";
 const BUCKET_METADATA_RELOAD_BUCKET: &str = "bucket-metadata-reload-barrier";

-async fn cleanup_object(client: &Client, key: &str) {
-    if let Err(e) = client.delete_object().bucket(BUCKET).key(key).send().await {
-        warn!("Failed to delete object '{}' from bucket '{}' during cleanup: {:?}", key, BUCKET, e);
-    }
+async fn cleanup_object(client: &Client, key: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    client.delete_object().bucket(BUCKET).key(key).send().await?;
+    Ok(())
 }

 async fn assert_bucket_cors_missing(client: &Client) {
@@ -83,14 +82,13 @@ async fn run_race_iteration(
    test_key: &str,
    iteration: usize,
 ) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
-    cleanup_object(&clients[0], test_key).await;
+    cleanup_object(&clients[0], test_key).await?;
    tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;

-    let head_result = clients[0].head_object().bucket(BUCKET).key(test_key).send().await;
-
-    if head_result.is_ok() {
-        warn!("Warning: Object still exists after cleanup, skipping iteration {}", iteration);
-        return Ok(0);
+    match clients[0].head_object().bucket(BUCKET).key(test_key).send().await {
+        Ok(_) => return Err(format!("object still exists after cleanup in iteration {iteration}").into()),
+        Err(error) if error.as_service_error().is_some_and(|error| error.is_not_found()) => {}
+        Err(error) => return Err(format!("failed to verify cleanup in iteration {iteration}: {error:?}").into()),
    }

    info!("\n=== Iteration {} ===", iteration);
@@ -132,14 +130,16 @@ async fn run_race_iteration(

    info!("Result: {} out of {} succeeded", success_count, clients.len());

+    if had_error {
+        return Err("one or more conditional PUTs failed unexpectedly".into());
+    }
+
    if success_count > 1 {
        info!(">>> RACE CONDITION DETECTED!");
    } else if success_count == 1 {
        info!(">>> Correct behavior: exactly 1 writer succeeded.");
-    } else if had_error {
-        return Err("all conditional PUTs failed (e.g. cluster/bucket not ready)".into());
    } else {
-        info!(">>> Unexpected: no writers succeeded.");
+        return Err("no conditional PUT succeeded".into());
    }

    Ok(success_count)
@@ -179,7 +179,7 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
            }
        }

-        cleanup_object(&clients[0], &test_key).await;
+        cleanup_object(&clients[0], &test_key).await?;
        tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
    }

@@ -189,7 +189,7 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
    info!("Total iterations:   {}", iterations);
    info!("Correct (1 winner): {}", correct_count);
    info!("Race conditions:    {}", races_detected);
-    info!("Errors (skipped):   {}", error_count);
+    info!("Failed iterations:  {}", error_count);

    assert_eq!(races_detected, 0, "Race conditions detected: {}/{}", races_detected, iterations);
    assert_eq!(
@@ -197,6 +197,10 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::E
        "{} iteration(s) failed due to errors (e.g. cluster not ready)",
        error_count
    );
+    assert_eq!(
+        correct_count, iterations,
+        "only {correct_count}/{iterations} iterations observed exactly one winner"
+    );

    Ok(())
 }
@@ -213,7 +217,7 @@ async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::

    let client = cluster.create_s3_client(0)?;
    let test_key = "basic-conditional-put";
-    cleanup_object(&client, test_key).await;
+    cleanup_object(&client, test_key).await?;

    let result = client
        .put_object()
@@ -245,7 +249,7 @@ async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::
        assert_eq!(code, "PreconditionFailed");
    }

-    cleanup_object(&client, test_key).await;
+    cleanup_object(&client, test_key).await?;
    Ok(())
 }

@@ -494,15 +494,20 @@ fn awscurl_binary_path() -> PathBuf {
        .unwrap_or_else(|| PathBuf::from("awscurl"))
 }

-pub fn awscurl_available() -> bool {
-    let path = awscurl_binary_path();
-    if path.components().count() > 1 || path.is_absolute() {
-        return path.is_file();
+fn verify_awscurl_path(path: &Path) -> std::io::Result<()> {
+    let output = Command::new(path).arg("--help").output()?;
+    if output.status.success() {
+        return Ok(());
    }

-    std::env::var_os("PATH")
-        .map(|paths| std::env::split_paths(&paths).any(|dir| dir.join(&path).is_file()))
-        .unwrap_or(false)
+    Err(std::io::Error::other(format!(
+        "awscurl prerequisite check failed: {}",
+        String::from_utf8_lossy(&output.stderr).trim()
+    )))
+}
+
+pub fn require_awscurl() -> std::io::Result<()> {
+    verify_awscurl_path(&awscurl_binary_path())
 }

 // Global initialization
@@ -1747,6 +1752,22 @@ mod tests {
        assert_eq!(normalize_rustfs_build_features(" , "), None);
    }

+    #[test]
+    fn missing_awscurl_is_a_prerequisite_failure() {
+        let missing = std::env::temp_dir().join(format!("missing-awscurl-{}", Uuid::new_v4()));
+
+        let error = verify_awscurl_path(&missing).expect_err("a missing awscurl binary must fail the test prerequisite");
+
+        assert_eq!(error.kind(), ErrorKind::NotFound);
+    }
+
+    #[test]
+    fn available_awscurl_client_passes_prerequisite_check() {
+        let executable = std::env::current_exe().expect("the test executable should have a path");
+
+        verify_awscurl_path(&executable).expect("an available client with a working help command should pass");
+    }
+
    #[test]
    fn capture_log_path_uses_temp_directory_basename() {
        assert_eq!(
@@ -16,9 +16,7 @@
 //! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit
 //! `Content-Type: application/x-www-form-urlencoded` on `POST /`.

-use crate::common::{
-    RustFSTestEnvironment, awscurl_available, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging,
-};
+use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging};
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging};
@@ -175,11 +173,6 @@ async fn cleanup_bucket_and_object(admin: &Client, bucket: &str, key: &str) {
 #[tokio::test]
 async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_iam_policy_existing_object_tag_get_object: awscurl not available");
-        return Ok(());
-    }
-
    let suffix = Uuid::new_v4();
    let user = format!("e2eiamtag-{suffix}");
    let user_secret = "longSecretKeyForTest123!";
@@ -233,11 +226,6 @@ async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box<
 #[tokio::test]
 async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_bucket_policy_existing_object_tag_get_object: awscurl not available");
-        return Ok(());
-    }
-
    let suffix = Uuid::new_v4();
    let user = format!("e2ebptag-{suffix}");
    let user_secret = "longSecretKeyForTest456!";
@@ -294,11 +282,6 @@ async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), B
 #[tokio::test]
 async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_sts_assume_role_session_policy_existing_object_tag: awscurl not available");
-        return Ok(());
-    }
-
    let suffix = Uuid::new_v4();
    let parent = format!("e2e-sts-par-{suffix}");
    let parent_secret = "longSecretKeyForParentSts99!";
@@ -370,11 +353,6 @@ async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result
 #[tokio::test]
 async fn test_e2e_sts_session_policy_delete_objects_object_prefix_only() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    if !awscurl_available() {
-        info!("Skipping test_e2e_sts_session_policy_delete_objects_object_prefix_only: awscurl not available");
-        return Ok(());
-    }
-
    let suffix = Uuid::new_v4();
    let parent = format!("e2e-sts-del-par-{suffix}");
    let parent_secret = "longSecretKeyForParentDelete99!";
@@ -14,7 +14,7 @@

 //! E2E tests for group management (fixes #2028).

-use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging};
+use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::{Client, Config};
 use tracing::info;
@@ -83,7 +83,6 @@ async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Bo

 /// Test that deleting a group with members fails, and deleting an empty group succeeds.
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();

@@ -91,29 +90,58 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
    env.start_rustfs_server(vec![]).await?;

    // 1. Create a user
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey=testuser1", env.url);
    let user_body = serde_json::json!({
        "secretKey": "testuser1secret",
        "status": "enabled"
    });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/add-user?accessKey=testuser1",
+        Some(user_body.to_string()),
+    )
+    .await?;
    info!("Created testuser1");

    // 2. Create a group with testuser1 as a member
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
    let add_member_body = serde_json::json!({
        "group": "testgroup",
        "members": ["testuser1"],
        "isRemove": false,
        "groupStatus": "enabled"
    });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
    info!("Added testuser1 to testgroup");

    // 3. Attempt to delete the group while it still has members — should fail
-    let delete_group_url = format!("{}/rustfs/admin/v3/group/testgroup", env.url);
-    let delete_result = awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await;
-    assert!(delete_result.is_err(), "deleting a non-empty group should fail");
+    let (delete_status, delete_body) = admin_request(
+        &env.url,
+        http::Method::DELETE,
+        "/rustfs/admin/v3/group/testgroup",
+        None,
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    assert_eq!(
+        delete_status,
+        reqwest::StatusCode::BAD_REQUEST,
+        "deleting a non-empty group must return HTTP 400, body: {delete_body}"
+    );
+    assert!(
+        delete_body.contains("<Code>InvalidRequest</Code>"),
+        "deleting a non-empty group must return InvalidRequest, body: {delete_body}"
+    );
+    assert!(
+        delete_body.contains("<Message>group is not empty</Message>"),
+        "deleting a non-empty group returned an unexpected message: {delete_body}"
+    );
    info!("Delete of non-empty group correctly rejected");

    // 4. Remove the member from the group
@@ -123,17 +151,42 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
        "isRemove": true,
        "groupStatus": "enabled"
    });
-    awscurl_put(&update_members_url, &remove_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(remove_member_body.to_string()),
+    )
+    .await?;
    info!("Removed testuser1 from testgroup");

    // 5. Delete the now-empty group — should succeed
-    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(&env, http::Method::DELETE, "/rustfs/admin/v3/group/testgroup", None).await?;
    info!("Deleted empty testgroup successfully");

    // 6. Verify the group no longer exists
-    let get_group_url = format!("{}/rustfs/admin/v3/group?group=testgroup", env.url);
-    let get_result = awscurl_get(&get_group_url, &env.access_key, &env.secret_key).await;
-    assert!(get_result.is_err(), "group should no longer exist after deletion");
+    let (get_status, get_body) = admin_request(
+        &env.url,
+        http::Method::GET,
+        "/rustfs/admin/v3/group?group=testgroup",
+        None,
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    assert_eq!(
+        get_status,
+        reqwest::StatusCode::NOT_FOUND,
+        "a deleted group must return HTTP 404, body: {get_body}"
+    );
+    assert!(
+        get_body.contains("<Code>NoSuchResource</Code>"),
+        "a deleted group must return NoSuchResource, body: {get_body}"
+    );
+    assert!(
+        get_body.contains("<Message>group &apos;testgroup&apos; does not exist</Message>"),
+        "a deleted group returned an unexpected message: {get_body}"
+    );
    info!("Confirmed testgroup no longer exists");

    Ok(())
@@ -142,7 +195,6 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box<dyn std
 /// Test that a user with only group membership (no explicit user policy) gets group policies
 /// and can perform actions allowed by the group (regression test for #2028.1).
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();

@@ -160,39 +212,56 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
        "Statement": [{
            "Effect": "Allow",
            "Action": ["s3:ListAllMyBuckets"],
-            "Resource": ["*"]
+            "Resource": ["arn:aws:s3:::*"]
        }]
    });
-    let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
-    awscurl_put(&add_policy_url, &policy_doc.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
+        Some(policy_doc.to_string()),
+    )
+    .await?;
    info!("Created canned policy {}", policy_name);

    // 2. Create user with no explicit policy
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
    let user_body = serde_json::json!({
        "secretKey": user_secret,
        "status": "enabled"
    });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
+        Some(user_body.to_string()),
+    )
+    .await?;
    info!("Created user {} with no explicit policy", user_name);

    // 3. Add user to group (creates group with this member; user_group_memberships must be updated)
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
    let add_member_body = serde_json::json!({
        "group": group_name,
        "members": [user_name],
        "isRemove": false,
        "groupStatus": "enabled"
    });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
    info!("Added {} to group {}", user_name, group_name);

    // 4. Attach policy to group
-    let set_policy_url = format!(
-        "{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=true",
-        env.url, policy_name, group_name
-    );
-    awscurl_put(&set_policy_url, "", &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={group_name}&isGroup=true"),
+        Some(String::new()),
+    )
+    .await?;
    info!("Attached policy {} to group {}", policy_name, group_name);

    // 5. User with only group (no user policy) should be able to list buckets
@@ -209,7 +278,6 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box<dyn s
 /// Test that after deleting a user who was the only member of a group, the group can be deleted
 /// (regression test for #2028.2: delete group uses backend membership, not stale cache).
 #[tokio::test(flavor = "multi_thread")]
-#[ignore = "requires awscurl and spawns a real RustFS server"]
 async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();

@@ -221,33 +289,47 @@ async fn test_delete_group_after_deleting_user() -> Result<(), Box<dyn std::erro
    let group_name = "soledeletegroup";

    // 1. Create user
-    let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, user_name);
    let user_body = serde_json::json!({
        "secretKey": user_secret,
        "status": "enabled"
    });
-    awscurl_put(&add_user_url, &user_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user_name}"),
+        Some(user_body.to_string()),
+    )
+    .await?;
    info!("Created user {}", user_name);

    // 2. Add user to group
-    let update_members_url = format!("{}/rustfs/admin/v3/update-group-members", env.url);
    let add_member_body = serde_json::json!({
        "group": group_name,
        "members": [user_name],
        "isRemove": false,
        "groupStatus": "enabled"
    });
-    awscurl_put(&update_members_url, &add_member_body.to_string(), &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/update-group-members",
+        Some(add_member_body.to_string()),
+    )
+    .await?;
    info!("Added {} to group {}", user_name, group_name);

    // 3. Delete the user (backend and cache update so group membership becomes empty)
-    let remove_user_url = format!("{}/rustfs/admin/v3/remove-user?accessKey={}", env.url, user_name);
-    awscurl_delete(&remove_user_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(
+        &env,
+        http::Method::DELETE,
+        &format!("/rustfs/admin/v3/remove-user?accessKey={user_name}"),
+        None,
+    )
+    .await?;
    info!("Deleted user {}", user_name);

    // 4. Deleting the group should succeed (backend has empty members; no stale cache)
-    let delete_group_url = format!("{}/rustfs/admin/v3/group/{}", env.url, group_name);
-    awscurl_delete(&delete_group_url, &env.access_key, &env.secret_key).await?;
+    admin_ok(&env, http::Method::DELETE, &format!("/rustfs/admin/v3/group/{group_name}"), None).await?;
    info!("Deleted group {} after user was removed", group_name);

    Ok(())
@@ -380,10 +380,24 @@ mod tests {
        cluster.start_node(1).await?;

        let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
-        let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
-        assert!(
-            !status_body.contains("MissingContentLength"),
-            "background heal status should not fail without an explicit Content-Length: {status_body}"
+        let mut recovered = serde_json::Value::Null;
+        for _ in 0..60 {
+            let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
+            assert!(
+                !status_body.contains("MissingContentLength"),
+                "background heal status should not fail without an explicit Content-Length: {status_body}"
+            );
+            recovered = serde_json::from_str(&status_body)
+                .map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
+            if recovered["clusterStatusComplete"] == serde_json::Value::Bool(true) {
+                break;
+            }
+            sleep(Duration::from_secs(1)).await;
+        }
+        assert_eq!(
+            recovered["clusterStatusComplete"],
+            serde_json::Value::Bool(true),
+            "cluster heal status should recover before root heal starts: {recovered}"
        );

        let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
@@ -37,7 +37,7 @@ async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box<dyn std::erro

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -159,7 +159,7 @@ async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::err

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -278,7 +278,7 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -475,7 +475,7 @@ async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box<d

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -570,7 +570,7 @@ async fn test_sse_kms_without_key_id_populates_default() -> Result<(), Box<dyn s

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -22,9 +22,7 @@
 //! - KMS backend configuration (Local and Vault)
 //! - SSE encryption testing utilities

-use crate::common::{
-    RustFSTestEnvironment, awscurl_available, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client,
-};
+use crate::common::{RustFSTestEnvironment, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::ServerSideEncryption;
@@ -59,15 +57,6 @@ pub fn init_logging() {
    // Additional KMS-specific logging configuration can be added here if needed
 }

-pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
-    if awscurl_available() {
-        return false;
-    }
-
-    info!("Skipping {} because awscurl is not available in PATH", test_name);
-    true
-}
-
 pub fn sse_customer_key_md5_base64(key: &str) -> String {
    let mut hasher = Md5::new();
    hasher.update(key.as_bytes());
@@ -189,34 +178,121 @@ pub async fn wait_for_kms_ready(
    access_key: &str,
    secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let total_deadline = Duration::from_secs(5);
+    wait_for_kms_ready_with_timeout(base_url, access_key, secret_key, Duration::from_secs(5)).await
+}
+
+async fn wait_for_kms_ready_with_timeout(
+    base_url: &str,
+    access_key: &str,
+    secret_key: &str,
+    total_deadline: Duration,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    let start = tokio::time::Instant::now();
+    let deadline = start + total_deadline;
    let mut backoff = Duration::from_millis(200);
    let max_backoff = Duration::from_secs(1);
-    let mut first_attempt = true;

    loop {
-        if !first_attempt {
-            if start.elapsed() >= total_deadline {
-                return Err("KMS failed to become ready within 5 seconds".into());
-            }
-            sleep(backoff).await;
-            backoff = (backoff * 2).min(max_backoff);
-        }
-        first_attempt = false;
-
-        match get_kms_status(base_url, access_key, secret_key).await {
-            Ok(status) => {
-                info!("KMS is ready (status: {})", status);
-                return Ok(());
-            }
-            Err(e) => {
-                if start.elapsed() >= total_deadline {
-                    return Err(format!("KMS did not become ready within 5 s: last error: {e}").into());
+        match tokio::time::timeout_at(deadline, get_kms_status(base_url, access_key, secret_key)).await {
+            Ok(Ok(status)) => {
+                let backend_status = serde_json::from_str::<serde_json::Value>(&status)
+                    .ok()
+                    .and_then(|value| value.get("backend_status")?.as_str().map(str::to_owned));
+                if backend_status.as_deref() == Some("healthy") {
+                    info!("KMS is ready (status: {})", status);
+                    return Ok(());
                }
-                warn!(error = %e, elapsed_ms = start.elapsed().as_millis() as u64, "KMS not ready yet, retrying…");
+                warn!(
+                    backend_status = backend_status.as_deref().unwrap_or("missing"),
+                    elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX),
+                    "KMS not ready yet, retrying…"
+                );
            }
+            Ok(Err(e)) => {
+                let elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
+                warn!(error = %e, elapsed_ms, "KMS not ready yet, retrying…");
+            }
+            Err(_) => return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into()),
        }
+
+        let now = tokio::time::Instant::now();
+        if now >= deadline {
+            return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into());
+        }
+        sleep((now + backoff).min(deadline) - now).await;
+        backoff = (backoff * 2).min(max_backoff);
+    }
+}
+
+#[cfg(test)]
+mod readiness_tests {
+    use super::{wait_for_kms_ready, wait_for_kms_ready_with_timeout};
+    use std::sync::{
+        Arc,
+        atomic::{AtomicUsize, Ordering},
+    };
+    use std::time::Duration;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::TcpListener;
+
+    #[tokio::test]
+    async fn kms_readiness_retries_http_success_until_backend_is_healthy() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
+        let address = listener.local_addr().expect("read readiness test server address");
+        let requests = Arc::new(AtomicUsize::new(0));
+        let server_requests = Arc::clone(&requests);
+        let server = tokio::spawn(async move {
+            for backend_status in ["error", "healthy"] {
+                let (mut socket, _) = listener.accept().await.expect("accept readiness request");
+                let mut request = Vec::new();
+                let mut chunk = [0_u8; 1024];
+                while !request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    let read = socket.read(&mut chunk).await.expect("read readiness request");
+                    if read == 0 {
+                        break;
+                    }
+                    request.extend_from_slice(&chunk[..read]);
+                }
+                server_requests.fetch_add(1, Ordering::SeqCst);
+
+                let body = format!(r#"{{"backend_status":"{backend_status}"}}"#);
+                let response = format!(
+                    "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+                    body.len()
+                );
+                socket.write_all(response.as_bytes()).await.expect("write readiness response");
+            }
+        });
+
+        wait_for_kms_ready(&format!("http://{address}"), "access-key", "secret-key")
+            .await
+            .expect("KMS should become ready after the healthy response");
+
+        let observed_requests = requests.load(Ordering::SeqCst);
+        server.abort();
+        assert_eq!(observed_requests, 2, "an HTTP 200 unhealthy status must be retried");
+    }
+
+    #[tokio::test]
+    async fn kms_readiness_deadline_covers_a_stalled_status_request() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
+        let address = listener.local_addr().expect("read readiness test server address");
+        let server = tokio::spawn(async move {
+            let (mut socket, _) = listener.accept().await.expect("accept readiness request");
+            let mut request = [0_u8; 1024];
+            let _ = socket.read(&mut request).await.expect("read readiness request");
+            std::future::pending::<()>().await;
+        });
+
+        let result = tokio::time::timeout(
+            Duration::from_secs(1),
+            wait_for_kms_ready_with_timeout(&format!("http://{address}"), "access-key", "secret-key", Duration::from_millis(50)),
+        )
+        .await
+        .expect("readiness helper must enforce its own deadline");
+
+        server.abort();
+        assert!(result.is_err(), "a stalled status request must not outlive the readiness deadline");
    }
 }

@@ -403,10 +479,6 @@ pub async fn test_kms_key_management(
    access_key: &str,
    secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    if skip_if_kms_admin_tool_unavailable("test_kms_key_management") {
-        return Ok(());
-    }
-
    info!("Testing KMS key management APIs");

    // Test CreateKey
@@ -61,7 +61,7 @@ async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() {
        )
        .await
        .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");

    let client = kms_env.base_env.create_s3_client();
    // Deliberately an UNVERSIONED bucket: that is the branch where the store layer can service
@@ -160,7 +160,7 @@ async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() {
        )
        .await
        .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");

    let client = kms_env.base_env.create_s3_client();
    // Unversioned, and deliberately WITHOUT a bucket default-encryption rule, so the copy below
@@ -256,7 +256,7 @@ async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decrypta
        )
        .await
        .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");

    let client = kms_env.base_env.create_s3_client();
    let bucket = "copy-object-self-copy-bucket-default-sse-test";
@@ -56,7 +56,7 @@ async fn test_self_copy_of_historical_sse_s3_version_is_readable() {
        )
        .await
        .expect("failed to start RustFS with local KMS");
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");

    let client = kms_env.base_env.create_s3_client();
    let bucket = "copy-object-version-restore-sse-test";
@@ -87,7 +87,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box<dyn s

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -147,7 +147,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -250,7 +250,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box<dyn std

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -24,7 +24,6 @@ use super::common::{
    test_sse_kms_encryption, test_sse_s3_encryption,
 };
 use crate::common::{TEST_BUCKET, init_logging};
-use tokio::time::{Duration, sleep};
 use tracing::info;

 /// Comprehensive test: Full KMS workflow with all encryption types
@@ -35,7 +34,7 @@ async fn test_comprehensive_kms_full_workflow() -> Result<(), Box<dyn std::error

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -103,7 +102,7 @@ async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Erro

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -137,7 +136,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -208,7 +207,7 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::e

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -253,7 +252,7 @@ async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::e

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -44,7 +44,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -117,7 +117,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -209,7 +209,7 @@ async fn test_kms_multipart_boundary_conditions() -> Result<(), Box<dyn std::err

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -284,7 +284,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -371,7 +371,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = Arc::new(kms_env.base_env.create_s3_client());
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -478,7 +478,7 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -37,7 +37,7 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box<dyn std::error::

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -127,7 +127,7 @@ async fn test_kms_corrupted_key_files() -> Result<(), Box<dyn std::error::Error

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -218,7 +218,7 @@ async fn test_kms_multipart_upload_interruption() -> Result<(), Box<dyn std::err

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -401,7 +401,7 @@ async fn test_kms_resource_constraints() -> Result<(), Box<dyn std::error::Error

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -20,8 +20,7 @@
 //! - Complete encryption/decryption lifecycle

 use super::common::{
-    LocalKMSTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
-    test_kms_key_management, test_sse_c_encryption,
+    LocalKMSTestEnvironment, get_kms_status, sse_customer_key_md5_base64, test_kms_key_management, test_sse_c_encryption,
 };
 use crate::common::{TEST_BUCKET, init_logging};
 use tracing::{error, info};
@@ -29,9 +28,6 @@ use tracing::{error, info};
 #[tokio::test]
 async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_local_kms_end_to_end") {
-        return Ok(());
-    }
    info!("Starting Local KMS End-to-End Test");

    // Create LocalKMS test environment
@@ -46,7 +42,7 @@ async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
        .expect("Failed to start RustFS with Local KMS");

    // Wait a moment for RustFS to fully start up and initialize KMS
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);

@@ -127,7 +123,7 @@ async fn test_local_kms_key_isolation() {
        .expect("Failed to start RustFS with Local KMS");

    // Wait a moment for RustFS to fully start up and initialize KMS
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");

    info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);

@@ -227,7 +223,7 @@ async fn test_local_kms_large_file() {
        .expect("Failed to start RustFS with Local KMS");

    // Wait a moment for RustFS to fully start up and initialize KMS
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");

    info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);

@@ -309,7 +305,7 @@ async fn test_local_kms_multipart_upload() {
        .expect("Failed to start RustFS with Local KMS");

    // Wait for KMS initialization
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await.expect("KMS ready");

    info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);

@@ -19,12 +19,11 @@
 //! multipart upload behaviour.

 use crate::common::{TEST_BUCKET, init_logging};
-use tokio::time::{Duration, sleep};
 use tracing::{error, info};

 use super::common::{
-    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
-    start_kms, test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
+    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, sse_customer_key_md5_base64, start_kms,
+    test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
    test_sse_kms_encryption, test_sse_s3_encryption,
 };

@@ -45,8 +44,8 @@ impl VaultKmsTestContext {

        start_kms(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;

-        // Allow Vault to finish initialising token auth and transit engine.
-        sleep(Duration::from_secs(2)).await;
+        // Wait for KMS to finish initialising.
+        super::common::wait_for_kms_ready(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;

        Ok(Self { env })
    }
@@ -63,9 +62,6 @@ impl VaultKmsTestContext {
 #[tokio::test]
 async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
-        return Ok(());
-    }
    info!("Starting Vault KMS End-to-End Test with default key {}", VAULT_KEY_NAME);

    let context = VaultKmsTestContext::new().await?;
@@ -118,9 +114,6 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
 #[tokio::test]
 async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
-        return Ok(());
-    }
    info!("Starting Vault KMS SSE-C key isolation test");

    let context = VaultKmsTestContext::new().await?;
@@ -204,9 +197,6 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
 #[tokio::test]
 async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
-        return Ok(());
-    }
    info!("Starting Vault KMS large file SSE-S3 test");

    let context = VaultKmsTestContext::new().await?;
@@ -268,9 +258,6 @@ async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + S
 #[tokio::test]
 async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
-        return Ok(());
-    }
    info!("Starting Vault KMS multipart upload encryption suite");

    let context = VaultKmsTestContext::new().await?;
@@ -298,9 +285,6 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Err
 #[tokio::test]
 async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
-        return Ok(());
-    }
    info!("Starting Vault KMS key operations test (CRUD)");

    let context = VaultKmsTestContext::new().await?;
@@ -39,9 +39,6 @@ mod kms_edge_cases_test;
 #[cfg(test)]
 mod kms_fault_recovery_test;

-#[cfg(test)]
-mod test_runner;
-
 #[cfg(test)]
 mod bucket_default_encryption_test;

@@ -33,7 +33,7 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::er

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -89,7 +89,7 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -187,7 +187,7 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -310,7 +310,7 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -435,7 +435,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::

    let mut kms_env = LocalKMSTestEnvironment::new().await?;
    let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
-    tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
+    kms_env.wait_for_kms_ready().await?;

    let s3_client = kms_env.base_env.create_s3_client();
    kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -1,499 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-#![allow(dead_code)]
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Unified KMS test suite runner
-//!
-//! This module provides a unified interface for running KMS tests with categorization,
-//! filtering, and comprehensive reporting capabilities.
-
-use crate::common::init_logging;
-use std::time::Instant;
-use tokio::time::{Duration, sleep};
-use tracing::{debug, error, info, warn};
-
-/// Test category for organization and filtering
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub enum TestCategory {
-    CoreFunctionality,
-    MultipartEncryption,
-    EdgeCases,
-    FaultRecovery,
-    Comprehensive,
-    Performance,
-}
-
-impl TestCategory {
-    pub fn as_str(&self) -> &'static str {
-        match self {
-            TestCategory::CoreFunctionality => "core-functionality",
-            TestCategory::MultipartEncryption => "multipart-encryption",
-            TestCategory::EdgeCases => "edge-cases",
-            TestCategory::FaultRecovery => "fault-recovery",
-            TestCategory::Comprehensive => "comprehensive",
-            TestCategory::Performance => "performance",
-        }
-    }
-}
-
-/// Test definition with metadata
-#[derive(Debug, Clone)]
-pub struct TestDefinition {
-    pub name: String,
-    pub description: String,
-    pub category: TestCategory,
-    pub estimated_duration: Duration,
-    pub is_critical: bool,
-}
-
-impl TestDefinition {
-    pub fn new(
-        name: impl Into<String>,
-        description: impl Into<String>,
-        category: TestCategory,
-        estimated_duration: Duration,
-        is_critical: bool,
-    ) -> Self {
-        Self {
-            name: name.into(),
-            description: description.into(),
-            category,
-            estimated_duration,
-            is_critical,
-        }
-    }
-}
-
-/// Test execution result
-#[derive(Debug, Clone)]
-pub struct TestResult {
-    pub test_name: String,
-    pub category: TestCategory,
-    pub success: bool,
-    pub duration: Duration,
-    pub error_message: Option<String>,
-}
-
-impl TestResult {
-    pub fn success(test_name: String, category: TestCategory, duration: Duration) -> Self {
-        Self {
-            test_name,
-            category,
-            success: true,
-            duration,
-            error_message: None,
-        }
-    }
-
-    pub fn failure(test_name: String, category: TestCategory, duration: Duration, error: String) -> Self {
-        Self {
-            test_name,
-            category,
-            success: false,
-            duration,
-            error_message: Some(error),
-        }
-    }
-}
-
-/// Comprehensive test suite configuration
-#[derive(Debug, Clone)]
-pub struct TestSuiteConfig {
-    pub categories: Vec<TestCategory>,
-    pub include_critical_only: bool,
-    pub max_duration: Option<Duration>,
-    pub parallel_execution: bool,
-}
-
-impl Default for TestSuiteConfig {
-    fn default() -> Self {
-        Self {
-            categories: vec![
-                TestCategory::CoreFunctionality,
-                TestCategory::MultipartEncryption,
-                TestCategory::EdgeCases,
-                TestCategory::FaultRecovery,
-                TestCategory::Comprehensive,
-            ],
-            include_critical_only: false,
-            max_duration: None,
-            parallel_execution: false,
-        }
-    }
-}
-
-/// Unified KMS test suite runner
-pub struct KMSTestSuite {
-    tests: Vec<TestDefinition>,
-    config: TestSuiteConfig,
-}
-
-impl KMSTestSuite {
-    /// Create a new test suite with default configuration
-    pub fn new() -> Self {
-        let tests = vec![
-            // Core Functionality Tests
-            TestDefinition::new(
-                "test_local_kms_end_to_end",
-                "End-to-end KMS test with all encryption types",
-                TestCategory::CoreFunctionality,
-                Duration::from_secs(60),
-                true,
-            ),
-            TestDefinition::new(
-                "test_local_kms_key_isolation",
-                "Test KMS key isolation and security",
-                TestCategory::CoreFunctionality,
-                Duration::from_secs(45),
-                true,
-            ),
-            // Multipart Encryption Tests
-            TestDefinition::new(
-                "test_local_kms_multipart_upload",
-                "Test large file multipart upload with encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(120),
-                true,
-            ),
-            TestDefinition::new(
-                "test_step1_basic_single_file_encryption",
-                "Basic single file encryption test",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(30),
-                false,
-            ),
-            TestDefinition::new(
-                "test_step2_basic_multipart_upload_without_encryption",
-                "Basic multipart upload without encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(45),
-                false,
-            ),
-            TestDefinition::new(
-                "test_step3_multipart_upload_with_sse_s3",
-                "Multipart upload with SSE-S3 encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(60),
-                true,
-            ),
-            TestDefinition::new(
-                "test_step4_large_multipart_upload_with_encryption",
-                "Large file multipart upload with encryption",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(90),
-                false,
-            ),
-            TestDefinition::new(
-                "test_step5_all_encryption_types_multipart",
-                "All encryption types multipart test",
-                TestCategory::MultipartEncryption,
-                Duration::from_secs(120),
-                true,
-            ),
-            // Edge Cases Tests
-            TestDefinition::new(
-                "test_kms_zero_byte_file_encryption",
-                "Test encryption of zero-byte files",
-                TestCategory::EdgeCases,
-                Duration::from_secs(20),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_single_byte_file_encryption",
-                "Test encryption of single-byte files",
-                TestCategory::EdgeCases,
-                Duration::from_secs(20),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_multipart_boundary_conditions",
-                "Test multipart upload boundary conditions",
-                TestCategory::EdgeCases,
-                Duration::from_secs(45),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_invalid_key_scenarios",
-                "Test invalid key scenarios",
-                TestCategory::EdgeCases,
-                Duration::from_secs(30),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_concurrent_encryption",
-                "Test concurrent encryption operations",
-                TestCategory::EdgeCases,
-                Duration::from_secs(60),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_key_validation_security",
-                "Test key validation security",
-                TestCategory::EdgeCases,
-                Duration::from_secs(30),
-                false,
-            ),
-            // Fault Recovery Tests
-            TestDefinition::new(
-                "test_kms_key_directory_unavailable",
-                "Test KMS when key directory is unavailable",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(45),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_corrupted_key_files",
-                "Test KMS with corrupted key files",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(30),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_multipart_upload_interruption",
-                "Test multipart upload interruption recovery",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(60),
-                false,
-            ),
-            TestDefinition::new(
-                "test_kms_resource_constraints",
-                "Test KMS under resource constraints",
-                TestCategory::FaultRecovery,
-                Duration::from_secs(90),
-                false,
-            ),
-            // Comprehensive Tests
-            TestDefinition::new(
-                "test_comprehensive_kms_full_workflow",
-                "Full KMS workflow comprehensive test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(300),
-                true,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_stress_test",
-                "KMS stress test with large datasets",
-                TestCategory::Comprehensive,
-                Duration::from_secs(400),
-                false,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_key_isolation",
-                "Comprehensive key isolation test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(180),
-                false,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_concurrent_operations",
-                "Comprehensive concurrent operations test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(240),
-                false,
-            ),
-            TestDefinition::new(
-                "test_comprehensive_performance_benchmark",
-                "KMS performance benchmark test",
-                TestCategory::Comprehensive,
-                Duration::from_secs(360),
-                false,
-            ),
-        ];
-
-        Self {
-            tests,
-            config: TestSuiteConfig::default(),
-        }
-    }
-
-    /// Configure the test suite
-    pub fn with_config(mut self, config: TestSuiteConfig) -> Self {
-        self.config = config;
-        self
-    }
-
-    /// Filter tests based on category
-    pub fn filter_by_category(&self, category: &TestCategory) -> Vec<&TestDefinition> {
-        self.tests.iter().filter(|test| &test.category == category).collect()
-    }
-
-    /// Filter tests based on criticality
-    pub fn filter_critical_tests(&self) -> Vec<&TestDefinition> {
-        self.tests.iter().filter(|test| test.is_critical).collect()
-    }
-
-    /// Get test summary by category
-    pub fn get_category_summary(&self) -> std::collections::HashMap<TestCategory, Vec<&TestDefinition>> {
-        let mut summary = std::collections::HashMap::new();
-        for test in &self.tests {
-            summary.entry(test.category.clone()).or_insert_with(Vec::new).push(test);
-        }
-        summary
-    }
-
-    /// Run the complete test suite
-    pub async fn run_test_suite(&self) -> Vec<TestResult> {
-        init_logging();
-        info!("🚀 Starting unified KMS test suite");
-
-        let start_time = Instant::now();
-        let mut results = Vec::new();
-
-        // Filter tests based on configuration
-        let tests_to_run: Vec<&TestDefinition> = self
-            .tests
-            .iter()
-            .filter(|test| self.config.categories.contains(&test.category))
-            .filter(|test| !self.config.include_critical_only || test.is_critical)
-            .collect();
-
-        info!("📊 Test plan: {} test(s) scheduled", tests_to_run.len());
-        for (i, test) in tests_to_run.iter().enumerate() {
-            info!("  {}. {} ({})", i + 1, test.name, test.category.as_str());
-        }
-
-        // Execute tests
-        for (i, test_def) in tests_to_run.iter().enumerate() {
-            info!("🧪 Running test {}/{}: {}", i + 1, tests_to_run.len(), test_def.name);
-            info!("   📝 Description: {}", test_def.description);
-            info!("   🏷️  Category: {}", test_def.category.as_str());
-            info!("   ⏱️  Estimated duration: {:?}", test_def.estimated_duration);
-
-            let test_start = Instant::now();
-            let result = self.run_single_test(test_def).await;
-            let test_duration = test_start.elapsed();
-
-            match result {
-                Ok(_) => {
-                    info!("✅ Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64());
-                    results.push(TestResult::success(test_def.name.clone(), test_def.category.clone(), test_duration));
-                }
-                Err(e) => {
-                    error!("❌ Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e);
-                    results.push(TestResult::failure(
-                        test_def.name.clone(),
-                        test_def.category.clone(),
-                        test_duration,
-                        e.to_string(),
-                    ));
-                }
-            }
-
-            // Add delay between tests to avoid resource conflicts
-            if i < tests_to_run.len() - 1 {
-                debug!("⏸️  Waiting two seconds before the next test...");
-                sleep(Duration::from_secs(2)).await;
-            }
-        }
-
-        let total_duration = start_time.elapsed();
-        self.print_test_summary(&results, total_duration);
-
-        results
-    }
-
-    /// Run a single test by dispatching to the appropriate test function
-    async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        // This is a placeholder for test dispatch logic
-        // In a real implementation, this would dispatch to actual test functions
-        warn!("⚠️  Test '{}' is not implemented in the unified runner; skipping", test_def.name);
-        Ok(())
-    }
-
-    /// Print comprehensive test summary
-    fn print_test_summary(&self, results: &[TestResult], total_duration: Duration) {
-        info!("📊 KMS test suite summary");
-        info!("⏱️  Total duration: {:.2} seconds", total_duration.as_secs_f64());
-        info!("📈 Total tests: {}", results.len());
-
-        let passed = results.iter().filter(|r| r.success).count();
-        let failed = results.iter().filter(|r| !r.success).count();
-
-        info!("✅ Passed: {}", passed);
-        info!("❌ Failed: {}", failed);
-        info!("📊 Success rate: {:.1}%", (passed as f64 / results.len() as f64) * 100.0);
-
-        // Summary by category
-        let mut category_summary: std::collections::HashMap<TestCategory, (usize, usize)> = std::collections::HashMap::new();
-        for result in results {
-            let (total, passed_count) = category_summary.entry(result.category.clone()).or_insert((0, 0));
-            *total += 1;
-            if result.success {
-                *passed_count += 1;
-            }
-        }
-
-        info!("📊 Category summary:");
-        for (category, (total, passed_count)) in category_summary {
-            info!(
-                "  🏷️  {}: {}/{} ({:.1}%)",
-                category.as_str(),
-                passed_count,
-                total,
-                (passed_count as f64 / total as f64) * 100.0
-            );
-        }
-
-        // List failed tests
-        if failed > 0 {
-            warn!("❌ Failing tests:");
-            for result in results.iter().filter(|r| !r.success) {
-                warn!("  - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error"));
-            }
-        }
-    }
-}
-
-/// Quick test suite for critical tests only
-#[tokio::test]
-async fn test_kms_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let config = TestSuiteConfig {
-        categories: vec![TestCategory::CoreFunctionality, TestCategory::MultipartEncryption],
-        include_critical_only: true,
-        max_duration: Some(Duration::from_secs(600)), // 10 minutes max
-        parallel_execution: false,
-    };
-
-    let suite = KMSTestSuite::new().with_config(config);
-    let results = suite.run_test_suite().await;
-
-    let failed_count = results.iter().filter(|r| !r.success).count();
-    if failed_count > 0 {
-        return Err(format!("Critical test suite failed: {failed_count} tests failed").into());
-    }
-
-    info!("✅ All critical tests passed");
-    Ok(())
-}
-
-/// Full comprehensive test suite
-#[tokio::test]
-async fn test_kms_full_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    let suite = KMSTestSuite::new();
-    let results = suite.run_test_suite().await;
-
-    let total_tests = results.len();
-    let failed_count = results.iter().filter(|r| !r.success).count();
-    let success_rate = ((total_tests - failed_count) as f64 / total_tests as f64) * 100.0;
-
-    info!("📊 Full suite success rate: {:.1}%", success_rate);
-
-    // Allow up to 10% failure rate for non-critical tests
-    if success_rate < 90.0 {
-        return Err(format!("Test suite success rate too low: {success_rate:.1}%").into());
-    }
-
-    info!("✅ Full test suite succeeded");
-    Ok(())
-}
@@ -61,6 +61,11 @@ mod get_codec_streaming_compat_test;
 #[cfg(test)]
 mod version_id_regression_test;

+// Receiver-side replication LWW (rustfs/backlog#1953): stale inbound
+// replication metadata must not overwrite a newer local category state.
+#[cfg(test)]
+mod replication_lww_receiver_test;
+
 // Data usage regression tests
 #[cfg(test)]
 mod data_usage_test;
@@ -41,13 +41,6 @@ async fn create_issue_3107_fixture(root: &Path) -> TestResult {
    Ok(())
 }

-fn mc_available() -> bool {
-    Command::new("mc")
-        .arg("--version")
-        .output()
-        .is_ok_and(|output| output.status.success())
-}
-
 fn run_mc(args: &[&str]) -> TestResult {
    let output = Command::new("mc").args(args).output()?;
    if !output.status.success() {
@@ -75,10 +68,7 @@ fn count_files(root: &Path) -> usize {
 async fn test_mc_mirror_small_bucket_completes_without_list_timeout() -> TestResult {
    crate::common::init_logging();
    info!("Starting issue #3107 mc mirror regression test");
-    if !mc_available() {
-        info!("Skipping issue #3107 mc mirror regression test because mc is not installed");
-        return Ok(());
-    }
+    run_mc(&["--version"])?;

    let mut env = RustFSTestEnvironment::new().await?;
    env.start_rustfs_server(vec![]).await?;
@@ -4278,10 +4278,6 @@ async fn test_signed_put_object_extract_preserves_pax_metadata_and_version_id()
 async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retention_conditions()
 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    init_logging();
-    if !crate::common::awscurl_available() {
-        return Ok(());
-    }
-
    let mut env = RustFSTestEnvironment::new().await?;
    env.start_rustfs_server(vec![]).await?;

@@ -16,6 +16,7 @@ use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_lo
 use aws_sdk_s3::primitives::ByteStream;
 use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
+use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE};
 use rustfs_signer::pre_sign_v4;
 use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
 use s3s::Body;
@@ -976,7 +977,8 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
    init_logging();

    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
+        .await?;

    let bucket = "object-lambda-e2e-disabled-target";
    let key = "input.txt";
@@ -992,17 +994,24 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
        .send()
        .await?;

-    configure_webhook_target_with_key_values(
-        &env,
-        "transformer",
-        vec![
-            ("endpoint", "http://127.0.0.1:9/transform".to_string()),
-            ("auth_token", "secret-token".to_string()),
-            ("enable", "off".to_string()),
-        ],
+    let queue_dir = format!("{}/disabled-target-queue", env.temp_dir);
+    tokio::fs::create_dir_all(&queue_dir).await?;
+    let config_url = format!("{}/rustfs/admin/v3/set-config-kv", env.url);
+    let directive = format!(
+        "notify_webhook:transformer enable=off endpoint=\"http://127.0.0.1:9/transform\" auth_token=\"secret-token\" queue_dir=\"{queue_dir}\""
+    );
+    let disable_response = signed_request(
+        http::Method::PUT,
+        &config_url,
+        &env.access_key,
+        &env.secret_key,
+        Some(directive.into_bytes()),
+        Some("text/plain"),
    )
    .await?;
-    wait_for_target_visibility(&env, "transformer").await?;
+    let disable_status = disable_response.status();
+    let disable_body = disable_response.text().await?;
+    assert_eq!(disable_status, StatusCode::OK, "failed to disable target: {disable_body}");

    let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
    let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
@@ -1021,7 +1030,8 @@ async fn test_configure_object_lambda_target_rejects_invalid_endpoint() -> Resul
    init_logging();

    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
+        .await?;

    let bucket = "object-lambda-e2e-invalid-endpoint";

@@ -1064,7 +1074,8 @@ async fn test_configure_object_lambda_notify_webhook_rejects_response_header_tim
    init_logging();

    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
+    env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
+        .await?;

    let response = send_configure_webhook_target_request(
        &env,
@@ -1173,6 +1184,8 @@ async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box
    init_logging();

    let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
+    cluster.set_env(ENV_NOTIFY_ENABLE, "true");
+    cluster.set_env(ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, "1");
    cluster.start().await?;

    let bucket = "listen-notification-cluster";
@@ -26,7 +26,7 @@ use aws_sdk_s3::config::{Credentials, Region};
 use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
 use russh::client::{self, Handle};
 use russh::keys::ssh_key::LineEnding;
-use russh::keys::{Algorithm, PrivateKey, PublicKey};
+use russh::keys::{Algorithm, PrivateKey, PublicKeyOrCertificate};
 use russh_sftp::client::SftpSession;
 use russh_sftp::protocol::OpenFlags;
 use std::path::Path;
@@ -46,7 +46,7 @@ pub struct AcceptAnyServerKey;
 impl client::Handler for AcceptAnyServerKey {
    type Error = anyhow::Error;

-    async fn check_server_key(&mut self, _server_public_key: &PublicKey) -> Result<bool, Self::Error> {
+    async fn check_server_key(&mut self, _server_public_key: &PublicKeyOrCertificate) -> Result<bool, Self::Error> {
        Ok(true)
    }
 }
@@ -18,15 +18,6 @@ use http::{Method, StatusCode};
 use tokio::time::{Duration, sleep, timeout};
 use tracing::{debug, info};

-fn skip_without_awscurl() -> bool {
-    if crate::common::awscurl_available() {
-        return false;
-    }
-
-    info!("Skipping quota test because awscurl is not available");
-    true
-}
-
 /// Test environment setup for quota tests
 pub struct QuotaTestEnv {
    pub env: RustFSTestEnvironment,
@@ -276,9 +267,6 @@ mod integration_tests {
    #[tokio::test]
    async fn test_quota_basic_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
        let env = QuotaTestEnv::new().await?;

        // Create test bucket
@@ -320,9 +308,6 @@ mod integration_tests {
    #[tokio::test]
    async fn test_quota_admission_aws_chunked_declared_encoding() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
        let env = QuotaTestEnv::new().await?;
        env.create_bucket().await?;

@@ -371,9 +356,6 @@ mod integration_tests {
    #[tokio::test]
    async fn test_quota_update_and_clear() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
        let env = QuotaTestEnv::new().await?;

        env.create_bucket().await?;
@@ -406,9 +388,6 @@ mod integration_tests {
    #[tokio::test]
    async fn test_quota_delete_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
        let env = QuotaTestEnv::new().await?;

        env.create_bucket().await?;
@@ -442,9 +421,6 @@ mod integration_tests {
    #[tokio::test]
    async fn test_quota_usage_tracking() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
        let env = QuotaTestEnv::new().await?;

        env.create_bucket().await?;
@@ -480,9 +456,6 @@ mod integration_tests {
    #[tokio::test]
    async fn test_quota_statistics() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
        let env = QuotaTestEnv::new().await?;

        env.create_bucket().await?;
@@ -513,9 +486,6 @@ mod integration_tests {
    #[tokio::test]
    async fn test_quota_check_api() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
        let env = QuotaTestEnv::new().await?;

        env.create_bucket().await?;
@@ -553,9 +523,6 @@ mod integration_tests {
    #[tokio::test]
    async fn test_quota_multiple_buckets() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
        let env = QuotaTestEnv::new().await?;

        // Create two buckets in the same environment
@@ -593,9 +560,6 @@ mod integration_tests {
    #[tokio::test]
    async fn test_quota_error_handling() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
        let env = QuotaTestEnv::new().await?;

        env.create_bucket().await?;
@@ -628,9 +592,6 @@ mod integration_tests {
    #[tokio::test]
    async fn test_quota_http_endpoints() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
        let env = QuotaTestEnv::new().await?;

        env.create_bucket().await?;
@@ -689,9 +650,6 @@ mod integration_tests {
    #[tokio::test]
    async fn test_quota_normal_user_permissions() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
        let env = QuotaTestEnv::new().await?;
        env.create_bucket().await?;

@@ -744,9 +702,6 @@ mod integration_tests {
    #[tokio::test]
    async fn test_quota_copy_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
        let env = QuotaTestEnv::new().await?;

        env.create_bucket().await?;
@@ -789,9 +744,6 @@ mod integration_tests {
    #[tokio::test]
    async fn test_quota_batch_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
        let env = QuotaTestEnv::new().await?;

        env.create_bucket().await?;
@@ -847,9 +799,6 @@ mod integration_tests {
    #[tokio::test]
    async fn test_quota_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        init_logging();
-        if skip_without_awscurl() {
-            return Ok(());
-        }
        let env = QuotaTestEnv::new().await?;

        env.create_bucket().await?;
@@ -1,52 +1,26 @@
 #![cfg(test)]

-use aws_config::meta::region::RegionProviderChain;
+use crate::common::{RustFSTestEnvironment, TEST_BUCKET, init_logging};
 use aws_sdk_s3::Client;
-use aws_sdk_s3::config::{Credentials, Region};
-use aws_sdk_s3::error::SdkError;
+use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
 use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
 use bytes::Bytes;
 use std::error::Error;
+use std::fmt::Debug;

-const ENDPOINT: &str = "http://localhost:9000";
-const ACCESS_KEY: &str = "rustfsadmin";
-const SECRET_KEY: &str = "rustfsadmin";
-const BUCKET: &str = "api-test";
+type TestResult = Result<(), Box<dyn Error + Send + Sync>>;

-async fn create_aws_s3_client() -> Result<Client, Box<dyn Error>> {
-    let region_provider = RegionProviderChain::default_provider().or_else(Region::new("us-east-1"));
-    let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest())
-        .region(region_provider)
-        .credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static"))
-        .endpoint_url(ENDPOINT)
-        .load()
-        .await;
-
-    let client = Client::from_conf(
-        aws_sdk_s3::Config::from(&shared_config)
-            .to_builder()
-            .force_path_style(true)
-            .build(),
+fn assert_s3_error_code<T, E>(result: Result<T, SdkError<E>>, expected: &str)
+where
+    T: Debug,
+    E: ProvideErrorMetadata + Debug,
+{
+    let error = result.expect_err("conditional request must fail");
+    assert_eq!(
+        error.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some(expected),
+        "unexpected conditional request error: {error:?}"
    );
-    Ok(client)
-}
-
-/// Setup test bucket, creating it if it doesn't exist
-async fn setup_test_bucket(client: &Client) -> Result<(), Box<dyn Error>> {
-    match client.create_bucket().bucket(BUCKET).send().await {
-        Ok(_) => {}
-        Err(SdkError::ServiceError(e)) => {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            if !error_code.eq("BucketAlreadyExists") {
-                return Err(e.into());
-            }
-        }
-        Err(e) => {
-            return Err(e.into());
-        }
-    }
-    Ok(())
 }

 /// Generate test data of specified size
@@ -60,7 +34,12 @@ fn generate_test_data(size: usize) -> Vec<u8> {
 }

 /// Upload an object and return its ETag
-async fn upload_object_with_metadata(client: &Client, bucket: &str, key: &str, data: &[u8]) -> Result<String, Box<dyn Error>> {
+async fn upload_object_with_metadata(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    data: &[u8],
+) -> Result<String, Box<dyn Error + Send + Sync>> {
    let response = client
        .put_object()
        .bucket(bucket)
@@ -69,188 +48,164 @@ async fn upload_object_with_metadata(client: &Client, bucket: &str, key: &str, d
        .send()
        .await?;

-    let etag = response.e_tag().unwrap_or("").to_string();
-    Ok(etag)
+    response
+        .e_tag()
+        .map(str::to_owned)
+        .ok_or_else(|| std::io::Error::other("put object response did not include an ETag").into())
 }

-/// Cleanup test objects from bucket
-async fn cleanup_objects(client: &Client, bucket: &str, keys: &[&str]) {
-    for key in keys {
-        let _ = client.delete_object().bucket(bucket).key(*key).send().await;
-    }
-}
-
-/// Generate unique test object key
-fn generate_test_key(prefix: &str) -> String {
-    use std::time::{SystemTime, UNIX_EPOCH};
-    let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos();
-    format!("{prefix}-{timestamp}")
+async fn object_body(client: &Client, key: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
+    let response = client.get_object().bucket(TEST_BUCKET).key(key).send().await?;
+    Ok(response.body.collect().await?.into_bytes())
 }

 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_conditional_put_okay() -> Result<(), Box<dyn std::error::Error>> {
-    let client = create_aws_s3_client().await?;
-    setup_test_bucket(&client).await?;
+async fn test_conditional_put_okay() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();

-    let test_key = generate_test_key("conditional-put-ok");
+    let test_key = "conditional-put-ok";
    let initial_data = generate_test_data(1024); // 1KB test data
-    let updated_data = generate_test_data(2048); // 2KB updated data
+    let matching_data = generate_test_data(2048); // 2KB updated data
+    let non_matching_data = generate_test_data(3072); // 3KB updated data

    // Upload initial object and get its ETag
-    let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &initial_data).await?;
+    let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &initial_data).await?;

    // Test 1: PUT with matching If-Match condition (should succeed)
-    let response1 = client
+    client
        .put_object()
-        .bucket(BUCKET)
-        .key(&test_key)
-        .body(Bytes::from(updated_data.clone()).into())
+        .bucket(TEST_BUCKET)
+        .key(test_key)
+        .body(Bytes::from(matching_data.clone()).into())
        .if_match(&initial_etag)
        .send()
-        .await;
-    assert!(response1.is_ok(), "PUT with matching If-Match should succeed");
+        .await?;
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), matching_data);

    // Test 2: PUT with non-matching If-None-Match condition (should succeed)
    let fake_etag = "\"fake-etag-12345\"";
-    let response2 = client
+    client
        .put_object()
-        .bucket(BUCKET)
-        .key(&test_key)
-        .body(Bytes::from(updated_data.clone()).into())
+        .bucket(TEST_BUCKET)
+        .key(test_key)
+        .body(Bytes::from(non_matching_data.clone()).into())
        .if_none_match(fake_etag)
        .send()
-        .await;
-    assert!(response2.is_ok(), "PUT with non-matching If-None-Match should succeed");
-
-    // Cleanup
-    cleanup_objects(&client, BUCKET, &[&test_key]).await;
+        .await?;
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), non_matching_data);

    Ok(())
 }

 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_conditional_put_failed() -> Result<(), Box<dyn std::error::Error>> {
-    let client = create_aws_s3_client().await?;
-    setup_test_bucket(&client).await?;
+async fn test_conditional_put_failed() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();

-    let test_key = generate_test_key("conditional-put-failed");
+    let test_key = "conditional-put-failed";
    let initial_data = generate_test_data(1024);
    let updated_data = generate_test_data(2048);

    // Upload initial object and get its ETag
-    let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &initial_data).await?;
+    let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &initial_data).await?;

    // Test 1: PUT with non-matching If-Match condition (should fail with 412)
    let fake_etag = "\"fake-etag-should-not-match\"";
    let response1 = client
        .put_object()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
        .body(Bytes::from(updated_data.clone()).into())
        .if_match(fake_etag)
        .send()
        .await;

-    assert!(response1.is_err(), "PUT with non-matching If-Match should fail");
-    if let Err(e) = response1 {
-        if let SdkError::ServiceError(e) = e {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            assert_eq!("PreconditionFailed", error_code);
-        } else {
-            panic!("Unexpected error: {e:?}");
-        }
-    }
+    assert_s3_error_code(response1, "PreconditionFailed");
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), initial_data);

    // Test 2: PUT with matching If-None-Match condition (should fail with 412)
    let response2 = client
        .put_object()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
        .body(Bytes::from(updated_data.clone()).into())
        .if_none_match(&initial_etag)
        .send()
        .await;

-    assert!(response2.is_err(), "PUT with matching If-None-Match should fail");
-    if let Err(e) = response2 {
-        if let SdkError::ServiceError(e) = e {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            assert_eq!("PreconditionFailed", error_code);
-        } else {
-            panic!("Unexpected error: {e:?}");
-        }
-    }
-
-    // Cleanup - only need to clean up the initial object since failed PUTs shouldn't create objects
-    cleanup_objects(&client, BUCKET, &[&test_key]).await;
+    assert_s3_error_code(response2, "PreconditionFailed");
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), initial_data);

    Ok(())
 }

 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_conditional_put_when_object_does_not_exist() -> Result<(), Box<dyn std::error::Error>> {
-    let client = create_aws_s3_client().await?;
-    setup_test_bucket(&client).await?;
+async fn test_conditional_put_when_object_does_not_exist() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();

-    let key = "some_key";
-    cleanup_objects(&client, BUCKET, &[key]).await;
+    let key = "conditional-put-missing";

    // When the object does not exist, the If-Match condition should always fail
    let response1 = client
        .put_object()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
        .key(key)
        .body(Bytes::from(generate_test_data(1024)).into())
        .if_match("*")
        .send()
        .await;
-    assert!(response1.is_err());
-    if let Err(e) = response1 {
-        if let SdkError::ServiceError(e) = e {
-            let e = e.into_err();
-            let error_code = e.meta().code().unwrap_or("");
-            assert_eq!("NoSuchKey", error_code);
-        } else {
-            panic!("Unexpected error: {e:?}");
-        }
-    }
+    assert_s3_error_code(response1, "NoSuchKey");

    // When the object does not exist, the If-None-Match condition should be able to succeed
-    let response2 = client
+    let created_data = generate_test_data(1024);
+    client
        .put_object()
-        .bucket(BUCKET)
+        .bucket(TEST_BUCKET)
        .key(key)
-        .body(Bytes::from(generate_test_data(1024)).into())
+        .body(Bytes::from(created_data.clone()).into())
        .if_none_match("*")
        .send()
-        .await;
-    assert!(response2.is_ok());
+        .await?;
+    assert_eq!(object_body(&client, key).await?.as_ref(), created_data);

-    cleanup_objects(&client, BUCKET, &[key]).await;
    Ok(())
 }

 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::Error>> {
-    let client = create_aws_s3_client().await?;
-    setup_test_bucket(&client).await?;
+async fn test_conditional_multi_part_upload() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    env.create_test_bucket(TEST_BUCKET).await?;
+    let client = env.create_s3_client();

-    let test_key = generate_test_key("multipart-upload-ok");
+    let test_key = "conditional-multipart-upload";
    let test_data = generate_test_data(1024);
-    let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &test_data).await?;
+    let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &test_data).await?;

    let part_size = 5 * 1024 * 1024; // 5MB per part (minimum for multipart)
    let num_parts = 3;
    let mut parts = Vec::new();
+    let mut expected_data = Vec::with_capacity(part_size * usize::try_from(num_parts)?);

    // Initiate multipart upload
-    let initiate_response = client.create_multipart_upload().bucket(BUCKET).key(&test_key).send().await?;
+    let initiate_response = client
+        .create_multipart_upload()
+        .bucket(TEST_BUCKET)
+        .key(test_key)
+        .send()
+        .await?;

    let upload_id = initiate_response
        .upload_id()
@@ -258,12 +213,13 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::

    // Upload parts
    for part_number in 1..=num_parts {
-        let part_data = generate_test_data(part_size);
+        let part_data = vec![u8::try_from(part_number)?; part_size];
+        expected_data.extend_from_slice(&part_data);

        let upload_part_response = client
            .upload_part()
-            .bucket(BUCKET)
-            .key(&test_key)
+            .bucket(TEST_BUCKET)
+            .key(test_key)
            .upload_id(upload_id)
            .part_number(part_number)
            .body(Bytes::from(part_data).into())
@@ -286,57 +242,62 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
    // Test 1: Multipart upload with wildcard If-None-Match, should fail
    let complete_response = client
        .complete_multipart_upload()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
        .upload_id(upload_id)
        .multipart_upload(completed_upload.clone())
        .if_none_match("*")
        .send()
        .await;

-    assert!(complete_response.is_err());
+    assert_s3_error_code(complete_response, "PreconditionFailed");

    // Test 2: Multipart upload with matching If-None-Match, should fail
    let complete_response = client
        .complete_multipart_upload()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
        .upload_id(upload_id)
        .multipart_upload(completed_upload.clone())
        .if_none_match(initial_etag.clone())
        .send()
        .await;

-    assert!(complete_response.is_err());
+    assert_s3_error_code(complete_response, "PreconditionFailed");

    // Test 3: Multipart upload with unmatching If-Match, should fail
    let complete_response = client
        .complete_multipart_upload()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
        .upload_id(upload_id)
        .multipart_upload(completed_upload.clone())
        .if_match("\"abcdef\"")
        .send()
        .await;

-    assert!(complete_response.is_err());
+    assert_s3_error_code(complete_response, "PreconditionFailed");
+
+    let staged_parts = client
+        .list_parts()
+        .bucket(TEST_BUCKET)
+        .key(test_key)
+        .upload_id(upload_id)
+        .send()
+        .await?;
+    assert_eq!(staged_parts.parts().len(), usize::try_from(num_parts)?);

    // Test 4: Multipart upload with matching If-Match, should succeed
-    let complete_response = client
+    client
        .complete_multipart_upload()
-        .bucket(BUCKET)
-        .key(&test_key)
+        .bucket(TEST_BUCKET)
+        .key(test_key)
        .upload_id(upload_id)
-        .multipart_upload(completed_upload.clone())
+        .multipart_upload(completed_upload)
        .if_match(initial_etag)
        .send()
-        .await;
-
-    assert!(complete_response.is_ok());
-
-    // Cleanup
-    cleanup_objects(&client, BUCKET, &[&test_key]).await;
+        .await?;
+    assert_eq!(object_body(&client, test_key).await?.as_ref(), expected_data);

    Ok(())
 }
@@ -13,207 +13,228 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use crate::common::workspace_root;
+use crate::common::RustFSTestEnvironment;
 use crate::storage_api::node_interact::{
    TonicInterceptor, VolumeInfo, WalkDirOptions, gen_tonic_signature_interceptor, node_service_time_out_client,
 };
-use futures::future::join_all;
+use aws_sdk_s3::primitives::ByteStream;
 use rmp_serde::{Deserializer, Serializer};
-use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
+use rustfs_filemeta::MetaCacheEntry;
 use rustfs_protos::proto_gen::node_service::WalkDirRequest;
 use rustfs_protos::{
    models::{PingBody, PingBodyBuilder},
-    proto_gen::node_service::{
-        ListVolumesRequest, LocalStorageInfoRequest, MakeVolumeRequest, PingRequest, PingResponse, ReadAllRequest,
-    },
+    proto_gen::node_service::{ListVolumesRequest, LocalStorageInfoRequest, MakeVolumeRequest, PingRequest, ReadAllRequest},
 };
 use serde::{Deserialize, Serialize};
 use std::error::Error;
 use std::io::Cursor;
-use std::path::PathBuf;
-use tokio::spawn;
 use tonic::Request;
 use tonic::codegen::tokio_stream::StreamExt;

-const CLUSTER_ADDR: &str = "http://localhost:9000";
+type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+
+const TEST_RPC_SECRET: &str = "rustfs-internode-signature-e2e-secret";

 fn signature_interceptor() -> TonicInterceptor {
    TonicInterceptor::Signature(gen_tonic_signature_interceptor())
 }

+fn rpc_client_error(error: Box<dyn Error>) -> std::io::Error {
+    std::io::Error::other(error.to_string())
+}
+
+async fn start_server() -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
+    let _ = rustfs_credentials::set_global_rpc_secret(TEST_RPC_SECRET.to_string());
+    let effective = rustfs_credentials::try_get_rpc_token().expect("RPC secret must resolve in the test process");
+    assert_eq!(effective, TEST_RPC_SECRET, "the test process uses an unexpected RPC secret");
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server_without_cleanup_with_env(&[
+        ("RUSTFS_RPC_SECRET", TEST_RPC_SECRET),
+        ("RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT", "false"),
+        ("RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT", "false"),
+        ("RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT", "false"),
+        ("RUST_LOG", "error"),
+    ])
+    .await?;
+    Ok(env)
+}
+
 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn ping() -> Result<(), Box<dyn Error>> {
+async fn ping() -> TestResult {
+    let env = start_server().await?;
    let mut fbb = flatbuffers::FlatBufferBuilder::new();
    let payload = fbb.create_vector(b"hello world");
-
    let mut builder = PingBodyBuilder::new(&mut fbb);
    builder.add_payload(payload);
    let root = builder.finish();
    fbb.finish(root, None);

-    let finished_data = fbb.finished_data();
-
-    let decoded_payload = flatbuffers::root::<PingBody>(finished_data);
-    assert!(decoded_payload.is_ok());
-
-    // Create client
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-
-    // Construct PingRequest
-    let request = Request::new(PingRequest {
-        version: 1,
-        body: bytes::Bytes::copy_from_slice(finished_data),
-    });
-
-    // Send request and get response
-    let response: PingResponse = client.ping(request).await?.into_inner();
-
-    // Print response
-    let ping_response_body = flatbuffers::root::<PingBody>(&response.body);
-    if let Err(e) = ping_response_body {
-        eprintln!("{e}");
-    } else {
-        println!("ping_resp:body(flatbuffer): {ping_response_body:?}");
-    }
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let response = client
+        .ping(Request::new(PingRequest {
+            version: 1,
+            body: bytes::Bytes::copy_from_slice(fbb.finished_data()),
+        }))
+        .await?
+        .into_inner();

+    assert_eq!(response.version, 1);
+    let body = flatbuffers::root::<PingBody>(&response.body)?;
+    assert_eq!(body.payload().expect("ping response must contain a payload").bytes(), b"hello, caller");
    Ok(())
 }

 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn make_volume() -> Result<(), Box<dyn Error>> {
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-    let request = Request::new(MakeVolumeRequest {
-        disk: "data".to_string(),
-        volume: "dandan".to_string(),
-    });
+async fn make_volume() -> TestResult {
+    let env = start_server().await?;
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let response = client
+        .make_volume(Request::new(MakeVolumeRequest {
+            disk: env.temp_dir.clone(),
+            volume: "node-rpc-volume".to_string(),
+        }))
+        .await?
+        .into_inner();

-    let response = client.make_volume(request).await?.into_inner();
-    if response.success {
-        println!("success");
-    } else {
-        println!("failed: {:?}", response.error);
-    }
+    assert!(response.success, "make_volume failed: {:?}", response.error);
+    assert!(std::path::Path::new(&env.temp_dir).join("node-rpc-volume").is_dir());
    Ok(())
 }

 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn list_volumes() -> Result<(), Box<dyn Error>> {
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-    let request = Request::new(ListVolumesRequest {
-        disk: "data".to_string(),
-    });
+async fn list_volumes() -> TestResult {
+    let env = start_server().await?;
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let created = client
+        .make_volume(Request::new(MakeVolumeRequest {
+            disk: env.temp_dir.clone(),
+            volume: "node-rpc-listed-volume".to_string(),
+        }))
+        .await?
+        .into_inner();
+    assert!(created.success, "make_volume failed: {:?}", created.error);

-    let response = client.list_volumes(request).await?.into_inner();
-    let volume_infos: Vec<VolumeInfo> = response
+    let response = client
+        .list_volumes(Request::new(ListVolumesRequest {
+            disk: env.temp_dir.clone(),
+        }))
+        .await?
+        .into_inner();
+    assert!(response.success, "list_volumes failed: {:?}", response.error);
+    let volumes = response
        .volume_infos
-        .into_iter()
-        .filter_map(|json_str| serde_json::from_str::<VolumeInfo>(&json_str).ok())
-        .collect();
-
-    println!("{volume_infos:?}");
+        .iter()
+        .map(|json| serde_json::from_str::<VolumeInfo>(json))
+        .collect::<Result<Vec<_>, _>>()?;
+    assert!(volumes.iter().any(|volume| volume.name == "node-rpc-listed-volume"));
    Ok(())
 }

 #[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn walk_dir() -> Result<(), Box<dyn Error>> {
-    println!("walk_dir");
-    // TODO: use writer
+async fn walk_dir() -> TestResult {
+    let env = start_server().await?;
+    let s3 = env.create_s3_client();
+    let bucket = "node-rpc-walk-bucket";
+    let key = "prefix/object.txt";
+    env.create_test_bucket(bucket).await?;
+    s3.put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"walk payload"))
+        .send()
+        .await?;
+
    let opts = WalkDirOptions {
-        bucket: "dandan".to_owned(),
-        base_dir: "".to_owned(),
+        bucket: bucket.to_string(),
        recursive: true,
        ..Default::default()
    };
-    let (rd, mut wr) = tokio::io::duplex(1024);
-    let mut buf = Vec::new();
-    opts.serialize(&mut Serializer::new(&mut buf))?;
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-    let disk_path = std::env::var_os("RUSTFS_DISK_PATH").map(PathBuf::from).unwrap_or_else(|| {
-        let mut path = workspace_root();
-        path.push("target");
-        path.push(if cfg!(debug_assertions) { "debug" } else { "release" });
-        path.push("data");
-        path
-    });
-    let request = Request::new(WalkDirRequest {
-        disk: disk_path.to_string_lossy().into_owned(),
-        walk_dir_options: buf.into(),
-    });
-    let mut response = client.walk_dir(request).await?.into_inner();
+    let mut encoded = Vec::new();
+    opts.serialize(&mut Serializer::new(&mut encoded))?;
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let mut stream = client
+        .walk_dir(Request::new(WalkDirRequest {
+            disk: env.temp_dir.clone(),
+            walk_dir_options: encoded.into(),
+        }))
+        .await?
+        .into_inner();

-    let job1 = spawn(async move {
-        let mut out = MetacacheWriter::new(&mut wr);
-        loop {
-            match response.next().await {
-                Some(Ok(resp)) => {
-                    if !resp.success {
-                        println!("{}", resp.error_info.unwrap_or_else(|| "".to_string()));
-                    }
-                    let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
-                        .map_err(|_e| std::io::Error::other(format!("Unexpected response: {response:?}")))
-                        .unwrap();
-                    out.write_obj(&entry).await.unwrap();
-                }
-                None => {
-                    let _ = out.close().await;
-                    break;
-                }
-                _ => {
-                    println!("Unexpected response: {response:?}");
-                    let _ = out.close().await;
-                    break;
-                }
-            }
-        }
-    });
-    let job2 = spawn(async move {
-        let mut reader = MetacacheReader::new(rd);
-        while let Ok(Some(entry)) = reader.peek().await {
-            println!("{entry:?}");
-        }
-    });
-
-    join_all(vec![job1, job2]).await;
-    Ok(())
-}
-
-#[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn read_all() -> Result<(), Box<dyn Error>> {
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-    let request = Request::new(ReadAllRequest {
-        disk: "data".to_string(),
-        volume: "ff".to_string(),
-        path: "format.json".to_string(),
-    });
-
-    let response = client.read_all(request).await?.into_inner();
-    let volume_infos = response.data;
-
-    println!("{}", response.success);
-    println!("{volume_infos:?}");
-    Ok(())
-}
-
-#[tokio::test]
-#[ignore = "requires running RustFS server at localhost:9000"]
-async fn storage_info() -> Result<(), Box<dyn Error>> {
-    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), signature_interceptor()).await?;
-    let request = Request::new(LocalStorageInfoRequest { metrics: true });
-
-    let response = client.local_storage_info(request).await?.into_inner();
-    if !response.success {
-        println!("{:?}", response.error_info);
-        return Ok(());
+    let mut entries = Vec::new();
+    while let Some(response) = stream.next().await {
+        let response = response?;
+        assert!(response.success, "walk_dir failed: {:?}", response.error_info);
+        entries.push(serde_json::from_str::<MetaCacheEntry>(&response.meta_cache_entry)?);
    }
-    let info = response.storage_info;
+    assert!(
+        entries.iter().any(|entry| entry.name == key),
+        "walk_dir did not return {key}: {entries:?}"
+    );
+    Ok(())
+}

-    let mut buf = Deserializer::new(Cursor::new(info));
-    let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut buf).unwrap();
-    println!("{storage_info:?}");
+#[tokio::test]
+async fn read_all() -> TestResult {
+    let env = start_server().await?;
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let volume = "node-rpc-read-volume";
+    let created = client
+        .make_volume(Request::new(MakeVolumeRequest {
+            disk: env.temp_dir.clone(),
+            volume: volume.to_string(),
+        }))
+        .await?
+        .into_inner();
+    assert!(created.success, "make_volume failed: {:?}", created.error);
+    tokio::fs::write(std::path::Path::new(&env.temp_dir).join(volume).join("payload.bin"), b"read payload").await?;
+
+    let response = client
+        .read_all(Request::new(ReadAllRequest {
+            disk: env.temp_dir.clone(),
+            volume: volume.to_string(),
+            path: "payload.bin".to_string(),
+        }))
+        .await?
+        .into_inner();
+    assert!(response.success, "read_all failed: {:?}", response.error);
+    assert_eq!(response.data.as_ref(), b"read payload");
+    Ok(())
+}
+
+#[tokio::test]
+async fn storage_info() -> TestResult {
+    let env = start_server().await?;
+    let mut client = node_service_time_out_client(&env.url, signature_interceptor())
+        .await
+        .map_err(rpc_client_error)?;
+    let response = client
+        .local_storage_info(Request::new(LocalStorageInfoRequest { metrics: true }))
+        .await?
+        .into_inner();
+    assert!(response.success, "local_storage_info failed: {:?}", response.error_info);
+
+    let mut decoder = Deserializer::new(Cursor::new(response.storage_info));
+    let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut decoder)?;
+    let expected_disk = std::fs::canonicalize(&env.temp_dir)?;
+    assert!(!storage_info.disks.is_empty(), "local_storage_info returned no disks");
+    assert!(
+        storage_info
+            .disks
+            .iter()
+            .any(|disk| std::path::Path::new(&disk.drive_path) == expected_disk),
+        "local_storage_info did not include the configured disk: {:?}",
+        storage_info.disks
+    );
    Ok(())
 }
@@ -13,9 +13,8 @@
 // limitations under the License.

 use crate::common::{
-    RustFSTestEnvironment, admin_create_user, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging,
-    local_http_client, replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client,
-    signed_request_with_session_token,
+    RustFSTestEnvironment, admin_create_user, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
+    replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client, signed_request_with_session_token,
 };
 use crate::fake_s3_target::{
    FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
@@ -57,7 +56,7 @@ use rustfs_madmin::{
    AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
    ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
 };
-use s3s::header::X_AMZ_REPLICATION_STATUS;
+use s3s::header::{X_AMZ_REPLICATION_STATUS, X_AMZ_TAGGING};
 use sha2::{Digest, Sha256};
 use std::collections::BTreeMap;
 use std::convert::Infallible;
@@ -2023,6 +2022,7 @@ async fn forward_replication_proxy_request(
    client: &reqwest::Client,
    request_count: &AtomicU64,
    mut replication_enabled: watch::Receiver<bool>,
+    mut held_tagging: watch::Receiver<Option<String>>,
 ) -> Response<Full<bytes::Bytes>> {
    let (parts, body) = request.into_parts();
    let is_replication = parts
@@ -2036,6 +2036,17 @@ async fn forward_replication_proxy_request(
                return proxy_error_response("replication gate closed");
            }
        }
+        // Content-keyed hold: park only the replication request whose
+        // `x-amz-tagging` matches the held value, letting every other delivery
+        // through, so a test can make one specific (stale) delivery the last
+        // write the backend sees.
+        if let Some(tagging) = parts.headers.get(X_AMZ_TAGGING).and_then(|value| value.to_str().ok()) {
+            while held_tagging.borrow().as_deref() == Some(tagging) {
+                if held_tagging.changed().await.is_err() {
+                    return proxy_error_response("replication tag hold closed");
+                }
+            }
+        }
    }

    let Some(path_and_query) = parts.uri.path_and_query() else {
@@ -2070,12 +2081,26 @@ async fn start_replication_counting_proxy(
    backend_url: &str,
    tasks: &mut JoinSet<()>,
 ) -> Result<(String, Arc<AtomicU64>, watch::Sender<bool>), Box<dyn Error + Send + Sync>> {
+    let (proxy_url, request_count, replication_enabled, _held_tagging) =
+        start_replication_counting_proxy_with_tag_hold(backend_url, tasks).await?;
+    Ok((proxy_url, request_count, replication_enabled))
+}
+
+/// [`start_replication_counting_proxy`] plus a content-keyed hold: while the
+/// returned `watch::Sender<Option<String>>` holds `Some(tagging)`, replication
+/// requests whose `x-amz-tagging` equals `tagging` are parked (and still
+/// counted); all other traffic flows. Send `None` to release them.
+async fn start_replication_counting_proxy_with_tag_hold(
+    backend_url: &str,
+    tasks: &mut JoinSet<()>,
+) -> Result<(String, Arc<AtomicU64>, watch::Sender<bool>, watch::Sender<Option<String>>), Box<dyn Error + Send + Sync>> {
    let listener = TcpListener::bind("127.0.0.1:0").await?;
    let proxy_url = format!("http://{}", listener.local_addr()?);
    let backend_url = backend_url.to_string();
    let request_count = Arc::new(AtomicU64::new(0));
    let task_request_count = request_count.clone();
    let (replication_enabled, task_replication_enabled) = watch::channel(true);
+    let (held_tagging, task_held_tagging) = watch::channel(None);
    tasks.spawn(async move {
        let client = local_http_client();
        let mut connections = JoinSet::new();
@@ -2087,12 +2112,14 @@ async fn start_replication_counting_proxy(
                    let client = client.clone();
                    let request_count = task_request_count.clone();
                    let replication_enabled = task_replication_enabled.clone();
+                    let held_tagging = task_held_tagging.clone();
                    connections.spawn(async move {
                        let service = service_fn(move |request| {
                            let backend_url = backend_url.clone();
                            let client = client.clone();
                            let request_count = request_count.clone();
                            let replication_enabled = replication_enabled.clone();
+                            let held_tagging = held_tagging.clone();
                            async move {
                                Ok::<_, Infallible>(
                                    forward_replication_proxy_request(
@@ -2101,6 +2128,7 @@ async fn start_replication_counting_proxy(
                                        &client,
                                        &request_count,
                                        replication_enabled,
+                                        held_tagging,
                                    )
                                    .await,
                                )
@@ -2113,7 +2141,7 @@ async fn start_replication_counting_proxy(
            }
        }
    });
-    Ok((proxy_url, request_count, replication_enabled))
+    Ok((proxy_url, request_count, replication_enabled, held_tagging))
 }

 async fn site_replication_remove(
@@ -6949,6 +6977,395 @@ async fn test_site_replication_active_active_converges_without_loops_real_dual_n
    }
 }

+/// Replication status a site reports for one object version via HEAD
+/// (`x-amz-replication-status`), or `None` when the header is absent.
+async fn head_replication_status(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
+    let head = client
+        .head_object()
+        .bucket(bucket)
+        .key(key)
+        .version_id(version_id)
+        .send()
+        .await?;
+    Ok(head.replication_status().map(|status| status.as_str().to_string()))
+}
+
+/// Poll one site until the version's replication status is one of `expected`.
+async fn wait_for_version_replication_status(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    expected: &[&str],
+    site: &str,
+) -> Result<String, Box<dyn Error + Send + Sync>> {
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
+    loop {
+        let last = head_replication_status(client, bucket, key, version_id).await?;
+        if let Some(status) = last.as_deref()
+            && expected.contains(&status)
+        {
+            return Ok(status.to_string());
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err(format!(
+                "{site}: {bucket}/{key}?versionId={version_id} replication status {last:?} never reached {expected:?}"
+            )
+            .into());
+        }
+        sleep(Duration::from_millis(200)).await;
+    }
+}
+
+async fn put_single_tag(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    tag_key: &str,
+    tag_value: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    client
+        .put_object_tagging()
+        .bucket(bucket)
+        .key(key)
+        .version_id(version_id)
+        .tagging(
+            aws_sdk_s3::types::Tagging::builder()
+                .tag_set(aws_sdk_s3::types::Tag::builder().key(tag_key).value(tag_value).build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn get_single_tag(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    tag_key: &str,
+) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
+    let tagging = client
+        .get_object_tagging()
+        .bucket(bucket)
+        .key(key)
+        .version_id(version_id)
+        .send()
+        .await?;
+    Ok(tagging
+        .tag_set()
+        .iter()
+        .find(|tag| tag.key() == tag_key)
+        .map(|tag| tag.value().to_string()))
+}
+
+/// Poll one site until the version's `tag_key` equals `expected`.
+async fn wait_for_single_tag(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    tag_key: &str,
+    expected: &str,
+    site: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
+    loop {
+        let observed = get_single_tag(client, bucket, key, version_id, tag_key).await?;
+        if observed.as_deref() == Some(expected) {
+            return Ok(());
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err(format!(
+                "{site}: {bucket}/{key}?versionId={version_id} tag {tag_key}={observed:?} never became {expected}"
+            )
+            .into());
+        }
+        sleep(Duration::from_millis(200)).await;
+    }
+}
+
+/// Tag key the dual-node LWW scenario edits on both sites.
+const LWW_TAG_KEY: &str = "owner";
+
+/// Assert the version's [`LWW_TAG_KEY`] stays `expected` on both sites for a
+/// full quiet window (no late stale delivery flips it back).
+async fn assert_tag_stable_on_both_sites(
+    site_a_client: &Client,
+    site_b_client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    expected: &str,
+    quiet: Duration,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let deadline = tokio::time::Instant::now() + quiet;
+    loop {
+        let on_a = get_single_tag(site_a_client, bucket, key, version_id, LWW_TAG_KEY).await?;
+        let on_b = get_single_tag(site_b_client, bucket, key, version_id, LWW_TAG_KEY).await?;
+        assert_eq!(on_a.as_deref(), Some(expected), "site A tag {LWW_TAG_KEY} regressed from the LWW winner");
+        assert_eq!(on_b.as_deref(), Some(expected), "site B tag {LWW_TAG_KEY} regressed from the LWW winner");
+        if tokio::time::Instant::now() >= deadline {
+            return Ok(());
+        }
+        sleep(Duration::from_millis(250)).await;
+    }
+}
+
+/// Wait until the counting proxy in front of a site has admitted `expected`
+/// replication requests in total (requests held by a closed gate still count).
+async fn wait_for_proxy_replication_requests(
+    counter: &AtomicU64,
+    expected: u64,
+    site: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
+    loop {
+        let observed = counter.load(Ordering::Relaxed);
+        if observed >= expected {
+            return Ok(());
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err(format!("{site} proxy saw {observed} replication requests, expected at least {expected}").into());
+        }
+        sleep(Duration::from_millis(25)).await;
+    }
+}
+
+/// rustfs/backlog#1953 (audit A4/P1-6): receiver-side LWW for replicated
+/// metadata categories, exercised end to end over the real dual-node
+/// active-active site-replication control plane — sender, worker, status
+/// bookkeeping and persisted failure recovery all participate (the single-server
+/// `replication_lww_receiver_test` only injects authorized replication PUTs).
+///
+/// Scenario on one versioned object:
+/// 1. reciprocal tag edits in real order (A then B) converge both sites on the
+///    newer tag and leave the author COMPLETED / the receiver REPLICA;
+/// 2. out-of-order delivery: A's edit is held at B's inbound proxy while B
+///    authors a newer edit that reaches A first; releasing the stale delivery
+///    must NOT roll B back — both sites settle on B's value and stay there
+///    through a quiet window, with no FAILED/PENDING status left behind;
+/// 3. persisted retry: B is stopped, A's delivery reaches FAILED, A restarts,
+///    then B returns and the scanner-replayed edit converges both sites forward.
+/// Durable metadata-MRF serialization/reconstruction is covered separately by
+/// `metadata_mrf_roundtrip_preserves_tags_and_admitted_targets`.
+#[tokio::test]
+async fn test_site_replication_tagging_lww_converges_active_active_real_dual_node() -> TestResult {
+    init_logging();
+
+    match tokio::time::timeout(Duration::from_secs(420), async {
+        // The scanner is fast for the final persisted-failure recovery phase.
+        // Step 2 finishes and proves a quiet stable winner before that phase,
+        // so a later scanner pass cannot mask its stale-delivery assertion.
+        let mut site_env = replication_fast_env();
+        site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
+        site_env.extend_from_slice(FAST_SCANNER_ENV);
+
+        let mut site_a_env = RustFSTestEnvironment::new().await?;
+        site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?;
+
+        let mut site_b_env = RustFSTestEnvironment::new().await?;
+        site_b_env.start_rustfs_server_with_env(vec![], &site_env).await?;
+
+        let mut proxy_tasks = JoinSet::new();
+        let (site_a_proxy, site_a_replication_requests, _site_a_replication_enabled, site_a_held_tagging) =
+            start_replication_counting_proxy_with_tag_hold(&site_a_env.url, &mut proxy_tasks).await?;
+        let (site_b_proxy, site_b_replication_requests, _site_b_replication_enabled, site_b_held_tagging) =
+            start_replication_counting_proxy_with_tag_hold(&site_b_env.url, &mut proxy_tasks).await?;
+
+        let site_a_client = site_a_env.create_s3_client();
+        let site_b_client = site_b_env.create_s3_client();
+        let bucket = "site-repl-tag-lww";
+        let key = "lww.txt";
+
+        let add_status = site_replication_add(
+            &site_a_env,
+            &[
+                PeerSite {
+                    name: "lww-site-a".to_string(),
+                    endpoint: site_a_env.url.clone(),
+                    access_key: site_a_env.access_key.clone(),
+                    secret_key: site_a_env.secret_key.clone(),
+                    ..Default::default()
+                },
+                PeerSite {
+                    name: "lww-site-b".to_string(),
+                    endpoint: site_b_env.url.clone(),
+                    access_key: site_b_env.access_key.clone(),
+                    secret_key: site_b_env.secret_key.clone(),
+                    ..Default::default()
+                },
+            ],
+        )
+        .await?;
+        assert!(add_status.success, "unexpected site add result: {add_status:?}");
+
+        let site_info = wait_for_site_replication_enabled(&site_a_env, 2).await?;
+        wait_for_site_replication_enabled(&site_b_env, 2).await?;
+
+        // Route both directions through the counting proxies so inbound
+        // replication to B can be held (out-of-order delivery) and observed.
+        for (env_url, proxy_url, label) in [(&site_a_env.url, &site_a_proxy, "A"), (&site_b_env.url, &site_b_proxy, "B")] {
+            let mut peer = site_info
+                .sites
+                .iter()
+                .find(|peer| peer.endpoint == *env_url)
+                .ok_or_else(|| format!("site {label} peer missing from replication info"))?
+                .clone();
+            peer.endpoint = proxy_url.clone();
+            peer.sync_state = SyncStatus::Enable;
+            let edit = site_replication_edit(&site_a_env, "", &peer).await?;
+            assert!(edit.success, "unexpected site {label} endpoint edit: {edit:?}");
+        }
+        for env in [&site_a_env, &site_b_env] {
+            wait_for_site_replication_info(env, |info| {
+                info.sites.iter().any(|peer| peer.endpoint == site_a_proxy)
+                    && info.sites.iter().any(|peer| peer.endpoint == site_b_proxy)
+            })
+            .await?;
+        }
+
+        site_a_client.create_bucket().bucket(bucket).send().await?;
+        wait_for_bucket_on_target(&site_b_client, bucket).await?;
+
+        let version_id = site_a_client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(b"tag lww payload"))
+            .send()
+            .await?
+            .version_id()
+            .ok_or("site A PUT omitted version ID")?
+            .to_string();
+        wait_for_replicated_object(&site_b_client, bucket, key, "tag lww payload").await?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
+        wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
+        wait_for_proxy_replication_requests(&site_b_replication_requests, 1, "site B").await?;
+
+        // --- 1. reciprocal edits in real order: A then B ----------------------
+        put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a1").await?;
+        wait_for_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "a1", "site B").await?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
+        wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
+        wait_for_proxy_replication_requests(&site_b_replication_requests, 2, "site B").await?;
+
+        put_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "b1").await?;
+        wait_for_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "b1", "site A").await?;
+        wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["COMPLETED"], "site B").await?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["REPLICA"], "site A").await?;
+        assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "b1", Duration::from_secs(3))
+            .await?;
+
+        // --- 2. concurrent edits, stale delivery last ------------------------
+        // Both sites edit the same version while each other's delivery is
+        // parked at the peer's inbound proxy (content-keyed: only the
+        // `owner=a2` / `owner=b2` replication PUTs wait, everything else
+        // flows). B's edit is the newer one. Releasing A's stale `a2` first
+        // makes it the last write B sees while A itself still holds `a2`, so
+        // nothing A could re-deliver carries the winner: only receiver-side
+        // LWW on B can keep `b2`. Releasing `b2` afterwards converges A.
+        site_b_held_tagging.send(Some("owner=a2".to_string()))?;
+        site_a_held_tagging.send(Some("owner=b2".to_string()))?;
+        let a2_parked_at = site_b_replication_requests.load(Ordering::Relaxed) + 1;
+        let b2_parked_at = site_a_replication_requests.load(Ordering::Relaxed) + 1;
+        put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a2").await?;
+        wait_for_proxy_replication_requests(&site_b_replication_requests, a2_parked_at, "site B").await?;
+        sleep(Duration::from_millis(50)).await;
+        put_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "b2").await?;
+        wait_for_proxy_replication_requests(&site_a_replication_requests, b2_parked_at, "site A").await?;
+        assert_eq!(
+            get_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY)
+                .await?
+                .as_deref(),
+            Some("a2")
+        );
+        assert_eq!(
+            get_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY)
+                .await?
+                .as_deref(),
+            Some("b2")
+        );
+
+        // Release the stale a2 delivery onto B: the newer local b2 must
+        // survive, and the delivery itself must still succeed (A reaches
+        // COMPLETED instead of looping through MRF with the stale value).
+        site_b_held_tagging.send(None)?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
+        let stale_deadline = tokio::time::Instant::now() + Duration::from_secs(3);
+        loop {
+            assert_eq!(
+                get_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY)
+                    .await?
+                    .as_deref(),
+                Some("b2"),
+                "a stale inbound delivery rolled back site B's newer tag (receiver-side LWW regression)"
+            );
+            if tokio::time::Instant::now() >= stale_deadline {
+                break;
+            }
+            sleep(Duration::from_millis(250)).await;
+        }
+
+        // Release b2 onto A: the newer edit wins there and both sites settle.
+        // B's own version may legitimately read REPLICA here: the stale inbound
+        // a2 write re-labelled it as a replica write (keeping B's tags); what
+        // must not remain is PENDING/FAILED.
+        site_a_held_tagging.send(None)?;
+        wait_for_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "b2", "site A").await?;
+        wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["COMPLETED", "REPLICA"], "site B")
+            .await?;
+        assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "b2", Duration::from_secs(4))
+            .await?;
+        for (client, site) in [(&site_a_client, "site A"), (&site_b_client, "site B")] {
+            let status = head_replication_status(client, bucket, key, &version_id).await?;
+            assert!(
+                matches!(status.as_deref(), Some("COMPLETED" | "REPLICA")),
+                "{site} must not be left PENDING/FAILED after the concurrent edits: {status:?}"
+            );
+        }
+
+        // --- 3. persisted FAILED state survives a source restart ------------
+        site_b_env.stop_server();
+        put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a3").await?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["FAILED"], "site A").await?;
+        site_a_env.restart_server_preserving_data(vec![], &site_env).await?;
+        wait_for_site_replication_enabled(&site_a_env, 2).await?;
+        site_b_env.restart_server_preserving_data(vec![], &site_env).await?;
+        wait_for_site_replication_enabled(&site_b_env, 2).await?;
+
+        wait_for_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "a3", "site B").await?;
+        wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?;
+        wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?;
+        assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "a3", Duration::from_secs(3))
+            .await?;
+
+        // The object itself never forked: one version on each side.
+        tokio::time::timeout(
+            Duration::from_secs(70),
+            assert_replication_converged(&site_a_client, bucket, &site_b_client, bucket),
+        )
+        .await??;
+        let state = list_replication_state(&site_a_client, bucket).await?;
+        assert_eq!(state.len(), 1, "tag edits must not create new object versions: {state:?}");
+        assert_eq!(state[0].version_id, version_id);
+
+        proxy_tasks.abort_all();
+        Ok(())
+    })
+    .await
+    {
+        Ok(result) => result,
+        Err(_) => Err("site replication tagging LWW test timed out".into()),
+    }
+}
 #[tokio::test]
 async fn test_site_replication_replicates_policy_backed_user_access_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
    init_logging();
@@ -7281,11 +7698,6 @@ async fn test_site_replication_replicates_multiple_service_accounts_real_dual_no
 async fn test_site_replication_replicates_service_accounts_created_from_sts_session_real_dual_node() -> TestResult {
    init_logging();

-    if !awscurl_available() {
-        eprintln!("Skipping STS site replication service-account test because awscurl is unavailable");
-        return Ok(());
-    }
-
    let mut source_env = RustFSTestEnvironment::new().await?;
    source_env
        .start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)
@@ -0,0 +1,155 @@
+#![cfg(test)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Receiver-side replication LWW over the wire (rustfs/backlog#1953, audit
+//! A4/P1-6).
+//!
+//! In an active-active topology both sites' metadata states arrive at the
+//! peer as authorized replication PUTs carrying per-category source
+//! timestamps (`x-rustfs-source-replication-tagging-timestamp` header
+//! family). Before the fix the receiver applied them unconditionally, so a
+//! stale delivery overwrote a newer local state and the two sites diverged
+//! permanently while both reported COMPLETED. This test drives one live
+//! `rustfs` server with simulated inbound replication PUTs for the same
+//! object version and asserts the newer tagging state wins regardless of
+//! delivery order, while a stale delivery still succeeds at the object level
+//! (a failure would loop through MRF re-delivering the stale value).
+//!
+//! The real dual-site path (sender, worker, status bookkeeping, MRF replay)
+//! is covered by
+//! `replication_extension_test::test_site_replication_tagging_lww_converges_active_active_real_dual_node`;
+//! this file stays as the fast, single-process receiver check.
+
+use crate::common::{RustFSTestEnvironment, init_logging};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+
+type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+
+const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
+const HDR_SOURCE_VERSION_ID: &str = "x-rustfs-source-version-id";
+const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
+const HDR_SOURCE_TAGGING_TIMESTAMP: &str = "x-rustfs-source-replication-tagging-timestamp";
+
+const SOURCE_MTIME: &str = "2026-01-01T00:00:00Z";
+const T_STALE: &str = "2026-01-01T00:00:01Z";
+const T_LOCAL: &str = "2026-02-01T00:00:00Z";
+const T_NEWER: &str = "2026-03-01T00:00:00Z";
+
+/// Simulated inbound authorized replication PUT: same object version, tags and
+/// the source-authored tagging timestamp carried in transport headers.
+async fn inbound_replication_put(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    tags: &str,
+    tagging_timestamp: &str,
+) -> TestResult {
+    let version_id = version_id.to_string();
+    let tagging_timestamp = tagging_timestamp.to_string();
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"lww-e2e-body"))
+        .tagging(tags)
+        .customize()
+        .mutate_request(move |req| {
+            req.headers_mut().insert(HDR_SOURCE_REPLICATION_REQUEST, "true");
+            req.headers_mut().insert(HDR_SOURCE_VERSION_ID, version_id.clone());
+            req.headers_mut().insert(HDR_SOURCE_MTIME, SOURCE_MTIME);
+            req.headers_mut()
+                .insert(HDR_SOURCE_TAGGING_TIMESTAMP, tagging_timestamp.clone());
+        })
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn tag_value(client: &Client, bucket: &str, key: &str, version_id: &str, tag_key: &str) -> Option<String> {
+    let tagging = client
+        .get_object_tagging()
+        .bucket(bucket)
+        .key(key)
+        .version_id(version_id)
+        .send()
+        .await
+        .expect("object tagging should be readable");
+    tagging
+        .tag_set()
+        .iter()
+        .find(|tag| tag.key() == tag_key)
+        .map(|tag| tag.value().to_string())
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn receiver_lww_keeps_newer_tags_across_delivery_orders() -> TestResult {
+    init_logging();
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+    let client = env.create_s3_client();
+
+    let bucket = "replication-lww-receiver";
+    let key = "object";
+    client.create_bucket().bucket(bucket).send().await?;
+    client
+        .put_bucket_versioning()
+        .bucket(bucket)
+        .versioning_configuration(
+            VersioningConfiguration::builder()
+                .status(BucketVersioningStatus::Enabled)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    // First delivery establishes version V with tags stamped T_LOCAL.
+    let version_id = uuid::Uuid::new_v4().to_string();
+    inbound_replication_put(&client, bucket, key, &version_id, "site=local", T_LOCAL).await?;
+    assert_eq!(
+        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
+        Some("local"),
+        "the first delivery must establish the tagged version"
+    );
+
+    // A stale delivery (older source timestamp) must succeed at the object
+    // level but must NOT overwrite the newer tags.
+    inbound_replication_put(&client, bucket, key, &version_id, "site=stale", T_STALE).await?;
+    assert_eq!(
+        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
+        Some("local"),
+        "a stale inbound delivery must not overwrite newer tags (rustfs/backlog#1953)"
+    );
+
+    // A newer delivery still converges the version onto the newest state.
+    inbound_replication_put(&client, bucket, key, &version_id, "site=newer", T_NEWER).await?;
+    assert_eq!(
+        tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
+        Some("newer"),
+        "a newer inbound delivery must overwrite older tags"
+    );
+
+    client
+        .delete_object()
+        .bucket(bucket)
+        .key(key)
+        .version_id(&version_id)
+        .send()
+        .await?;
+    env.delete_test_bucket(bucket).await.ok();
+    Ok(())
+}
@@ -21,12 +21,11 @@
 //! - SSRF prevention (internal/private endpoints rejected for tiering)
 //! - Race condition handling (concurrent writes converge without corruption)

-use crate::common::{RustFSTestEnvironment, awscurl_available, awscurl_put, init_logging};
+use crate::common::{RustFSTestEnvironment, awscurl_put, init_logging, require_awscurl};
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, Tag, Tagging};
 use std::error::Error;
-use tracing::info;

 /// Oversized tagging payloads must be rejected by the per-object tag limit.
 ///
@@ -225,16 +224,12 @@ async fn test_concurrent_object_operations() -> Result<(), Box<dyn Error + Send
 /// outcome — the internal endpoint is not accepted — is asserted here.
 ///
 /// The admin API is exercised via signed `awscurl` requests, matching the
-/// pattern used by the other admin-API E2E tests in this crate; the test is
-/// skipped when `awscurl` is not installed.
+/// pattern used by the other admin-API E2E tests in this crate. The full E2E
+/// lane installs and verifies the pinned `awscurl` prerequisite.
 #[tokio::test]
 async fn test_tiering_url_validation() -> Result<(), Box<dyn Error + Send + Sync>> {
    init_logging();
-    if !awscurl_available() {
-        info!("Skipping tiering URL validation test because awscurl is not available");
-        return Ok(());
-    }
-
+    require_awscurl()?;
    let mut env = RustFSTestEnvironment::new().await?;
    env.start_rustfs_server(vec![]).await?;

@@ -15,7 +15,6 @@
 use crate::common::{RustFSTestClusterEnvironment, init_logging};
 use aws_sdk_s3::error::SdkError;
 use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::CompletedMultipartUpload;
 use tokio::time::{Duration, sleep};
 use tracing::info;
 use uuid::Uuid;
@@ -43,32 +42,18 @@ async fn list_parts_reports_missing_upload(
    }
 }

-async fn complete_reports_missing_upload(
+async fn multipart_listing_reports_missing_upload(
    client: &aws_sdk_s3::Client,
    bucket: &str,
    key: &str,
    upload_id: &str,
 ) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
-    let result = client
-        .complete_multipart_upload()
-        .bucket(bucket)
-        .key(key)
-        .upload_id(upload_id)
-        .multipart_upload(CompletedMultipartUpload::builder().build())
-        .send()
-        .await;
-    match result {
-        Ok(_) => Ok(false),
-        Err(SdkError::ServiceError(err)) => {
-            let code = err.err().meta().code().unwrap_or("");
-            if code == "NoSuchUpload" {
-                Ok(true)
-            } else {
-                Err(format!("unexpected complete_multipart_upload service error: code={code}, err={err:?}").into())
-            }
-        }
-        Err(err) => Err(format!("unexpected complete_multipart_upload error: {err:?}").into()),
-    }
+    let result = client.list_multipart_uploads().bucket(bucket).prefix(key).send().await?;
+
+    Ok(!result
+        .uploads()
+        .iter()
+        .any(|upload| upload.key() == Some(key) && upload.upload_id() == Some(upload_id)))
 }

 async fn wait_for_cleanup_on_all_nodes(
@@ -81,8 +66,8 @@ async fn wait_for_cleanup_on_all_nodes(
        let mut all_cleaned = true;
        for (idx, client) in clients.iter().enumerate() {
            let list_parts_missing = list_parts_reports_missing_upload(client, bucket, key, upload_id).await?;
-            let complete_missing = complete_reports_missing_upload(client, bucket, key, upload_id).await?;
-            if !(list_parts_missing && complete_missing) {
+            let listing_missing = multipart_listing_reports_missing_upload(client, bucket, key, upload_id).await?;
+            if !(list_parts_missing && listing_missing) {
                info!("stale multipart still visible on node {} at attempt {}", idx, attempt + 1);
                all_cleaned = false;
                break;
@@ -146,6 +131,10 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
        1,
        "multipart upload should be visible before background cleanup"
    );
+    assert!(
+        !multipart_listing_reports_missing_upload(&clients[2], CLEANUP_BUCKET, &key, &upload_id).await?,
+        "multipart upload listing should contain the upload before background cleanup"
+    );

    wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?;

@@ -161,7 +161,7 @@ pub mod bucket {
        pub mod objectlock_sys {
            pub use crate::bucket::object_lock::objectlock_sys::{
                BucketObjectLockSys, ObjectLockBlockReason, add_years, check_object_lock_for_deletion,
-                check_retention_for_modification, is_retention_active,
+                check_retention_for_modification, is_retention_active, replication_write_may_pass_worm_gate,
            };
        }
    }
@@ -187,22 +187,24 @@ pub mod bucket {
        pub use crate::bucket::replication::{
            BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats,
            DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric,
-            MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION,
-            REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE,
-            REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
-            ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
+            MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, OperatorRuleContract,
+            REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
+            REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
+            REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
            ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
            ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
            ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
            ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
            ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
-            VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
-            delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
-            get_global_replication_stats, get_proxy_targets, init_background_replication,
-            invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog,
-            replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
-            resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
-            should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
+            VersionPurgeStatusType, XferStats, assign_site_replication_rule_priorities, commit_force_delete_intent,
+            complete_force_delete_intent, delete_replication_state_from_config, delete_replication_version_id,
+            get_global_replication_pool, get_global_replication_stats, get_proxy_targets, init_background_replication,
+            invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule,
+            merge_incoming_replication_config, merge_user_replication_config, persist_force_delete_intent,
+            read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map,
+            replication_target_arn_deployment_id, replication_target_arns, resync_start_conflict_id,
+            should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
+            should_use_existing_delete_replication_source, site_replication_rule_deployment_id,
            unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
            version_purge_status_to_filemeta,
        };
@@ -317,8 +319,6 @@ pub mod config {
 }

 pub mod data_usage {
-    #[cfg(feature = "test-util")]
-    pub use crate::data_usage::seed_bucket_usage_memory_for_test;
    pub use crate::data_usage::{
        DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
        init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
@@ -330,6 +330,8 @@ pub mod data_usage {
        remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
        store_data_usage_in_backend,
    };
+    #[cfg(feature = "test-util")]
+    pub use crate::data_usage::{get_bucket_usage_memory, seed_bucket_usage_memory_for_test};
 }

 pub mod disk {
@@ -456,11 +458,13 @@ pub mod rpc {
        ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
        check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
        gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
-        normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_put_file_capability,
-        sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
-        tonic_rpc_auth_failure_reason, verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature,
-        verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
-        verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
+        normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
+        sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
+        tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
+        verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
+        verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
+        verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+        verify_tonic_rpc_signature_with_bootstrap,
    };
 }

@@ -58,8 +58,8 @@ use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RU
 use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url};
 use rustfs_utils::http::{
    AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
-    AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header, is_minio_header,
-    is_rustfs_header, is_standard_header, is_storageclass_header,
+    AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING_LOWER, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header,
+    is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header,
 };
 use rustfs_utils::http::{
    SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_PROXY_REQUEST,
@@ -1774,6 +1774,22 @@ impl PutObjectOptions {
            Self::insert_checked(&mut header, AMZ_BUCKET_REPLICATION_STATUS, self.internal.replication_status.as_str());
        }

+        // MinIO PutObjectOptions.Header parity: object tags travel on the
+        // `x-amz-tagging` header (form-urlencoded). `replication_put_object_options`
+        // fills `user_tags` from the source version; without this header the
+        // whole-object transport delivered a tagless replica, so tag edits
+        // never reached the peer and the receiver-side LWW comparison
+        // (rustfs/backlog#1953) had nothing to judge.
+        if !self.user_tags.is_empty() {
+            let mut tags: Vec<(&String, &String)> = self.user_tags.iter().collect();
+            tags.sort();
+            let mut encoded = url::form_urlencoded::Serializer::new(String::new());
+            for (key, value) in tags {
+                encoded.append_pair(key, value);
+            }
+            Self::insert_checked(&mut header, AMZ_OBJECT_TAGGING_LOWER, &encoded.finish());
+        }
+
        for (k, v) in &self.user_metadata {
            let Ok(header_value) = HeaderValue::from_str(v) else {
                warn!("skipping user metadata header with invalid value: {}", k);
@@ -3195,6 +3211,29 @@ mod tests {
        }
    }

+    #[test]
+    fn put_object_headers_carry_user_tags_on_x_amz_tagging() {
+        // rustfs/backlog#1953: tag edits replicate through the whole-object
+        // transport, so the source tags must travel on x-amz-tagging.
+        let mut opts = PutObjectOptions::default();
+        opts.user_tags.insert("owner".to_string(), "site a".to_string());
+        opts.user_tags.insert("env".to_string(), "prod".to_string());
+
+        let header = opts.header();
+        let tagging = header
+            .get(AMZ_OBJECT_TAGGING_LOWER)
+            .expect("user tags must be transported on x-amz-tagging")
+            .to_str()
+            .expect("tag header must be ASCII");
+        // Deterministic key order; values are form-urlencoded.
+        assert_eq!(tagging, "env=prod&owner=site+a");
+
+        assert!(
+            PutObjectOptions::default().header().get(AMZ_OBJECT_TAGGING_LOWER).is_none(),
+            "a tagless source must not send an empty x-amz-tagging header"
+        );
+    }
+
    #[test]
    fn put_object_headers_omit_unset_replication_timestamps() {
        // UNIX_EPOCH means "never modified on the source"; sending it would
@@ -3425,6 +3464,44 @@ mod tests {
        assert!(mutexes.contains_key("second"));
    }

+    #[tokio::test]
+    async fn update_all_targets_publishes_disable_proxy_on_target_client() {
+        // The read-proxy selector (replication_proxy::get_proxy_targets) skips
+        // targets whose TargetClient carries disable_proxy — the persisted
+        // per-target opt-out must survive client publication.
+        let sys = BucketTargetSys::default();
+        let target = |arn: &str, disable_proxy: bool| BucketTarget {
+            arn: arn.to_string(),
+            endpoint: "192.168.1.10:9000".to_string(),
+            target_bucket: "target-bucket".to_string(),
+            region: "us-east-1".to_string(),
+            disable_proxy,
+            credentials: Some(Credentials {
+                access_key: "access".to_string(),
+                secret_key: "secret".to_string(),
+                session_token: None,
+                expiration: None,
+            }),
+            ..Default::default()
+        };
+        let targets = BucketTargets {
+            targets: vec![target("arn:proxied", false), target("arn:opted-out", true)],
+        };
+
+        sys.update_all_targets("bucket", Some(&targets)).await;
+
+        let proxied = sys
+            .get_remote_target_client("bucket", "arn:proxied")
+            .await
+            .expect("client should be published");
+        assert!(!proxied.disable_proxy);
+        let opted_out = sys
+            .get_remote_target_client("bucket", "arn:opted-out")
+            .await
+            .expect("client should be published");
+        assert!(opted_out.disable_proxy, "disable_proxy must reach the published TargetClient");
+    }
+
    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn target_updates_serialize_client_build_through_publication_per_bucket() {
        let sys = Arc::new(BucketTargetSys::default());
@@ -3739,17 +3739,55 @@ impl ManualTransitionRunReport {
    }

    pub fn merge_scan_report_preserving_worker(&mut self, scan_report: &ManualTransitionRunReport) {
+        let previous = self.clone();
+        let resumed_after_checkpoint = previous.continuation_token.is_some() && scan_report.scanned < previous.scanned;
        let mut tier_failure_by_reason = self.tier_failure_by_reason.clone();
        for (reason, count) in &scan_report.tier_failure_by_reason {
            let current = tier_failure_by_reason.get(reason).copied().unwrap_or_default();
-            tier_failure_by_reason.insert(*reason, current.max(*count));
+            let merged = if resumed_after_checkpoint {
+                current.saturating_add(*count)
+            } else {
+                current.max(*count)
+            };
+            tier_failure_by_reason.insert(*reason, merged);
        }
        let transition_completed = self.transition_completed;
        let transition_failed = self.transition_failed;
        *self = scan_report.clone();
+        if resumed_after_checkpoint {
+            self.scanned = previous.scanned.saturating_add(scan_report.scanned);
+            self.eligible = previous.eligible.saturating_add(scan_report.eligible);
+            self.enqueued = previous.enqueued.saturating_add(scan_report.enqueued);
+            self.dry_run_eligible = previous.dry_run_eligible.saturating_add(scan_report.dry_run_eligible);
+            self.skipped_not_transition = previous
+                .skipped_not_transition
+                .saturating_add(scan_report.skipped_not_transition);
+            self.skipped_tier = previous.skipped_tier.saturating_add(scan_report.skipped_tier);
+            self.skipped_delete_marker = previous
+                .skipped_delete_marker
+                .saturating_add(scan_report.skipped_delete_marker);
+            self.skipped_directory = previous.skipped_directory.saturating_add(scan_report.skipped_directory);
+            self.skipped_replication = previous.skipped_replication.saturating_add(scan_report.skipped_replication);
+            self.skipped_already_transitioned = previous
+                .skipped_already_transitioned
+                .saturating_add(scan_report.skipped_already_transitioned);
+            self.skipped_already_in_flight = previous
+                .skipped_already_in_flight
+                .saturating_add(scan_report.skipped_already_in_flight);
+            self.skipped_queue_full = previous.skipped_queue_full.saturating_add(scan_report.skipped_queue_full);
+            self.skipped_queue_closed = previous.skipped_queue_closed.saturating_add(scan_report.skipped_queue_closed);
+            self.skipped_queue_timeout = previous
+                .skipped_queue_timeout
+                .saturating_add(scan_report.skipped_queue_timeout);
+            self.tier_failure = previous.tier_failure.saturating_add(scan_report.tier_failure);
+        }
+        self.lifecycle_config_found = previous.lifecycle_config_found || scan_report.lifecycle_config_found;
+        self.truncated_by_limit = previous.truncated_by_limit || scan_report.truncated_by_limit;
+        self.truncated_by_duration = previous.truncated_by_duration || scan_report.truncated_by_duration;
+        self.cancelled = previous.cancelled || scan_report.cancelled;
        self.transition_completed = transition_completed;
        self.transition_failed = transition_failed;
-        self.tier_failure = scan_report.tier_failure.saturating_add(transition_failed);
+        self.tier_failure = self.tier_failure.saturating_add(transition_failed);
        self.tier_failure_by_reason = tier_failure_by_reason;
    }

@@ -3765,7 +3803,10 @@ struct ManualTransitionContinuationToken {
    version_marker: Option<String>,
 }

-fn encode_manual_transition_continuation_token(marker: Option<String>, version_marker: Option<String>) -> Option<String> {
+pub(super) fn encode_manual_transition_continuation_token(
+    marker: Option<String>,
+    version_marker: Option<String>,
+) -> Option<String> {
    if marker.is_none() && version_marker.is_none() {
        return None;
    }
@@ -9136,6 +9177,7 @@ mod tests {
        assert_eq!(loaded.report.scanned, 37);
        assert_eq!(loaded.report.eligible, 11);
        assert_eq!(loaded.report.enqueued, 5);
+        assert_eq!(loaded.cursor_revision, Some(37));
        assert!(loaded.lease_expires_at_unix_nanos > 0);
        let token = loaded
            .report
@@ -9154,6 +9196,30 @@ mod tests {
        assert_eq!(admission.lease_id, loaded.lease_id);
        assert_eq!(admission.lease_expires_at_unix_nanos, loaded.lease_expires_at_unix_nanos);

+        let mut same_marker_report = report.clone();
+        same_marker_report.scanned += 1;
+        persist_manual_transition_page_checkpoint(
+            &checkpoint_options,
+            &same_marker_report,
+            Some("logs/page-end".to_string()),
+            Some("opaque-next-version".to_string()),
+        )
+        .await
+        .expect("same-marker version checkpoint should persist through the durable progress sink");
+        let same_marker_checkpointed = load_manual_transition_job_record(ecstore.clone(), job_id)
+            .await
+            .expect("same-marker version checkpoint should reload");
+        assert_eq!(same_marker_checkpointed.cursor_revision, Some(38));
+        let (_, version_marker) = decode_manual_transition_continuation_token(
+            same_marker_checkpointed
+                .report
+                .continuation_token
+                .as_deref()
+                .expect("same-marker version checkpoint should persist a cursor"),
+        )
+        .expect("same-marker version cursor should decode");
+        assert_eq!(version_marker.as_deref(), Some("opaque-next-version"));
+
        create_test_bucket(&ecstore, &bucket).await;
        let lifecycle_xml = format!(
            r#"<?xml version="1.0" encoding="UTF-8"?>
@@ -9210,6 +9276,7 @@ mod tests {
        assert_eq!(checkpointed.report.scanned, 1000);
        assert_eq!(checkpointed.report.eligible, 1000);
        assert_eq!(checkpointed.report.dry_run_eligible, 1000);
+        assert_eq!(checkpointed.cursor_revision, Some(1000));
        let token = checkpointed
            .report
            .continuation_token
@@ -20,10 +20,16 @@ use serde::{Deserialize, Serialize};
 use time::OffsetDateTime;
 use uuid::Uuid;

+#[cfg(test)]
+use crate::bucket::lifecycle::bucket_lifecycle_ops::encode_manual_transition_continuation_token;
 use crate::bucket::lifecycle::bucket_lifecycle_ops::{
    ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport,
 };
 use crate::bucket::lifecycle::config_boundary;
+use crate::bucket::lifecycle::durable_namespace::{
+    MANUAL_TRANSITION_JOB_NAMESPACE, MANUAL_TRANSITION_SCOPE_NAMESPACE, MANUAL_TRANSITION_TASK_NAMESPACE,
+    MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
+};
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result as EcstoreResult};
 use crate::object_api::ObjectOptions;
@@ -34,10 +40,10 @@ use crate::store::ECStore;
 pub const MANUAL_TRANSITION_JOB_SCHEMA: &str = "rustfs-manual-transition-job-v1";
 pub const MANUAL_TRANSITION_TASK_SCHEMA: &str = "rustfs-manual-transition-task-v1";
 pub const MANUAL_TRANSITION_WORKER_RESULT_SCHEMA: &str = "rustfs-manual-transition-worker-result-v1";
-pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = "ilm/manual-transition/jobs";
-pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = "ilm/manual-transition/scopes";
-pub const MANUAL_TRANSITION_TASK_PREFIX: &str = "ilm/manual-transition/tasks";
-pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = "ilm/manual-transition/results";
+pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = MANUAL_TRANSITION_JOB_NAMESPACE.prefix;
+pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = MANUAL_TRANSITION_SCOPE_NAMESPACE.prefix;
+pub const MANUAL_TRANSITION_TASK_PREFIX: &str = MANUAL_TRANSITION_TASK_NAMESPACE.prefix;
+pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE.prefix;
 pub const MAX_MANUAL_TRANSITION_JOB_RECORD_SIZE: usize = 64 * 1024;
 pub const MAX_MANUAL_TRANSITION_TASK_RECORD_SIZE: usize = 16 * 1024;
 pub const MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE: usize = 8 * 1024;
@@ -195,6 +201,8 @@ pub struct ManualTransitionJobRecord {
    pub updated_at_unix_nanos: i128,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub completed_at_unix_nanos: Option<i128>,
+    #[serde(default, skip_serializing)]
+    pub cursor_revision: Option<u64>,
    pub report: ManualTransitionRunReport,
    pub queue_snapshot: ManualTransitionQueueSnapshot,
    #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -224,6 +232,7 @@ impl ManualTransitionJobRecord {
            created_at_unix_nanos: now,
            updated_at_unix_nanos: now,
            completed_at_unix_nanos: None,
+            cursor_revision: None,
            report: ManualTransitionRunReport {
                bucket: bucket.to_string(),
                prefix: options.prefix.clone(),
@@ -238,7 +247,7 @@ impl ManualTransitionJobRecord {

    pub fn complete(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
        self.scan_completed = true;
-        self.report.merge_scan_report_preserving_worker(&report);
+        self.merge_scan_report(&report);
        self.queue_snapshot = queue_snapshot;
        self.error = None;
        self.mark_terminal_if_worker_drained();
@@ -316,7 +325,7 @@ impl ManualTransitionJobRecord {
            }
        }
        self.queue_snapshot = queue_snapshot;
-        self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+        self.advance_updated_at();
        self.mark_terminal_if_worker_drained();
    }

@@ -359,14 +368,14 @@ impl ManualTransitionJobRecord {
        self.report.tier_failure = scan_tier_failure.saturating_add(transition_failed);
        self.report.tier_failure_by_reason = scan_tier_failure_by_reason;
        self.queue_snapshot = queue_snapshot;
-        self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+        self.advance_updated_at();
        self.mark_terminal_if_worker_drained();
        true
    }

    pub fn mark_cancel_requested(&mut self) {
        self.cancel_requested = true;
-        self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+        self.advance_updated_at();
    }

    pub fn claim_recovery_lease(&mut self, owner_id: impl Into<String>, queue_snapshot: ManualTransitionQueueSnapshot) {
@@ -380,7 +389,7 @@ impl ManualTransitionJobRecord {
    pub fn abandon_recovery_lease(&mut self, lease_id: Uuid) {
        if self.state == ManualTransitionJobState::Running && self.lease_id == lease_id {
            self.lease_expires_at_unix_nanos = 0;
-            self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
+            self.advance_updated_at();
        }
    }

@@ -398,7 +407,7 @@ impl ManualTransitionJobRecord {

    pub fn renew_lease(&mut self, queue_snapshot: ManualTransitionQueueSnapshot) {
        let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
-        self.updated_at_unix_nanos = now;
+        self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
        self.lease_expires_at_unix_nanos = manual_transition_job_lease_expires_at(now);
        self.queue_snapshot = queue_snapshot;
    }
@@ -438,11 +447,16 @@ impl ManualTransitionJobRecord {

    pub fn update_running_progress(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
        if self.state == ManualTransitionJobState::Running {
-            self.report.merge_scan_report_preserving_worker(&report);
+            self.merge_scan_report(&report);
            self.renew_lease(queue_snapshot);
        }
    }

+    fn merge_scan_report(&mut self, report: &ManualTransitionRunReport) {
+        self.report.merge_scan_report_preserving_worker(report);
+        self.cursor_revision = manual_transition_cursor_revision(&self.report);
+    }
+
    pub fn mark_unknown_if_unowned(&mut self) {
        if self.state == ManualTransitionJobState::Running {
            self.state = ManualTransitionJobState::Unknown;
@@ -463,9 +477,13 @@ impl ManualTransitionJobRecord {
    }

    fn mark_updated_terminal(&mut self) {
+        self.advance_updated_at();
+        self.completed_at_unix_nanos = Some(self.updated_at_unix_nanos);
+    }
+
+    fn advance_updated_at(&mut self) {
        let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
-        self.updated_at_unix_nanos = now;
-        self.completed_at_unix_nanos = Some(now);
+        self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
    }

    fn mark_terminal_if_worker_drained(&mut self) {
@@ -543,6 +561,7 @@ impl ManualTransitionJobRecord {
        if job.state == ManualTransitionJobState::Cancelled && job.cancel_requested {
            job.report.cancelled = true;
        }
+        job.cursor_revision = manual_transition_cursor_revision(&job.report);
        job.validate()?;
        Ok(job)
    }
@@ -1109,7 +1128,8 @@ pub fn manual_transition_scope_record_object_name(scope_key: &str) -> Result<Str
 pub async fn save_manual_transition_job_record(api: Arc<ECStore>, job: &ManualTransitionJobRecord) -> EcstoreResult<()> {
    let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
    let data = job.encode().map_err(manual_transition_job_store_error)?;
-    config_boundary::save_config(api, &object, data).await
+    config_boundary::save_config(api.clone(), &object, data.clone()).await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }

 pub async fn load_manual_transition_job_record(api: Arc<ECStore>, job_id: Uuid) -> EcstoreResult<ManualTransitionJobRecord> {
@@ -1142,9 +1162,9 @@ pub async fn save_manual_transition_job_record_if_current(
    let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
    let data = job.encode().map_err(manual_transition_job_store_error)?;
    config_boundary::save_config_with_opts_quiet(
-        api,
+        api.clone(),
        &object,
-        data,
+        data.clone(),
        &ObjectOptions {
            max_parity: true,
            http_preconditions: Some(HTTPPreconditions {
@@ -1154,7 +1174,8 @@ pub async fn save_manual_transition_job_record_if_current(
            ..Default::default()
        },
    )
-    .await
+    .await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }

 /// Applies a job-record mutation with optimistic concurrency control.
@@ -1592,9 +1613,9 @@ pub async fn save_manual_transition_scope_admission_if_absent(
    let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
    let data = serde_json::to_vec(admission).map_err(Error::other)?;
    config_boundary::save_config_with_opts(
-        api,
+        api.clone(),
        &object,
-        data,
+        data.clone(),
        &ObjectOptions {
            max_parity: true,
            http_preconditions: Some(HTTPPreconditions {
@@ -1604,7 +1625,8 @@ pub async fn save_manual_transition_scope_admission_if_absent(
            ..Default::default()
        },
    )
-    .await
+    .await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }

 pub async fn load_manual_transition_scope_admission(
@@ -1642,9 +1664,9 @@ pub async fn save_manual_transition_scope_admission_if_current(
    let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
    let data = serde_json::to_vec(admission).map_err(Error::other)?;
    match config_boundary::save_config_with_opts(
-        api,
+        api.clone(),
        &object,
-        data,
+        data.clone(),
        &ObjectOptions {
            max_parity: true,
            http_preconditions: Some(HTTPPreconditions {
@@ -1660,7 +1682,8 @@ pub async fn save_manual_transition_scope_admission_if_current(
            Err(Error::PreconditionFailed)
        }
        result => result,
-    }
+    }?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }

 pub async fn claim_manual_transition_scope_admission(
@@ -1953,13 +1976,15 @@ pub async fn delete_manual_transition_scope_admission_if_current(
    job_id: Uuid,
    lease_id: Uuid,
 ) -> EcstoreResult<bool> {
-    let etag = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
-        Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => etag,
+    let (admission, etag) = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
+        Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => (admission, etag),
        Ok(_) => return Ok(false),
        Err(Error::ConfigNotFound) => return Ok(true),
        Err(err) => return Err(err),
    };
    let object = manual_transition_scope_record_object_name(scope_key).map_err(manual_transition_job_store_error)?;
+    let data = serde_json::to_vec(&admission).map_err(Error::other)?;
+    api.record_durable_ilm_decommission_terminal(&object, &data).await?;
    match config_boundary::delete_config_if_match(api, &object, &etag).await {
        Ok(()) | Err(Error::ConfigNotFound) => Ok(true),
        Err(Error::PreconditionFailed) => Ok(false),
@@ -1971,6 +1996,11 @@ fn manual_transition_job_store_error(err: ManualTransitionJobError) -> Error {
    Error::other(err)
 }

+fn manual_transition_cursor_revision(report: &ManualTransitionRunReport) -> Option<u64> {
+    report.continuation_token.as_ref()?;
+    (report.scanned > 0).then_some(report.scanned)
+}
+
 pub fn manual_transition_scope_admission_lease_expired(admission: &ManualTransitionScopeAdmission) -> bool {
    OffsetDateTime::now_utc().unix_timestamp_nanos() > admission.lease_expires_at_unix_nanos
 }
@@ -2210,6 +2240,76 @@ mod tests {
        );
    }

+    #[test]
+    fn manual_transition_job_scan_progress_accumulates_resumed_checkpoint_counters() {
+        let options = ManualTransitionRunOptions::default();
+        let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
+        let first_token =
+            encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
+        record.update_running_progress(
+            ManualTransitionRunReport {
+                bucket: "bucket".to_string(),
+                lifecycle_config_found: true,
+                scanned: 1000,
+                eligible: 800,
+                enqueued: 50,
+                dry_run_eligible: 10,
+                skipped_not_transition: 2,
+                skipped_tier: 3,
+                skipped_delete_marker: 4,
+                skipped_directory: 5,
+                skipped_replication: 6,
+                skipped_already_transitioned: 7,
+                skipped_already_in_flight: 8,
+                skipped_queue_full: 9,
+                skipped_queue_closed: 10,
+                skipped_queue_timeout: 11,
+                tier_failure: 12,
+                truncated_by_duration: true,
+                continuation_token: first_token,
+                ..Default::default()
+            },
+            ManualTransitionQueueSnapshot::default(),
+        );
+
+        let next_token =
+            encode_manual_transition_continuation_token(Some("logs/page-b".to_string()), Some("version-b".to_string()));
+        record.update_running_progress(
+            ManualTransitionRunReport {
+                bucket: "bucket".to_string(),
+                scanned: 3,
+                eligible: 2,
+                enqueued: 1,
+                dry_run_eligible: 1,
+                skipped_not_transition: 1,
+                tier_failure: 1,
+                continuation_token: next_token.clone(),
+                ..Default::default()
+            },
+            ManualTransitionQueueSnapshot::default(),
+        );
+
+        assert_eq!(record.report.scanned, 1003);
+        assert_eq!(record.report.eligible, 802);
+        assert_eq!(record.report.enqueued, 51);
+        assert_eq!(record.report.dry_run_eligible, 11);
+        assert_eq!(record.report.skipped_not_transition, 3);
+        assert_eq!(record.report.skipped_tier, 3);
+        assert_eq!(record.report.skipped_delete_marker, 4);
+        assert_eq!(record.report.skipped_directory, 5);
+        assert_eq!(record.report.skipped_replication, 6);
+        assert_eq!(record.report.skipped_already_transitioned, 7);
+        assert_eq!(record.report.skipped_already_in_flight, 8);
+        assert_eq!(record.report.skipped_queue_full, 9);
+        assert_eq!(record.report.skipped_queue_closed, 10);
+        assert_eq!(record.report.skipped_queue_timeout, 11);
+        assert_eq!(record.report.tier_failure, 13);
+        assert!(record.report.lifecycle_config_found);
+        assert!(record.report.truncated_by_duration);
+        assert_eq!(record.report.continuation_token, next_token);
+        assert_eq!(record.cursor_revision, Some(1003));
+    }
+
    #[test]
    fn manual_transition_job_apply_worker_result_counts_preserves_existing_failure_reasons() {
        let options = ManualTransitionRunOptions::default();
@@ -2577,6 +2677,98 @@ mod tests {
        assert!(decoded.report.tier_failure_by_reason.is_empty());
    }

+    #[test]
+    fn manual_transition_job_record_derives_revision_from_legacy_cursor() {
+        let options = ManualTransitionRunOptions::default();
+        let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
+        let continuation_token =
+            encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
+        record.update_running_progress(
+            ManualTransitionRunReport {
+                bucket: "bucket".to_string(),
+                scanned: 9,
+                continuation_token,
+                ..Default::default()
+            },
+            ManualTransitionQueueSnapshot::default(),
+        );
+        let encoded = record.encode().expect("job record should encode");
+        let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
+        value["job"]
+            .as_object_mut()
+            .expect("job should be object")
+            .remove("cursor_revision");
+        let record_bytes = serde_json::to_vec(&value["job"]).expect("legacy job should encode");
+        value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned));
+        let legacy = serde_json::to_vec(&value).expect("legacy envelope should encode");
+
+        let decoded = ManualTransitionJobRecord::decode(record.job_id, &legacy).expect("legacy job should decode");
+
+        assert_eq!(decoded.cursor_revision, Some(9));
+    }
+
+    #[test]
+    fn manual_transition_job_record_omits_cursor_revision_for_old_readers() {
+        #[allow(dead_code)]
+        #[derive(serde::Deserialize)]
+        #[serde(deny_unknown_fields)]
+        struct LegacyPersistedManualTransitionJobRecord {
+            schema: String,
+            content_sha256: String,
+            job: LegacyManualTransitionJobRecord,
+        }
+
+        #[allow(dead_code)]
+        #[derive(serde::Deserialize)]
+        #[serde(deny_unknown_fields)]
+        struct LegacyManualTransitionJobRecord {
+            job_id: Uuid,
+            scope_key: String,
+            bucket: String,
+            prefix: String,
+            tier: Option<String>,
+            dry_run: bool,
+            max_objects: Option<u64>,
+            max_duration: Option<std::time::Duration>,
+            owner_id: String,
+            lease_id: Uuid,
+            lease_expires_at_unix_nanos: i128,
+            state: ManualTransitionJobState,
+            scan_completed: bool,
+            cancel_requested: bool,
+            created_at_unix_nanos: i128,
+            updated_at_unix_nanos: i128,
+            completed_at_unix_nanos: Option<i128>,
+            report: ManualTransitionRunReport,
+            queue_snapshot: ManualTransitionQueueSnapshot,
+            error: Option<String>,
+        }
+
+        let options = ManualTransitionRunOptions::default();
+        let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
+        let continuation_token =
+            encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string()));
+        record.update_running_progress(
+            ManualTransitionRunReport {
+                bucket: "bucket".to_string(),
+                scanned: 7,
+                continuation_token: continuation_token.clone(),
+                ..Default::default()
+            },
+            ManualTransitionQueueSnapshot::default(),
+        );
+        assert_eq!(record.cursor_revision, Some(7));
+
+        let encoded = record.encode().expect("job record should encode");
+        let value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
+        assert!(value["job"].get("cursor_revision").is_none());
+        let legacy: LegacyPersistedManualTransitionJobRecord =
+            serde_json::from_slice(&encoded).expect("old reader should accept new job record");
+
+        assert_eq!(legacy.job.job_id, record.job_id);
+        assert_eq!(legacy.job.report.continuation_token, continuation_token);
+    }
+
    #[test]
    fn manual_transition_job_record_rejects_unknown_report_fields() {
        let options = ManualTransitionRunOptions::default();
@@ -16,6 +16,7 @@ pub mod bucket_lifecycle_audit;
 pub mod bucket_lifecycle_ops;
 mod config_boundary;
 pub mod core;
+mod durable_namespace;
 pub mod evaluator;
 pub mod manual_transition_job;
 mod metadata_boundary;
@@ -31,3 +32,8 @@ pub mod tier_free_version_recovery;
 pub mod tier_last_day_stats;
 pub mod tier_sweeper;
 pub mod transition_transaction;
+
+pub(crate) use durable_namespace::{
+    DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
+    validate_durable_ilm_record,
+};
@@ -20,6 +20,7 @@ use tokio_util::sync::CancellationToken;
 use tracing::{debug, warn};

 use crate::bucket::lifecycle::config_boundary;
+use crate::bucket::lifecycle::durable_namespace::TIER_DELETE_JOURNAL_NAMESPACE;
 use crate::bucket::lifecycle::runtime_boundary;
 use crate::bucket::lifecycle::tier_sweeper::{
    Jentry, TierDeleteJournalState, TierDeleteSourceIdentity,
@@ -49,7 +50,7 @@ const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
 const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
 const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
 const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
-pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
+pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = TIER_DELETE_JOURNAL_NAMESPACE.prefix;

 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(deny_unknown_fields)]
@@ -432,6 +433,21 @@ async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jen
        )
        .await?;
    }
+    let path = tier_delete_journal_object_name(je);
+    let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?;
+    let target_pool_indices = api
+        .record_durable_ilm_decommission_terminal_target_pools(&path, &data)
+        .await
+        .map_err(std::io::Error::other)?;
+    if let Some(target_pool_indices) = target_pool_indices {
+        for target_pool_idx in target_pool_indices {
+            match config_boundary::delete_config(api.pools[target_pool_idx].clone(), &path).await {
+                Ok(()) | Err(Error::ConfigNotFound) => {}
+                Err(err) => return Err(std::io::Error::other(err)),
+            }
+        }
+        return Ok(());
+    }
    remove_tier_delete_journal_entry(api, je).await
 }

@@ -21,6 +21,7 @@ use tracing::{debug, warn};
 use uuid::Uuid;

 use crate::bucket::lifecycle::config_boundary;
+use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE;
 use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
 use crate::bucket::lifecycle::tier_sweeper::{
    delete_confirmed_transition_candidate_exact_with_lease_idempotent,
@@ -42,7 +43,7 @@ const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(6
 const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
 pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1";
 pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions";
-pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = "ilm/transition-transactions/records";
+pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix;
 pub const MAX_TRANSITION_TRANSACTION_SIZE: usize = 64 * 1024;

 pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
@@ -584,7 +585,8 @@ pub(crate) async fn save_transition_transaction_record(
    let object =
        transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
    let data = transaction.encode().map_err(transition_transaction_store_error)?;
-    config_boundary::save_config(api, &object, data).await
+    config_boundary::save_config(api.clone(), &object, data.clone()).await?;
+    api.record_durable_ilm_decommission_progress(&object, &data).await
 }

 pub(crate) async fn load_transition_transaction_record(
@@ -596,8 +598,14 @@ pub(crate) async fn load_transition_transaction_record(
    TransitionTransaction::decode(transaction_id, &data).map_err(transition_transaction_store_error)
 }

-pub(crate) async fn delete_transition_transaction_record(api: Arc<ECStore>, transaction_id: Uuid) -> EcstoreResult<()> {
-    let object = transition_transaction_record_object_name(transaction_id).map_err(transition_transaction_store_error)?;
+pub(crate) async fn delete_transition_transaction_record(
+    api: Arc<ECStore>,
+    transaction: &TransitionTransaction,
+) -> EcstoreResult<()> {
+    let object =
+        transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
+    let data = transaction.encode().map_err(transition_transaction_store_error)?;
+    api.record_durable_ilm_decommission_terminal(&object, &data).await?;
    match config_boundary::delete_config(api, &object).await {
        Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
        Err(err) => Err(err),
@@ -813,7 +821,7 @@ pub async fn finalize_missing_transition_transaction_for_operator(
    if probe != TransitionOperatorProbe::Missing {
        return Err(TransitionOperatorError::CandidateNotMissing(probe));
    }
-    delete_transition_transaction_record(api, transaction_id)
+    delete_transition_transaction_record(api, &transaction)
        .await
        .map_err(TransitionOperatorError::Store)
 }
@@ -849,22 +857,22 @@ pub async fn process_transition_transaction_record(
    match transaction.state {
        TransitionTransactionState::Uploaded => {
            delete_transition_remote_candidate(api.clone(), transaction).await?;
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
            Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
        }
        TransitionTransactionState::CleanupPending => match local_commit_matches_transaction(api.clone(), transaction).await {
            Ok(true) => {
-                delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                delete_transition_transaction_record(api, transaction).await?;
                Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
            }
            Ok(false) => {
                delete_transition_remote_candidate(api.clone(), transaction).await?;
-                delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                delete_transition_transaction_record(api, transaction).await?;
                Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
            }
            Err(err) if transition_source_is_missing(&err) => {
                delete_transition_remote_candidate(api.clone(), transaction).await?;
-                delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                delete_transition_transaction_record(api, transaction).await?;
                Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
            }
            Err(err) => Err(err),
@@ -872,7 +880,7 @@ pub async fn process_transition_transaction_record(
        TransitionTransactionState::LocalCommitStarted => {
            match local_commit_matches_transaction(api.clone(), transaction).await {
                Ok(true) => {
-                    delete_transition_transaction_record(api, transaction.transaction_id).await?;
+                    delete_transition_transaction_record(api, transaction).await?;
                    Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
                }
                Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
@@ -881,7 +889,7 @@ pub async fn process_transition_transaction_record(
            }
        }
        TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
            Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
        }
        TransitionTransactionState::UploadOutcomeUnknown => recover_unknown_upload_outcome(api, transaction).await,
@@ -907,7 +915,7 @@ async fn recover_unknown_upload_outcome(
        .map_err(Error::other)?
    {
        TransitionCandidateProbe::Missing => {
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
            Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
        }
        TransitionCandidateProbe::UnversionedPresent => {
@@ -925,7 +933,7 @@ async fn recover_unknown_upload_outcome(
            )
            .await
            .map_err(Error::other)?;
-            delete_transition_transaction_record(api, transaction.transaction_id).await?;
+            delete_transition_transaction_record(api, transaction).await?;
            Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
        }
        TransitionCandidateProbe::VersionedPresent(version_id) => {
@@ -958,7 +966,7 @@ async fn cleanup_recovered_unknown_upload_candidate(
        .map_err(transition_transaction_store_error)?;
    save_transition_transaction_record(api.clone(), &cleanup).await?;
    delete_transition_remote_candidate(api.clone(), &cleanup).await?;
-    delete_transition_transaction_record(api, cleanup.transaction_id).await?;
+    delete_transition_transaction_record(api, &cleanup).await?;
    Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
 }

@@ -15,7 +15,7 @@
 use crate::bucket::metadata_sys::{ObjectLockConfigState, get_object_lock_config, get_object_lock_config_state};
 use crate::bucket::object_lock::objectlock;
 use crate::error::{Error, Result, StorageError};
-use crate::object_api::ObjectInfo;
+use crate::object_api::{ObjectInfo, ObjectOptions};
 use s3s::dto::{Date, DefaultRetention, ObjectLockConfiguration, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
 use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
 use std::sync::Arc;
@@ -136,12 +136,50 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {

 /// Check if an object has legal hold enabled.
 /// Returns true if legal hold is ON.
-#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
 fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
    let lhold = objectlock::get_object_legalhold_meta(user_defined);
    matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
 }

+/// Whether an authorized replication write (`ObjectOptions::replication_request`)
+/// may overwrite a locked destination version.
+///
+/// The source's lock state governs a replica (MinIO `checkPutObjectLockAllowed`
+/// skips the existing-version check for replicas), and a source-side hold
+/// release or retention change reaches this site only through this write. The
+/// overwrite is allowed only when the write carries the source timestamp of
+/// every category that currently locks the version, so receiver-side LWW
+/// (`merge_replication_metadata_lww`) judges each of them: a category locked
+/// more recently here is kept, otherwise the source's newer state wins. A write
+/// without that timestamp carries no source decision for the category — the
+/// metadata replace would lift the lock unjudged — so it stays WORM-rejected.
+///
+/// The locking categories come from the same authoritative evaluation as the
+/// commit-time WORM gate (`check_object_lock_for_deletion_with_state`): the
+/// bucket default retention locks a version that carries no explicit
+/// retention keys, so it is judged here too rather than read off the keys.
+/// Malformed persisted lock metadata or a non-authoritative bucket
+/// configuration is an error, never a pass.
+pub fn replication_write_may_pass_worm_gate(
+    state: &ObjectLockConfigState,
+    obj_info: &ObjectInfo,
+    opts: &ObjectOptions,
+) -> Result<bool> {
+    if !opts.replication_request {
+        return Ok(false);
+    }
+    if obj_info.delete_marker {
+        // Delete markers are never locked (same as the WORM gate).
+        return Ok(true);
+    }
+    let config = object_lock_config_from_state(state)?;
+    if legal_hold_locks(obj_info)? && opts.replication_legalhold_timestamp.is_none() {
+        return Ok(false);
+    }
+    let retention_locked = active_retention(config, obj_info)?.is_some();
+    Ok(!(retention_locked && opts.replication_retention_timestamp.is_none()))
+}
+
 /// Check if an object is locked based on its metadata.
 /// This is a common function used by both lifecycle evaluation and deletion checks.
 ///
@@ -239,69 +277,101 @@ pub(crate) fn check_object_lock_for_deletion_with_config(
        return Ok(None);
    }

-    if let Some(status) = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) {
-        if status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::ON) {
-            return Ok(Some(ObjectLockBlockReason::LegalHold));
-        }
-        if !status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::OFF) {
-            return Err(Error::other("persisted object legal-hold metadata is invalid"));
-        }
+    if legal_hold_locks(obj_info)? {
+        return Ok(Some(ObjectLockBlockReason::LegalHold));
    }

-    let mode = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str());
-    let retain_until = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str());
-    let explicit_ret = match (mode, retain_until) {
-        (None, None) => None,
+    if let Some((mode_str, retain_until)) = active_retention(config, obj_info)?
+        && let Some(reason) = check_retention_blocks_deletion(mode_str, Some(retain_until), bypass_governance)
+    {
+        return Ok(Some(reason));
+    }
+
+    Ok(None)
+}
+
+/// A cleared retention / legal hold is persisted as empty strings (the MinIO
+/// on-disk shape, `parse_object_lock_retention`); read it as "no lock" rather
+/// than as corrupt metadata.
+fn persisted_lock_value<'a>(obj_info: &'a ObjectInfo, key: &str) -> Option<&'a String> {
+    obj_info.user_defined.get(key).filter(|value| !value.is_empty())
+}
+
+/// Whether the version's persisted legal hold is ON. Any other non-empty
+/// value than ON/OFF is malformed metadata and fails closed.
+fn legal_hold_locks(obj_info: &ObjectInfo) -> Result<bool> {
+    let Some(status) = persisted_lock_value(obj_info, X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) else {
+        return Ok(false);
+    };
+    if status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::ON) {
+        return Ok(true);
+    }
+    if !status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::OFF) {
+        return Err(Error::other("persisted object legal-hold metadata is invalid"));
+    }
+    Ok(false)
+}
+
+/// The retention that currently locks the version, if any: the explicit
+/// persisted retention when the keys are present, otherwise the bucket
+/// default retention computed from the version's modification time. Returns
+/// `(mode, retain_until)` only while the retention is still active.
+fn active_retention<'a>(
+    config: Option<&'a ObjectLockConfiguration>,
+    obj_info: &ObjectInfo,
+) -> Result<Option<(&'a str, OffsetDateTime)>> {
+    let mode = persisted_lock_value(obj_info, X_AMZ_OBJECT_LOCK_MODE.as_str());
+    let retain_until = persisted_lock_value(obj_info, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str());
+    match (mode, retain_until) {
+        (None, None) => {}
        (Some(mode), Some(retain_until)) => {
            let mode =
                objectlock::parse_ret_mode(mode).ok_or_else(|| Error::other("persisted object retention mode is invalid"))?;
            let retain_until = OffsetDateTime::parse(retain_until, &time::format_description::well_known::Iso8601::DEFAULT)
                .map(Date::from)
                .map_err(|_| Error::other("persisted object retention date is invalid"))?;
-            Some((mode, retain_until))
+            let mode_str = match mode.as_str() {
+                ObjectLockRetentionMode::COMPLIANCE => ObjectLockRetentionMode::COMPLIANCE,
+                ObjectLockRetentionMode::GOVERNANCE => ObjectLockRetentionMode::GOVERNANCE,
+                _ => return Err(Error::other("persisted object retention mode is invalid")),
+            };
+            return Ok(is_retention_active(mode_str, Some(&retain_until)).then(|| (mode_str, OffsetDateTime::from(retain_until))));
        }
        _ => return Err(Error::other("persisted object retention metadata is incomplete")),
+    }
+
+    let Some(default_retention) = config.and_then(|config| config.rule.as_ref()?.default_retention.as_ref()) else {
+        return Ok(None);
    };
-
-    if let Some((mode, retain_until)) = &explicit_ret {
-        let mode_str = mode.as_str();
-        if is_retention_active(mode_str, Some(retain_until))
-            && let Some(reason) =
-                check_retention_blocks_deletion(mode_str, Some(OffsetDateTime::from(retain_until.clone())), bypass_governance)
-        {
-            return Ok(Some(reason));
-        }
+    let Some(mode) = &default_retention.mode else {
+        return Ok(None);
+    };
+    let mode_str = mode.as_str();
+    if mode_str != ObjectLockRetentionMode::COMPLIANCE && mode_str != ObjectLockRetentionMode::GOVERNANCE {
+        return Ok(None);
    }
+    // Calculate retention expiration date from object modification time
+    let mod_time = obj_info
+        .mod_time
+        .ok_or_else(|| Error::other("persisted object modification time is missing"))?;
+    let now = objectlock::utc_now_ntp();
+    let retain_until = if let Some(days) = default_retention.days {
+        mod_time.saturating_add(time::Duration::days(i64::from(days)))
+    } else {
+        let years = default_retention
+            .years
+            .ok_or_else(|| Error::other("persisted bucket Object Lock retention period is invalid"))?;
+        add_years(mod_time, years)
+    };
+    Ok((retain_until.unix_timestamp() > now.unix_timestamp()).then_some((mode_str, retain_until)))
+}

-    if explicit_ret.is_none()
-        && let Some(default_retention) = config.and_then(|config| config.rule.as_ref()?.default_retention.as_ref())
-        && let Some(mode) = &default_retention.mode
-    {
-        let mode_str = mode.as_str();
-        if mode_str == ObjectLockRetentionMode::COMPLIANCE || mode_str == ObjectLockRetentionMode::GOVERNANCE {
-            // Calculate retention expiration date from object modification time
-            let mod_time = obj_info
-                .mod_time
-                .ok_or_else(|| Error::other("persisted object modification time is missing"))?;
-            let now = objectlock::utc_now_ntp();
-            let retain_until = if let Some(days) = default_retention.days {
-                mod_time.saturating_add(time::Duration::days(i64::from(days)))
-            } else {
-                let years = default_retention
-                    .years
-                    .ok_or_else(|| Error::other("persisted bucket Object Lock retention period is invalid"))?;
-                add_years(mod_time, years)
-            };
-
-            if retain_until.unix_timestamp() > now.unix_timestamp()
-                && let Some(reason) = check_retention_blocks_deletion(mode_str, Some(retain_until), bypass_governance)
-            {
-                return Ok(Some(reason));
-            }
-        }
+fn object_lock_config_from_state(state: &ObjectLockConfigState) -> Result<Option<&ObjectLockConfiguration>> {
+    match state {
+        ObjectLockConfigState::Configured { config, .. } => Ok(Some(config)),
+        ObjectLockConfigState::ConfirmedAbsent => Ok(None),
+        ObjectLockConfigState::Fabricated => Err(Error::other("bucket Object Lock metadata is not authoritative")),
    }
-
-    Ok(None)
 }

 pub(crate) fn check_object_lock_for_deletion_with_state(
@@ -309,13 +379,7 @@ pub(crate) fn check_object_lock_for_deletion_with_state(
    obj_info: &ObjectInfo,
    bypass_governance: bool,
 ) -> Result<Option<ObjectLockBlockReason>> {
-    match state {
-        ObjectLockConfigState::Configured { config, .. } => {
-            check_object_lock_for_deletion_with_config(Some(config), obj_info, bypass_governance)
-        }
-        ObjectLockConfigState::ConfirmedAbsent => check_object_lock_for_deletion_with_config(None, obj_info, bypass_governance),
-        ObjectLockConfigState::Fabricated => Err(Error::other("bucket Object Lock metadata is not authoritative")),
-    }
+    check_object_lock_for_deletion_with_config(object_lock_config_from_state(state)?, obj_info, bypass_governance)
 }

 /// Compatibility wrapper for callers that predate fallible metadata lookup.
@@ -486,6 +550,210 @@ mod tests {
        }
    }

+    fn replication_opts(hold_ts: bool, retention_ts: bool) -> ObjectOptions {
+        ObjectOptions {
+            replication_request: true,
+            replication_legalhold_timestamp: hold_ts.then_some(OffsetDateTime::UNIX_EPOCH),
+            replication_retention_timestamp: retention_ts.then_some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        }
+    }
+
+    fn lock_metadata(entries: &[&[(&str, &str)]]) -> std::collections::HashMap<String, String> {
+        entries
+            .iter()
+            .flat_map(|entries| entries.iter())
+            .map(|(key, value)| (key.to_string(), value.to_string()))
+            .collect()
+    }
+
+    fn lock_object_info(user_defined: std::collections::HashMap<String, String>) -> ObjectInfo {
+        ObjectInfo {
+            user_defined: Arc::new(user_defined),
+            mod_time: Some(OffsetDateTime::now_utc()),
+            ..Default::default()
+        }
+    }
+
+    /// A replication write passes the WORM gate only when it carries the
+    /// source timestamp of every category that currently locks the version.
+    #[test]
+    fn replication_write_passes_worm_gate_only_with_every_locking_category_timestamp() {
+        use rustfs_utils::http::headers::{
+            AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
+        };
+
+        let hold = [(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "ON")];
+        let retention = [
+            (AMZ_OBJECT_LOCK_MODE_LOWER, "GOVERNANCE"),
+            (AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2099-01-01T00:00:00Z"),
+        ];
+        let expired = [
+            (AMZ_OBJECT_LOCK_MODE_LOWER, "COMPLIANCE"),
+            (AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2000-01-01T00:00:00Z"),
+        ];
+        let absent = ObjectLockConfigState::ConfirmedAbsent;
+        let passes = |state: &ObjectLockConfigState, entries: &[&[(&str, &str)]], opts: &ObjectOptions| {
+            replication_write_may_pass_worm_gate(state, &lock_object_info(lock_metadata(entries)), opts)
+                .expect("well-formed lock metadata must be judged")
+        };
+
+        assert!(passes(&absent, &[&hold, &retention], &replication_opts(true, true)));
+        assert!(!passes(&absent, &[&hold, &retention], &replication_opts(true, false)));
+        assert!(!passes(&absent, &[&hold, &retention], &replication_opts(false, true)));
+
+        assert!(passes(&absent, &[&hold], &replication_opts(true, false)));
+        assert!(!passes(&absent, &[&hold], &replication_opts(false, true)));
+        assert!(passes(&absent, &[&retention], &replication_opts(false, true)));
+        assert!(!passes(&absent, &[&retention], &replication_opts(true, false)));
+
+        // Expired retention and a released hold no longer lock anything.
+        assert!(passes(
+            &absent,
+            &[&expired, &[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "OFF")]],
+            &replication_opts(false, false)
+        ));
+
+        // Never for a non-replication write, whatever it carries.
+        let local = ObjectOptions {
+            replication_request: false,
+            ..replication_opts(true, true)
+        };
+        assert!(!passes(&absent, &[&hold], &local));
+    }
+
+    /// The bucket default retention locks a version that carries no explicit
+    /// retention keys (`check_object_lock_for_deletion_with_config` judges it
+    /// from the modification time), so the replication bypass must demand the
+    /// retention source timestamp for it too — a tagging-only replication
+    /// write must not overwrite the default-protected version unjudged.
+    #[test]
+    fn replication_write_under_bucket_default_retention_requires_retention_timestamp() {
+        use rustfs_utils::http::headers::{AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER};
+
+        for mode in [ObjectLockRetentionMode::COMPLIANCE, ObjectLockRetentionMode::GOVERNANCE] {
+            let state = ObjectLockConfigState::Configured {
+                config: default_retention_config(mode),
+                updated_at: OffsetDateTime::now_utc(),
+            };
+            let no_keys = lock_object_info(std::collections::HashMap::new());
+            assert!(
+                check_object_lock_for_deletion_with_state(&state, &no_keys, false)
+                    .expect("default retention must be judged")
+                    .is_some(),
+                "{mode}: the gate must report the default retention lock"
+            );
+
+            let tagging_only = ObjectOptions {
+                replication_request: true,
+                replication_tagging_timestamp: Some(OffsetDateTime::UNIX_EPOCH),
+                ..Default::default()
+            };
+            assert!(
+                !replication_write_may_pass_worm_gate(&state, &no_keys, &tagging_only).expect("judged"),
+                "{mode}: a tagging-only replication write must not pass the default retention lock"
+            );
+            assert!(
+                replication_write_may_pass_worm_gate(&state, &no_keys, &replication_opts(false, true)).expect("judged"),
+                "{mode}: the retention source timestamp lets LWW judge the default retention"
+            );
+
+            // Default retention plus a legal hold: both categories need a timestamp.
+            let held = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "ON")]]));
+            assert!(!replication_write_may_pass_worm_gate(&state, &held, &replication_opts(false, true)).expect("judged"));
+            assert!(!replication_write_may_pass_worm_gate(&state, &held, &replication_opts(true, false)).expect("judged"));
+            assert!(replication_write_may_pass_worm_gate(&state, &held, &replication_opts(true, true)).expect("judged"));
+
+            // A version whose default retention has already expired (old
+            // mod_time) is not locked by the default any more.
+            let expired_default = ObjectInfo {
+                mod_time: Some(make_datetime(2000, 1, 1)),
+                ..lock_object_info(std::collections::HashMap::new())
+            };
+            assert!(replication_write_may_pass_worm_gate(&state, &expired_default, &tagging_only).expect("judged"));
+
+            // A delete marker is never locked, so there is nothing to judge.
+            let delete_marker = ObjectInfo {
+                delete_marker: true,
+                ..lock_object_info(std::collections::HashMap::new())
+            };
+            assert!(replication_write_may_pass_worm_gate(&state, &delete_marker, &tagging_only).expect("judged"));
+
+            // Cleared (empty) explicit keys fall back to the bucket default.
+            let cleared = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_MODE_LOWER, "")]]));
+            assert!(!replication_write_may_pass_worm_gate(&state, &cleared, &tagging_only).expect("judged"));
+        }
+    }
+
+    /// The replication bypass never judges from a non-authoritative bucket
+    /// state or malformed persisted lock metadata; both are errors, not a pass.
+    #[test]
+    fn replication_write_worm_gate_fails_closed_on_unverifiable_lock_state() {
+        use rustfs_utils::http::headers::AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER;
+
+        let opts = replication_opts(true, true);
+        let err = replication_write_may_pass_worm_gate(
+            &ObjectLockConfigState::Fabricated,
+            &lock_object_info(std::collections::HashMap::new()),
+            &opts,
+        )
+        .expect_err("fabricated bucket lock metadata must not be judged");
+        assert!(err.to_string().contains("not authoritative"));
+
+        let malformed = lock_object_info(lock_metadata(&[&[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "MAYBE")]]));
+        let err = replication_write_may_pass_worm_gate(&ObjectLockConfigState::ConfirmedAbsent, &malformed, &opts)
+            .expect_err("malformed legal hold must not be judged");
+        assert!(err.to_string().contains("legal-hold"));
+
+        let state = ObjectLockConfigState::Configured {
+            config: default_retention_config(ObjectLockRetentionMode::COMPLIANCE),
+            updated_at: OffsetDateTime::now_utc(),
+        };
+        let no_mod_time = ObjectInfo::default();
+        let err = replication_write_may_pass_worm_gate(&state, &no_mod_time, &opts)
+            .expect_err("default retention without a modification time must not be judged");
+        assert!(err.to_string().contains("modification time"));
+    }
+
+    /// A local PutObjectRetention / PutObjectLegalHold "clear" persists the
+    /// lock keys as empty strings (the MinIO on-disk shape, see
+    /// `parse_object_lock_retention`); that is "no lock", not corruption, and
+    /// must not wedge later explicit-version PUTs or deletes
+    /// (rustfs/backlog#1953).
+    #[test]
+    fn deletion_treats_cleared_empty_lock_metadata_as_unlocked() {
+        use rustfs_utils::http::headers::{
+            AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
+        };
+
+        let cases: [(&str, &[&str]); 3] = [
+            (
+                "cleared retention",
+                &[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
+            ),
+            ("cleared legal hold", &[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER]),
+            (
+                "all cleared",
+                &[
+                    AMZ_OBJECT_LOCK_MODE_LOWER,
+                    AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
+                    AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER,
+                ],
+            ),
+        ];
+
+        for (case, keys) in cases {
+            let user_defined = keys.iter().map(|key| (key.to_string(), String::new())).collect();
+            let obj_info = ObjectInfo {
+                user_defined: Arc::new(user_defined),
+                ..Default::default()
+            };
+
+            let result = check_object_lock_for_deletion_with_config(None, &obj_info, false);
+            assert!(matches!(result, Ok(None)), "{case}: empty lock keys must read as unlocked: {result:?}");
+        }
+    }
+
    #[test]
    fn deletion_rejects_invalid_persisted_legal_hold_metadata() {
        let mut user_defined = std::collections::HashMap::new();
@@ -44,11 +44,14 @@ mod replication_versioning_boundary;
 mod runtime_boundary;

 pub use replication_config_boundary::{
-    ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
-    REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
-    ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
-    invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
-    unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
+    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
+    REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
+    REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
+    assign_site_replication_rule_priorities, invalid_replication_config_status_field, is_site_replication_role,
+    is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
+    replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
+    site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
+    validate_replication_config_target_arns,
 };
 pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
 pub use replication_filemeta_boundary::{
@@ -13,9 +13,12 @@
 // limitations under the License.

 pub use rustfs_replication::{
-    ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
-    REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
-    ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
-    invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
-    unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
+    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
+    REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
+    REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt,
+    ReplicationTargetValidationError, assign_site_replication_rule_priorities, invalid_replication_config_status_field,
+    is_site_replication_role, is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
+    replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
+    site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
+    validate_replication_config_target_arns,
 };
@@ -2855,7 +2855,7 @@ fn replicate_object_info_from_object_info(
        .map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
    let mut rstate = oi.replication_state();
    rstate.replicate_decision_str = dsc.to_string();
-    let asz = oi.get_actual_size().unwrap_or_default();
+    let asz = oi.get_actual_size_or_physical();
    let ssec = replication_object_is_ssec_encrypted(&oi.user_defined);
    let checksum = if ssec { oi.checksum.clone() } else { None };

@@ -4194,6 +4194,30 @@ mod tests {
        assert_eq!(ri.checksum, Some(checksum));
    }

+    #[test]
+    fn metadata_mrf_roundtrip_preserves_tags_and_admitted_targets() {
+        let target = "arn:rustfs:replication:target-a";
+        let object = ObjectInfo {
+            bucket: "source".to_string(),
+            name: "object".to_string(),
+            version_id: Some(Uuid::new_v4()),
+            user_tags: Arc::new("owner=a3".to_string()),
+            ..Default::default()
+        };
+        let live =
+            replicate_object_info_from_object_info(object.clone(), test_replicate_decision(&[target]), ReplicationType::Metadata);
+        let persisted = live.to_mrf_entry();
+        let encoded = encode_mrf_file(std::slice::from_ref(&persisted)).expect("metadata MRF entry should encode");
+        let decoded = decode_mrf_file(&encoded).expect("metadata MRF entry should decode");
+
+        assert_eq!(decoded[0].op, MrfOpKind::Metadata);
+        assert_eq!(decoded[0].target_arns, vec![target.to_string()]);
+        let replayed = admitted_mrf_replicate_object(object, &decoded[0], ReplicationType::Metadata);
+        assert_eq!(replayed.op_type, ReplicationType::Metadata);
+        assert_eq!(replayed.user_tags, "owner=a3");
+        assert_eq!(replayed.admitted_target_arns(), vec![target.to_string()]);
+    }
+
    #[tokio::test]
    async fn mrf_save_admission_waits_for_capacity_instead_of_dropping() {
        let (tx, mut rx) = mpsc::channel(1);
@@ -76,7 +76,8 @@ use metrics::counter;
 use rmp_serde;
 use rustfs_s3_types::EventName;
 use rustfs_utils::http::{
-    AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, has_internal_suffix, insert_str,
+    AMZ_BUCKET_REPLICATION_STATUS, AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS,
+    has_internal_suffix, insert_str,
 };
 use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
 #[cfg(test)]
@@ -174,6 +175,14 @@ fn has_raw_status(err: &SdkError<HeadObjectError>, status: u16) -> bool {
    err.raw_response().is_some_and(|r| r.status().as_u16() == status)
 }

+fn metadata_requires_existing_target(op_type: ReplicationType, object_info: &ObjectInfo) -> bool {
+    op_type == ReplicationType::Metadata
+        && object_info
+            .user_defined
+            .get(AMZ_BUCKET_REPLICATION_STATUS)
+            .is_some_and(|status| status.eq_ignore_ascii_case(ReplicationStatusType::Replica.as_str()))
+}
+
 const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_identity_drift_total";

 /// Targets that already produced a version-identity-drift warning this
@@ -1412,7 +1421,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
    };
    let mut replication_state = oi.replication_state();
    replication_state.replicate_decision_str = dsc.to_string();
-    let actual_size = oi.get_actual_size().unwrap_or_default();
+    let actual_size = oi.get_actual_size_or_physical();

    Ok(ReplicateObjectInfo {
        name: oi.name.clone(),
@@ -3494,6 +3503,7 @@ async fn resolve_replicate_all_action(
        start_time,
        ssec_audit_required,
    } = ctx;
+    let require_existing_target = metadata_requires_existing_target(roi.op_type, &object_info);
    let replication_action;
    match head_object_for_worker(tgt_client.as_ref(), &tgt_client.bucket, object, roi.version_id.map(|v| v.to_string())).await {
        Ok(oi) => {
@@ -3555,7 +3565,13 @@ async fn resolve_replicate_all_action(
                // Version-ID format mismatch: retry without versionId and compare ETags.
                match head_object_fallback(tgt_client, object).await {
                    Ok(Some(oi)) => {
-                        replication_action = if replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref()) {
+                        let etags_match = replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref());
+                        if require_existing_target && !etags_match {
+                            rinfo.error = Some("replica metadata target does not contain matching object data".to_string());
+                            rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
+                            return None;
+                        }
+                        replication_action = if etags_match {
                            if ssec_audit_required
                                && !settle_ssec_passthrough_evidence(&oi, tgt_client, bucket, object, rinfo).await
                            {
@@ -3568,6 +3584,11 @@ async fn resolve_replicate_all_action(
                        };
                    }
                    Ok(None) => {
+                        if require_existing_target {
+                            rinfo.error = Some("replica metadata target does not contain this object version".to_string());
+                            rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
+                            return None;
+                        }
                        replication_action = ReplicationAction::All;
                    }
                    Err(e2) => {
@@ -3593,7 +3614,12 @@ async fn resolve_replicate_all_action(
                        return None;
                    }
                }
-            } else if e.as_service_error().is_some_and(|se| se.is_not_found()) {
+            } else if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) {
+                if require_existing_target {
+                    rinfo.error = Some("replica metadata target does not contain this object version".to_string());
+                    rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
+                    return None;
+                }
                replication_action = ReplicationAction::All;
            } else {
                rinfo.error = Some(e.to_string());
@@ -3868,6 +3894,7 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
                actual_size,
                object_info.etag.clone().unwrap_or_default(),
                object_info.mod_time,
+                &put_opts.internal,
            ),
        )
        .await
@@ -3921,6 +3948,113 @@ mod tests {
        })
    }

+    fn spawn_head_status_server(status: u16) -> (String, std::thread::JoinHandle<()>) {
+        use std::io::{Read, Write};
+
+        let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).expect("test HTTP listener should bind");
+        let endpoint = format!("http://{}", listener.local_addr().expect("test HTTP listener should have an address"));
+        let handle = std::thread::spawn(move || {
+            let (mut stream, _) = listener.accept().expect("test HTTP client should connect");
+            let mut request = [0_u8; 8192];
+            let bytes_read = stream.read(&mut request).expect("test HTTP request should be read");
+            assert!(bytes_read > 0, "test HTTP request should not be empty");
+            assert!(request[..bytes_read].starts_with(b"HEAD "), "replication comparison must use HEAD");
+            write!(stream, "HTTP/1.1 {status} Test\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+                .expect("test HTTP response should be written");
+        });
+        (endpoint, handle)
+    }
+
+    #[tokio::test]
+    async fn replica_metadata_missing_target_stops_before_full_put() {
+        let (endpoint, server) = spawn_head_status_server(404);
+        let target = test_target_client(endpoint);
+        let roi = ReplicateObjectInfo {
+            bucket: "source".to_string(),
+            name: "object".to_string(),
+            version_id: Some(Uuid::new_v4()),
+            op_type: ReplicationType::Metadata,
+            // Normal metadata writes replace REPLICA with per-target PENDING
+            // before constructing the worker request.
+            replication_status: ReplicationStatusType::Pending,
+            ..Default::default()
+        };
+        let object_info = ObjectInfo {
+            bucket: roi.bucket.clone(),
+            name: roi.name.clone(),
+            version_id: roi.version_id,
+            etag: Some("source-etag".to_string()),
+            user_defined: Arc::new(HashMap::from([(
+                AMZ_BUCKET_REPLICATION_STATUS.to_string(),
+                ReplicationStatusType::Replica.as_str().to_string(),
+            )])),
+            ..Default::default()
+        };
+        let mut rinfo = replicate_all_target_info(&roi, &target);
+
+        let action = resolve_replicate_all_action(
+            ReplicateAllActionContext {
+                roi: &roi,
+                tgt_client: &target,
+                bucket: &roi.bucket,
+                object: &roi.name,
+                start_time: OffsetDateTime::now_utc(),
+                ssec_audit_required: false,
+            },
+            object_info,
+            &mut rinfo,
+        )
+        .await;
+
+        assert!(action.is_none(), "missing replica metadata targets must not reach the payload PUT path");
+        assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed);
+        assert_eq!(
+            rinfo.error.as_deref(),
+            Some("replica metadata target does not contain this object version")
+        );
+        server.join().expect("test HTTP server should finish");
+    }
+
+    #[tokio::test]
+    async fn source_metadata_missing_target_rebuilds_object() {
+        let (endpoint, server) = spawn_head_status_server(404);
+        let target = test_target_client(endpoint);
+        let roi = ReplicateObjectInfo {
+            bucket: "source".to_string(),
+            name: "object".to_string(),
+            version_id: Some(Uuid::new_v4()),
+            op_type: ReplicationType::Metadata,
+            replication_status: ReplicationStatusType::Pending,
+            ..Default::default()
+        };
+        let object_info = ObjectInfo {
+            bucket: roi.bucket.clone(),
+            name: roi.name.clone(),
+            version_id: roi.version_id,
+            etag: Some("source-etag".to_string()),
+            ..Default::default()
+        };
+        let mut rinfo = replicate_all_target_info(&roi, &target);
+
+        let action = resolve_replicate_all_action(
+            ReplicateAllActionContext {
+                roi: &roi,
+                tgt_client: &target,
+                bucket: &roi.bucket,
+                object: &roi.name,
+                start_time: OffsetDateTime::now_utc(),
+                ssec_audit_required: false,
+            },
+            object_info,
+            &mut rinfo,
+        )
+        .await;
+
+        assert!(matches!(action, Some((ReplicationAction::All, _))));
+        assert!(rinfo.error.is_none());
+        server.join().expect("test HTTP server should finish");
+    }
+
    async fn register_test_target(target: &Arc<TargetClient>) {
        ReplicationTargetStore::register_test_target(target).await;
    }
@@ -389,7 +389,7 @@ fn replication_source_object(object_info: &ObjectInfo) -> ReplicationSourceObjec
            .map(|mod_time| OffsetDateTime::from_unix_timestamp(mod_time.unix_timestamp()).unwrap_or(mod_time)),
        version_id: object_info.version_id.map(|version_id| version_id.to_string()),
        etag: object_info.etag.as_deref(),
-        actual_size: object_info.get_actual_size().unwrap_or_default(),
+        actual_size: object_info.get_actual_size_or_physical(),
        delete_marker: object_info.delete_marker,
        content_type: object_info.content_type.as_deref(),
        content_encoding: object_info.content_encoding.as_deref(),
@@ -472,6 +472,7 @@ pub(crate) fn replication_complete_multipart_options(
    actual_size: String,
    source_etag: String,
    source_mtime: Option<OffsetDateTime>,
+    source_internal: &AdvancedPutOptions,
 ) -> PutObjectOptions {
    let mut user_metadata = HashMap::new();
    insert_header_map(&mut user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, actual_size);
@@ -484,6 +485,14 @@ pub(crate) fn replication_complete_multipart_options(
            // mtime must degrade to epoch so header() suppresses the header
            // instead of asserting the replication time as the object's mtime.
            source_mtime: source_mtime.unwrap_or(OffsetDateTime::UNIX_EPOCH),
+            // Carry the per-category LWW timestamps on the complete request as
+            // well: the receiver's CompleteMultipartUpload options builder
+            // parses the same headers, so the multipart transport gets the
+            // same receiver-side LWW as the single-PUT transport
+            // (rustfs/backlog#1953). Epoch values keep the headers suppressed.
+            tagging_timestamp: source_internal.tagging_timestamp,
+            retention_timestamp: source_internal.retention_timestamp,
+            legalhold_timestamp: source_internal.legalhold_timestamp,
            replication_status: ReplicationStatusType::Replica,
            replication_request: true,
            ..Default::default()
@@ -542,6 +551,20 @@ mod tests {
        assert!(replication_target_head_is_newer_null_version(&source, &target));
    }

+    #[test]
+    fn replication_source_uses_physical_size_for_unknown_compressed_object() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
+        let source = ObjectInfo {
+            size: 128,
+            actual_size: -1,
+            user_defined: Arc::new(metadata),
+            ..Default::default()
+        };
+
+        assert_eq!(replication_source_object(&source).actual_size, 128);
+    }
+
    #[test]
    fn replication_target_head_content_matches_compare_etag_only() {
        let source = ObjectInfo {
@@ -649,20 +672,39 @@ mod tests {
    #[test]
    fn replication_complete_multipart_options_sets_actual_size() {
        let source_mtime = OffsetDateTime::from_unix_timestamp(1_716_170_000).expect("valid test timestamp");
+        let source_internal = AdvancedPutOptions {
+            tagging_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_100).expect("valid test timestamp"),
+            retention_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_200).expect("valid test timestamp"),
+            legalhold_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_300).expect("valid test timestamp"),
+            ..Default::default()
+        };
        let options = replication_complete_multipart_options(
            "1024".to_string(),
            "0123456789abcdef0123456789abcdef-3".to_string(),
            Some(source_mtime),
+            &source_internal,
        );
        assert_eq!(options.internal.source_etag, "0123456789abcdef0123456789abcdef-3");
        assert_eq!(options.internal.source_mtime, source_mtime);

+        // The complete request must carry the same per-category LWW timestamps
+        // as the initiate request; the receiver reads them from the complete
+        // headers (rustfs/backlog#1953).
+        assert_eq!(options.internal.tagging_timestamp, source_internal.tagging_timestamp);
+        assert_eq!(options.internal.retention_timestamp, source_internal.retention_timestamp);
+        assert_eq!(options.internal.legalhold_timestamp, source_internal.legalhold_timestamp);
+
        // Absent source mtime must degrade to epoch (header suppressed), not
        // the AdvancedPutOptions default of now_utc() — that default would
        // stamp the replication time as the replica's mtime and break the
-        // multipart HEAD convergence.
-        let options_no_mtime = replication_complete_multipart_options("1024".to_string(), String::new(), None);
+        // multipart HEAD convergence. Unset category timestamps stay epoch so
+        // header() keeps suppressing them.
+        let options_no_mtime =
+            replication_complete_multipart_options("1024".to_string(), String::new(), None, &AdvancedPutOptions::default());
        assert_eq!(options_no_mtime.internal.source_mtime.unix_timestamp(), 0);
+        assert_eq!(options_no_mtime.internal.tagging_timestamp.unix_timestamp(), 0);
+        assert_eq!(options_no_mtime.internal.retention_timestamp.unix_timestamp(), 0);
+        assert_eq!(options_no_mtime.internal.legalhold_timestamp.unix_timestamp(), 0);

        assert_eq!(
            get_header_map(&options.user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE).as_deref(),
@@ -32,6 +32,10 @@ pub struct Credentials {
    pub access_key: String,
    #[serde(rename = "secretKey")]
    pub secret_key: String,
+    // The aliases accept madmin's JSON tags (MinIO-written bucket-targets
+    // metadata and mc request bodies) without changing the snake_case
+    // persisted/peer wire format this struct serializes to.
+    #[serde(alias = "sessionToken")]
    pub session_token: Option<String>,
    pub expiration: Option<Timestamp>,
 }
@@ -202,12 +206,14 @@ pub struct BucketTarget {
    #[serde(default)]
    pub region: String,

-    #[serde(alias = "bandwidth", default)]
+    // madmin-go v3.0.109 tags this `bandwidthlimit`; `bandwidth` is a legacy
+    // alias kept for inputs written before the madmin tag was verified.
+    #[serde(alias = "bandwidthlimit", alias = "bandwidth", default)]
    pub bandwidth_limit: i64,

    #[serde(rename = "replicationSync", default)]
    pub replication_sync: bool,
-    #[serde(default)]
+    #[serde(alias = "storageclass", default)]
    pub storage_class: String,
    #[serde(rename = "skipTlsVerify", default)]
    pub skip_tls_verify: bool,
@@ -220,7 +226,7 @@ pub struct BucketTarget {

    #[serde(rename = "resetBeforeDate", with = "time::serde::rfc3339::option", default)]
    pub reset_before_date: Option<OffsetDateTime>,
-    #[serde(default)]
+    #[serde(alias = "resetID", default)]
    pub reset_id: String,
    #[serde(rename = "totalDowntime", with = "duration_seconds", default)]
    pub total_downtime: Duration,
@@ -233,7 +239,7 @@ pub struct BucketTarget {
    #[serde(default)]
    pub latency: LatencyStat,

-    #[serde(default)]
+    #[serde(alias = "deploymentID", default)]
    pub deployment_id: String,

    #[serde(default)]
@@ -531,6 +537,85 @@ mod tests {
        assert_eq!(value["totalDowntime"], 90);
    }

+    #[test]
+    fn bucket_target_persisted_wire_keys_stay_snake_case() {
+        // bucket-targets.json (persisted via `serde_json::to_vec(&BucketTargets)`
+        // in the admin set/remove handlers) and the msgpack struct-map form
+        // (`BucketTargets::marshal_msg`) both come straight from this struct's
+        // serde field names. madmin naming is applied only in the admin
+        // response layer (`remote_target_admin_json`); renaming here would
+        // silently break every existing deployment's persisted metadata.
+        let targets = BucketTargets {
+            targets: vec![BucketTarget {
+                credentials: Some(Credentials {
+                    access_key: "ak".to_string(),
+                    secret_key: "sk".to_string(),
+                    session_token: Some("token".to_string()),
+                    expiration: None,
+                }),
+                bandwidth_limit: 5,
+                storage_class: "STANDARD".to_string(),
+                reset_id: "reset-1".to_string(),
+                deployment_id: "deploy-1".to_string(),
+                ..Default::default()
+            }],
+        };
+
+        let json = serde_json::to_value(&targets).expect("targets should serialize to JSON");
+        let msgpack: serde_json::Value =
+            rmp_serde::from_slice(&targets.marshal_msg().expect("targets should marshal to msgpack"))
+                .expect("msgpack struct map should decode into a JSON value");
+
+        for (wire, entry) in [("JSON", &json["targets"][0]), ("msgpack", &msgpack["targets"][0])] {
+            assert_eq!(entry["bandwidth_limit"], 5, "{wire} key `bandwidth_limit` must stay");
+            assert_eq!(entry["storage_class"], "STANDARD", "{wire} key `storage_class` must stay");
+            assert_eq!(entry["reset_id"], "reset-1", "{wire} key `reset_id` must stay");
+            assert_eq!(entry["deployment_id"], "deploy-1", "{wire} key `deployment_id` must stay");
+            assert_eq!(entry["credentials"]["session_token"], "token", "{wire} key `session_token` must stay");
+        }
+    }
+
+    #[test]
+    fn minio_written_bucket_targets_json_populates_madmin_named_fields() {
+        // A MinIO-written bucket-targets.json carries madmin's JSON tags
+        // (`bandwidthlimit`, `storageclass`, `resetID`, `deploymentID`,
+        // `credentials.sessionToken` — madmin-go v3.0.109 bucket-targets.go).
+        // On migration these must land in the matching fields instead of
+        // silently defaulting (backlog#1951).
+        let targets: BucketTargets = serde_json::from_value(serde_json::json!({
+            "targets": [{
+                "sourcebucket": "src",
+                "endpoint": "minio.example:9000",
+                "credentials": {
+                    "accessKey": "ak",
+                    "secretKey": "sk",
+                    "sessionToken": "minio-session-token"
+                },
+                "targetbucket": "dst",
+                "type": "replication",
+                "replicationSync": true,
+                "bandwidthlimit": 107374182400i64,
+                "storageclass": "STANDARD",
+                "resetID": "reset-789",
+                "deploymentID": "deploy-123"
+            }]
+        }))
+        .expect("MinIO-written bucket-targets.json must deserialize");
+
+        let target = &targets.targets[0];
+        assert_eq!(target.bandwidth_limit, 107374182400);
+        assert_eq!(target.storage_class, "STANDARD");
+        assert_eq!(target.reset_id, "reset-789");
+        assert_eq!(target.deployment_id, "deploy-123");
+        assert_eq!(
+            target
+                .credentials
+                .as_ref()
+                .and_then(|credentials| credentials.session_token.as_deref()),
+            Some("minio-session-token")
+        );
+    }
+
    #[test]
    fn test_bucket_target_debug_redacts_credentials() {
        let target = BucketTarget {
@@ -94,6 +94,7 @@ const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13;
 const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 4096;
 const REPLAY_CACHE_AUTO_MAX_CAPACITY: usize = 33_554_432;
 const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
+const NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-tier-registry-generation-v1";
 pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
 static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock<bool> = LazyLock::new(|| {
    get_env_bool(
@@ -636,40 +637,79 @@ pub fn verify_put_file_capability(challenge: Uuid, server_epoch: Uuid, version:
        .map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid put_file capability proof"))
 }

-fn update_ns_scanner_capability_mac(mac: &mut HmacSha256, challenge: Uuid, server_epoch: Uuid) {
+fn update_ns_scanner_capability_mac(
+    mac: &mut HmacSha256,
+    challenge: Uuid,
+    server_epoch: Uuid,
+    supports_tier_registry_generation: bool,
+) {
    mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
    mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
    mac.update(challenge.as_bytes());
    mac.update(server_epoch.as_bytes());
+    if supports_tier_registry_generation {
+        // The optional response capability is part of the authenticated
+        // scope. A proxy cannot turn an old/unsupported peer into a worker
+        // that receives generation-fenced scanner work.
+        mac.update(NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN);
+    }
 }

-fn generate_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
+fn generate_ns_scanner_capability_proof(
+    secret: &str,
+    challenge: Uuid,
+    server_epoch: Uuid,
+    supports_tier_registry_generation: bool,
+) -> std::io::Result<Vec<u8>> {
    if challenge.is_nil() || server_epoch.is_nil() {
        return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
    }
    let mut mac =
        <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
-    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
+    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
    Ok(mac.finalize().into_bytes().to_vec())
 }

-fn verify_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
+fn verify_ns_scanner_capability_proof(
+    secret: &str,
+    challenge: Uuid,
+    server_epoch: Uuid,
+    proof: &[u8],
+    supports_tier_registry_generation: bool,
+) -> std::io::Result<()> {
    if challenge.is_nil() || server_epoch.is_nil() {
        return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
    }
    let mut mac =
        <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
-    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch);
+    update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation);
    mac.verify_slice(proof)
        .map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof"))
 }

 pub fn sign_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
-    generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch)
+    sign_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, false)
 }

 pub fn verify_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
-    verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof)
+    verify_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, proof, false)
+}
+
+pub fn sign_ns_scanner_capability_with_tier_registry_generation(
+    challenge: Uuid,
+    server_epoch: Uuid,
+    supports_tier_registry_generation: bool,
+) -> std::io::Result<Vec<u8>> {
+    generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, supports_tier_registry_generation)
+}
+
+pub fn verify_ns_scanner_capability_with_tier_registry_generation(
+    challenge: Uuid,
+    server_epoch: Uuid,
+    proof: &[u8],
+    supports_tier_registry_generation: bool,
+) -> std::io::Result<()> {
+    verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof, supports_tier_registry_generation)
 }

 #[derive(Clone, Copy)]
@@ -1709,13 +1749,28 @@ mod tests {
        let secret = "test-scanner-capability-secret";
        let challenge = Uuid::new_v4();
        let server_epoch = Uuid::new_v4();
-        let proof =
-            generate_ns_scanner_capability_proof(secret, challenge, server_epoch).expect("capability proof should be generated");
+        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
+            .expect("capability proof should be generated");

-        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof).is_ok());
-        assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof).is_err());
-        assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof).is_err());
-        assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof).is_err());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_ok());
+        assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false).is_err());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false).is_err());
+        assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false).is_err());
+    }
+
+    #[test]
+    fn namespace_scanner_capability_proof_binds_tier_registry_generation_support() {
+        let secret = "test-scanner-capability-secret";
+        let challenge = Uuid::new_v4();
+        let server_epoch = Uuid::new_v4();
+        let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true)
+            .expect("generation capability proof should be generated");
+
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true).is_ok());
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_err());
+        let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false)
+            .expect("legacy capability proof should be generated");
+        assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true).is_err());
    }

    /// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed,
@@ -13,17 +13,18 @@
 // limitations under the License.

 use crate::cluster::rpc::{
-    build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability, verify_put_file_capability,
+    build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability_with_tier_registry_generation,
+    verify_put_file_capability,
 };
 use crate::disk::error::{Error, Result};
 use crate::disk::{FileReader, FileWriter};
 use crate::storage_api_contracts::internode::{
    NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY,
    NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY,
-    NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY,
-    PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION,
-    PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY,
-    WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
+    NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY,
+    NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY,
+    PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY,
+    PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
 };
 use async_trait::async_trait;
 use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
@@ -137,6 +138,12 @@ fn put_file_capability_status_is_legacy(status: u16) -> bool {
    status == 404
 }

+fn ns_scanner_capability_error_allows_legacy(error: &Error) -> bool {
+    [400, 404, 405, 426]
+        .into_iter()
+        .any(|status| error.is_internode_http_status(status))
+}
+
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
 #[allow(
    dead_code,
@@ -220,6 +227,7 @@ pub struct NsScannerStreamRequest {
 #[derive(Debug, Clone)]
 pub struct NsScannerCapabilityRequest {
    pub endpoint: String,
+    pub supports_tier_registry_generation: bool,
 }

 /// Data-plane stream opener used by `RemoteDisk`.
@@ -252,6 +260,15 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
    async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result<Uuid> {
        Err(Error::MethodNotAllowed)
    }
+    async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
+        let server_epoch = self.probe_ns_scanner(request).await?;
+        Ok(NsScannerCapabilityResponse {
+            version: NS_SCANNER_PROTOCOL_VERSION,
+            server_epoch,
+            proof: Vec::new(),
+            supports_tier_registry_generation: None,
+        })
+    }
    // Interface facet nobody calls yet: every transport implements both, but no
    // caller negotiates on them. Kept for the internode transport split
    // (backlog#1350); deleting them would delete the seam and six impls.
@@ -335,27 +352,44 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
    }

    async fn probe_ns_scanner(&self, request: NsScannerCapabilityRequest) -> Result<Uuid> {
-        let challenge = Uuid::new_v4();
-        let url = build_ns_scanner_capability_url(&request, challenge);
-        let mut headers = msgpack_headers();
-        build_auth_headers(&url, &Method::GET, &mut headers)?;
-        let reader = HttpReader::new(url, Method::GET, headers, None).await?;
-        let mut body = Vec::new();
-        reader
-            .take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX))
-            .read_to_end(&mut body)
-            .await?;
-        if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE {
-            return Err(Error::other("invalid remote namespace scanner capability response size"));
+        Ok(self.probe_ns_scanner_capability(request).await?.server_epoch)
+    }
+
+    async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
+        if request.supports_tier_registry_generation {
+            return match self.probe_ns_scanner_capability_once(&request).await {
+                Ok(response) => Ok(response),
+                Err(marked_error) if ns_scanner_capability_error_allows_legacy(&marked_error) => {
+                    // A v3 peer may reject the additive query marker, ignore
+                    // it, or return its legacy proof. Retry once without the
+                    // marker and only downgrade after that legacy response is
+                    // authenticated; an unverified epoch is never trusted.
+                    let legacy_request = NsScannerCapabilityRequest {
+                        endpoint: request.endpoint.clone(),
+                        supports_tier_registry_generation: false,
+                    };
+                    match self.probe_ns_scanner_capability_once(&legacy_request).await {
+                        Ok(mut response) => {
+                            response.supports_tier_registry_generation = None;
+                            Ok(response)
+                        }
+                        Err(legacy_error) if ns_scanner_capability_error_allows_legacy(&legacy_error) => {
+                            // Some old deployments expose only the legacy
+                            // protocol response (or advertise 426). Treat
+                            // the pair as an explicit unsupported result so
+                            // the scanner can use its coordinator fallback.
+                            Err(Error::MethodNotAllowed)
+                        }
+                        Err(_) => Err(marked_error),
+                    }
+                }
+                // A server failure, network failure, or authentication error
+                // is not evidence of an old parser. Do not issue an
+                // unauthenticated legacy probe or silently downgrade.
+                Err(marked_error) => Err(marked_error),
+            };
        }
-        let response: NsScannerCapabilityResponse =
-            rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?;
-        if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
-            return Err(Error::other("incompatible remote namespace scanner capability response"));
-        }
-        verify_ns_scanner_capability(challenge, response.server_epoch, &response.proof)
-            .map_err(|err| Error::other(format!("remote namespace scanner capability authentication failed: {err}")))?;
-        Ok(response.server_epoch)
+        self.probe_ns_scanner_capability_once(&request).await
    }

    fn name(&self) -> &'static str {
@@ -368,6 +402,53 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
 }

 impl TcpHttpInternodeDataTransport {
+    async fn probe_ns_scanner_capability_once(
+        &self,
+        request: &NsScannerCapabilityRequest,
+    ) -> Result<NsScannerCapabilityResponse> {
+        let challenge = Uuid::new_v4();
+        let url = build_ns_scanner_capability_url(request, challenge);
+        let mut headers = msgpack_headers();
+        build_auth_headers(&url, &Method::GET, &mut headers)?;
+        let reader = HttpReader::new(url, Method::GET, headers, None).await?;
+        let mut body = Vec::new();
+        reader
+            .take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX))
+            .read_to_end(&mut body)
+            .await?;
+        if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE {
+            return Err(Error::other("invalid remote namespace scanner capability response size"));
+        }
+        let mut response: NsScannerCapabilityResponse =
+            rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?;
+        if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
+            return Err(Error::other("incompatible remote namespace scanner capability response"));
+        }
+        if let Err(err) = verify_ns_scanner_capability_with_tier_registry_generation(
+            challenge,
+            response.server_epoch,
+            &response.proof,
+            request.supports_tier_registry_generation,
+        ) {
+            // A permissive older peer can ignore the additive marker and
+            // return a valid legacy-scope proof with HTTP 200. Accept that
+            // response only after independently authenticating the legacy
+            // scope; all other verification failures remain fail-closed.
+            if request.supports_tier_registry_generation && ns_scanner_capability_legacy_proof_is_valid(challenge, &response) {
+                response.supports_tier_registry_generation = None;
+                return Ok(response);
+            }
+            return Err(Error::other(format!("remote namespace scanner capability authentication failed: {err}")));
+        }
+        // The proof authenticates the requested capability scope, not the
+        // optional response field. Derive the client-facing bit from that
+        // verified scope so an intermediary cannot strip or rewrite the field
+        // and force a silent downgrade after a successful generation-bound
+        // handshake.
+        normalize_ns_scanner_capability_response(&mut response, request.supports_tier_registry_generation);
+        Ok(response)
+    }
+
    async fn put_file_auth_capability(&self, endpoint: &str) -> Result<Option<Uuid>> {
        resolve_put_file_auth_capability(endpoint, || async {
            tokio::time::timeout(PUT_FILE_CAPABILITY_PROBE_TIMEOUT, self.probe_put_file_auth(endpoint))
@@ -649,6 +730,14 @@ fn build_walk_dir_url(request: &WalkDirStreamRequest) -> String {
    )
 }

+fn normalize_ns_scanner_capability_response(response: &mut NsScannerCapabilityResponse, requested_generation_support: bool) {
+    response.supports_tier_registry_generation = requested_generation_support.then_some(true);
+}
+
+fn ns_scanner_capability_legacy_proof_is_valid(challenge: Uuid, response: &NsScannerCapabilityResponse) -> bool {
+    verify_ns_scanner_capability_with_tier_registry_generation(challenge, response.server_epoch, &response.proof, false).is_ok()
+}
+
 fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
    let body_sha256 = hex_simd::encode_to_string(Sha256::digest(&request.body), hex_simd::AsciiCase::Lower);
    format!(
@@ -675,13 +764,18 @@ fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {

 fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challenge: Uuid) -> String {
    format!(
-        "{}{}?{}={}&{}={}",
+        "{}{}?{}={}&{}={}{}",
        request.endpoint,
        NS_SCANNER_PATH,
        NS_SCANNER_PROTOCOL_VERSION_QUERY,
        NS_SCANNER_PROTOCOL_VERSION,
        NS_SCANNER_CAPABILITY_CHALLENGE_QUERY,
-        challenge
+        challenge,
+        if request.supports_tier_registry_generation {
+            format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY)
+        } else {
+            String::new()
+        }
    )
 }

@@ -794,6 +888,7 @@ mod tests {
        let probe_err = transport
            .probe_ns_scanner(NsScannerCapabilityRequest {
                endpoint: "http://node1:9000".to_string(),
+                supports_tier_registry_generation: false,
            })
            .await
            .expect_err("legacy transport should report namespace scanner as unsupported");
@@ -1387,6 +1482,7 @@ mod tests {
        let url = build_ns_scanner_capability_url(
            &NsScannerCapabilityRequest {
                endpoint: "http://node1:9000".to_string(),
+                supports_tier_registry_generation: false,
            },
            challenge,
        );
@@ -1399,6 +1495,85 @@ mod tests {
        );
    }

+    #[test]
+    fn ns_scanner_capability_url_marks_generation_support_only_when_requested() {
+        let challenge = Uuid::new_v4();
+        let url = build_ns_scanner_capability_url(
+            &NsScannerCapabilityRequest {
+                endpoint: "http://node1:9000".to_string(),
+                supports_tier_registry_generation: true,
+            },
+            challenge,
+        );
+
+        assert!(url.contains(&format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY)));
+    }
+
+    #[test]
+    fn ns_scanner_capability_legacy_fallback_requires_explicit_compatibility_status() {
+        for status in [400, 404, 405, 426] {
+            let error = Error::from(rustfs_rio::new_test_internode_http_io_error(
+                rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::from_u16(status).expect("test status")),
+            ));
+            assert!(
+                ns_scanner_capability_error_allows_legacy(&error),
+                "status {status} should permit legacy retry"
+            );
+        }
+
+        let marked_server_error = Error::from(rustfs_rio::new_test_internode_http_io_error(
+            rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::INTERNAL_SERVER_ERROR),
+        ));
+        let network_error = Error::from(rustfs_rio::new_test_internode_http_io_error(
+            rustfs_rio::InternodeHttpErrorKind::ConnectionRefused,
+        ));
+        let authentication_error = Error::other("remote namespace scanner capability authentication failed");
+        assert!(!ns_scanner_capability_error_allows_legacy(&marked_server_error));
+        assert!(!ns_scanner_capability_error_allows_legacy(&network_error));
+        assert!(!ns_scanner_capability_error_allows_legacy(&authentication_error));
+    }
+
+    #[test]
+    fn authenticated_ns_scanner_capability_ignores_unprotected_response_bit() {
+        let mut response = NsScannerCapabilityResponse {
+            version: NS_SCANNER_PROTOCOL_VERSION,
+            server_epoch: Uuid::new_v4(),
+            proof: Vec::new(),
+            supports_tier_registry_generation: None,
+        };
+
+        normalize_ns_scanner_capability_response(&mut response, true);
+        assert_eq!(response.supports_tier_registry_generation, Some(true));
+
+        response.supports_tier_registry_generation = Some(false);
+        normalize_ns_scanner_capability_response(&mut response, false);
+        assert_eq!(response.supports_tier_registry_generation, None);
+    }
+
+    #[test]
+    fn ns_scanner_capability_accepts_only_authenticated_legacy_scope_after_marker_mismatch() {
+        crate::runtime::sources::ensure_test_rpc_secret();
+        let challenge = Uuid::new_v4();
+        let response = NsScannerCapabilityResponse {
+            version: NS_SCANNER_PROTOCOL_VERSION,
+            server_epoch: Uuid::new_v4(),
+            proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, Uuid::new_v4())
+                .expect("placeholder proof should be generated"),
+            supports_tier_registry_generation: None,
+        };
+        // A proof bound to a different challenge cannot authorize the legacy
+        // fallback, even though the response has the expected shape.
+        assert!(!ns_scanner_capability_legacy_proof_is_valid(challenge, &response));
+
+        let server_epoch = response.server_epoch;
+        let valid_response = NsScannerCapabilityResponse {
+            proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, server_epoch)
+                .expect("legacy proof should be generated"),
+            ..response
+        };
+        assert!(ns_scanner_capability_legacy_proof_is_valid(challenge, &valid_response));
+    }
+
    #[test]
    fn transport_config_defaults_to_tcp_http() {
        let transport = build_internode_data_transport(None).unwrap();
@@ -35,8 +35,9 @@ pub use http_auth::{
    TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
    gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
    set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
-    sign_ns_scanner_capability, sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
-    tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_put_file_auth_trailer,
+    sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability,
+    sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
+    verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
    verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
    verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
    verify_tonic_rpc_signature_with_bootstrap,
@@ -1090,7 +1090,9 @@ impl PeerRestClient {
                    .await?
                    .max_decoding_message_size(BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE);
                let response = match client
-                    .background_heal_status(Request::new(BackgroundHealStatusRequest::default()))
+                    .background_heal_status(Request::new(BackgroundHealStatusRequest {
+                        protocol_version: rustfs_protos::BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION,
+                    }))
                    .await
                {
                    Ok(response) => response.into_inner(),
@@ -42,8 +42,9 @@ use futures::lock::Mutex;
 use metrics::counter;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use rustfs_io_metrics::internode_metrics::{
-    INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE,
-    INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
+    INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE,
+    INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE,
+    INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
 };
 use rustfs_protos::ChannelClass;
 use rustfs_protos::evict_failed_connection;
@@ -98,6 +99,7 @@ const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
 const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50);
 const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ";
 const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC";
+const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
 const BATCH_METADATA_RPC_OFF: &str = "off";
 const BATCH_METADATA_RPC_AUTO: &str = "auto";
 const BATCH_METADATA_RPC_ON: &str = "on";
@@ -202,7 +204,8 @@ fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode {
 }

 fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode {
-    rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ)
+    rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
+        .or_else(|| rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ))
        .or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC))
        .as_deref()
        .map(parse_batch_metadata_rpc_mode)
@@ -778,14 +781,16 @@ impl RemoteDisk {
        if self.health.is_faulty() {
            return Err(DiskError::FaultyDisk);
        }
-        let probe = self.data_transport.probe_ns_scanner(NsScannerCapabilityRequest {
+        let probe = self.data_transport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
            endpoint: self.endpoint.grid_host(),
+            supports_tier_registry_generation: true,
        });
        let result = timeout(NS_SCANNER_CAPABILITY_PROBE_TIMEOUT, probe)
            .await
            .map_err(|_| DiskError::other("remote namespace scanner capability probe timed out"))?;
        match result {
-            Ok(server_epoch) => Ok(Some(server_epoch)),
+            Ok(response) if response.supports_tier_registry_generation == Some(true) => Ok(Some(response.server_epoch)),
+            Ok(_) => Ok(None),
            // RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): old peers and legacy transports lack the authenticated startup-epoch handshake. Remove after every supported peer implements namespace scanner protocol v3.
            Err(DiskError::MethodNotAllowed) => Ok(None),
            Err(err)
@@ -1826,6 +1831,12 @@ fn record_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
    }
 }

+fn record_batch_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
+    if let Some(started_at) = started_at {
+        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed());
+    }
+}
+
 /// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads
 /// and falling back to the JSON compatibility strings. Used to size the RPC for the payload
 /// histogram / large-payload alerting (grpc-optimization P0 instrumentation).
@@ -1936,6 +1947,27 @@ fn decode_batch_read_version_response_items(
    Ok(batch_read_version_resps)
 }

+fn batch_read_version_request_payload_len(req: &BatchReadVersionReq, req_json: &str, req_bin: &[u8]) -> usize {
+    req.items
+        .iter()
+        .fold(req_json.len().saturating_add(req_bin.len()), |total, item| {
+            total
+                .saturating_add(item.org_volume.len())
+                .saturating_add(item.volume.len())
+                .saturating_add(item.path.len())
+                .saturating_add(item.version_id.len())
+        })
+}
+
+fn batch_read_version_response_payload_len(response: &BatchReadVersionResponse) -> usize {
+    response
+        .batch_read_version_resps
+        .iter()
+        .map(String::len)
+        .sum::<usize>()
+        .saturating_add(response.batch_read_version_resps_bin.iter().map(Bytes::len).sum::<usize>())
+}
+
 fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
    file_info.validate_for_metadata_read().map_err(Into::into)
 }
@@ -2837,14 +2869,19 @@ impl DiskAPI for RemoteDisk {
            state = "started",
            "Remote disk RPC started"
        );
+        let batch_read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
+        let encode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
        let batch_read_version_req = compat_json(&req)?;
        let batch_read_version_req_bin = encode_msgpack(&req)?;
-
+        record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, encode_started);
+        let request_payload_bytes = batch_read_version_attribution_enabled
+            .then(|| batch_read_version_request_payload_len(&req, &batch_read_version_req, &batch_read_version_req_bin));
        let batch_result = self
            .execute_with_timeout_for_op(
                "batch_read_version",
                move || async move {
                    let disk = self.disk_ref().await;
+                    let disk_len = disk.len();
                    let mut client = self
                        .get_bulk_client()
                        .await
@@ -2855,9 +2892,20 @@ impl DiskAPI for RemoteDisk {
                        batch_read_version_req_bin: batch_read_version_req_bin.into(),
                    });

+                    crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request();
+                    if let Some(request_payload_bytes) = request_payload_bytes {
+                        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_sent_bytes(
+                            request_payload_bytes.saturating_add(disk_len),
+                        );
+                    }
+                    let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
                    let response = match client.batch_read_version(request).await {
-                        Ok(response) => response.into_inner(),
+                        Ok(response) => {
+                            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
+                            response.into_inner()
+                        }
                        Err(status) if status.code() == Code::Unimplemented => {
+                            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
                            if mode.should_fallback_on_unimplemented() {
                                record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED);
                                warn!(
@@ -2874,6 +2922,7 @@ impl DiskAPI for RemoteDisk {
                            }

                            record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK);
+                            crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
                            warn!(
                                event = EVENT_REMOTE_DISK_RPC,
                                component = LOG_COMPONENT_ECSTORE,
@@ -2886,14 +2935,33 @@ impl DiskAPI for RemoteDisk {
                            );
                            return Err(Error::from(status));
                        }
-                        Err(status) => return Err(Error::from(status)),
+                        Err(status) => {
+                            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
+                            crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
+                            return Err(Error::from(status));
+                        }
                    };

                    if !response.success {
+                        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
                        return Err(response.error.unwrap_or_default().into());
                    }

-                    decode_batch_read_version_response_items(response, &self.endpoint).map(Some)
+                    crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_recv_bytes(
+                        batch_read_version_response_payload_len(&response),
+                    );
+                    let decode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
+                    match decode_batch_read_version_response_items(response, &self.endpoint) {
+                        Ok(batch_read_version_resps) => {
+                            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
+                            Ok(Some(batch_read_version_resps))
+                        }
+                        Err(err) => {
+                            record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
+                            crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
+                            Err(err)
+                        }
+                    }
                },
                get_max_timeout_duration(),
            )
@@ -3974,10 +4042,21 @@ mod tests {
        NsScannerProbe(NsScannerCapabilityRequest),
    }

-    #[derive(Debug, Clone, Default)]
+    #[derive(Debug, Clone)]
    struct RecordingInternodeDataTransport {
        calls: Arc<StdMutex<Vec<RecordedTransportCall>>>,
        ns_scanner_probe_status: Arc<StdMutex<Option<u16>>>,
+        ns_scanner_generation_support: Arc<StdMutex<Option<bool>>>,
+    }
+
+    impl Default for RecordingInternodeDataTransport {
+        fn default() -> Self {
+            Self {
+                calls: Arc::default(),
+                ns_scanner_probe_status: Arc::default(),
+                ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
+            }
+        }
    }

    #[derive(Clone, Debug)]
@@ -4197,6 +4276,15 @@ mod tests {
            Self {
                calls: Arc::default(),
                ns_scanner_probe_status: Arc::new(StdMutex::new(Some(status))),
+                ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))),
+            }
+        }
+
+        fn with_ns_scanner_generation_support(support: Option<bool>) -> Self {
+            Self {
+                calls: Arc::default(),
+                ns_scanner_probe_status: Arc::default(),
+                ns_scanner_generation_support: Arc::new(StdMutex::new(support)),
            }
        }

@@ -4621,6 +4709,7 @@ mod tests {
            } else {
                "file version not found".to_string()
            },
+            error_code: if success { 0 } else { DiskError::FileVersionNotFound.to_u32() },
        }
    }

@@ -4740,6 +4829,7 @@ mod tests {
    fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() {
        temp_env::with_vars(
            [
+                (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
                (ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")),
                (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
            ],
@@ -4749,10 +4839,25 @@ mod tests {
        );
    }

+    #[test]
+    fn batch_metadata_rpc_mode_uses_get_coalescer_env_before_batch_env() {
+        temp_env::with_vars(
+            [
+                (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("on")),
+                (ENV_RUSTFS_METADATA_BATCH_READ, Some("off")),
+                (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("off")),
+            ],
+            || {
+                assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On);
+            },
+        );
+    }
+
    #[test]
    fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() {
        temp_env::with_vars(
            [
+                (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
                (ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>),
                (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
            ],
@@ -4862,6 +4967,23 @@ mod tests {
            Ok(Uuid::from_u128(1))
        }

+        async fn probe_ns_scanner_capability(
+            &self,
+            request: NsScannerCapabilityRequest,
+        ) -> Result<crate::storage_api_contracts::internode::NsScannerCapabilityResponse> {
+            let server_epoch = self.probe_ns_scanner(request).await?;
+            let supports_tier_registry_generation = *self
+                .ns_scanner_generation_support
+                .lock()
+                .expect("namespace scanner generation support lock poisoned");
+            Ok(crate::storage_api_contracts::internode::NsScannerCapabilityResponse {
+                version: crate::storage_api_contracts::internode::NS_SCANNER_PROTOCOL_VERSION,
+                server_epoch,
+                proof: Vec::new(),
+                supports_tier_registry_generation,
+            })
+        }
+
        fn name(&self) -> &'static str {
            "recording"
        }
@@ -6674,6 +6796,22 @@ mod tests {
        }
    }

+    #[tokio::test]
+    async fn test_remote_disk_namespace_scanner_capability_falls_back_without_generation_support() {
+        for support in [None, Some(false)] {
+            let transport = RecordingInternodeDataTransport::with_ns_scanner_generation_support(support);
+            let remote_disk = new_remote_disk_with_transport(Arc::new(transport)).await;
+
+            assert_eq!(
+                remote_disk
+                    .ns_scanner_server_epoch()
+                    .await
+                    .expect("missing generation support should be classified as unsupported"),
+                None
+            );
+        }
+    }
+
    #[tokio::test]
    async fn test_remote_disk_namespace_scanner_capability_rejects_legacy_transport() {
        let remote_disk = new_remote_disk_with_transport(Arc::new(RetryingOpenReadInternodeDataTransport::default())).await;
@@ -14,9 +14,10 @@

 use rustfs_io_metrics::internode_metrics::{
    INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE,
-    INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION,
-    INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
-    INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
+    INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
+    INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM,
+    INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
+    global_internode_metrics,
 };
 use std::time::Duration;

@@ -93,6 +94,59 @@ pub(crate) fn record_remote_disk_grpc_read_version_request() {
    );
 }

+pub(crate) fn record_remote_disk_grpc_batch_read_version_request() {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics().record_outgoing_request_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+    );
+}
+
+pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static str, duration: Duration) {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics().record_stage_duration_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        stage,
+        duration,
+    );
+}
+
+pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics()
+        .record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
+}
+
+pub(crate) fn record_remote_disk_grpc_batch_read_version_sent_bytes(bytes: usize) {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics().record_sent_bytes_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        bytes,
+    );
+}
+
+pub(crate) fn record_remote_disk_grpc_batch_read_version_recv_bytes(bytes: usize) {
+    if !rustfs_io_metrics::get_stage_metrics_enabled() {
+        return;
+    }
+    global_internode_metrics().record_recv_bytes_for_operation_and_backend(
+        INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
+        INTERNODE_TRANSPORT_BACKEND_GRPC,
+        bytes,
+    );
+    record_grpc_payload_size(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, bytes);
+}
+
 pub(crate) fn record_remote_disk_grpc_read_version_error() {
    if !rustfs_io_metrics::get_stage_metrics_enabled() {
        return;
@@ -406,6 +406,25 @@ where
    Ok(data)
 }

+pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
+where
+    S: EcstoreObjectIO,
+{
+    let (data, _obj) = read_config_limited_preserve_empty_with_metadata(api, file, max_bytes).await?;
+    Ok(data)
+}
+
+pub(crate) async fn read_config_limited_preserve_empty_with_metadata<S>(
+    api: Arc<S>,
+    file: &str,
+    max_bytes: usize,
+) -> Result<(Vec<u8>, ObjectInfo)>
+where
+    S: EcstoreObjectIO,
+{
+    read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, Some(max_bytes)).await
+}
+
 /// Read an existing config object without treating an empty payload as absent.
 /// Callers that validate their own payload format need to distinguish corruption
 /// from `ConfigNotFound`.
@@ -248,10 +248,13 @@ impl Config {
        let shard_size = shard_size as usize;
        // Keep the historical two-data-shard object budget while preventing
        // wider EC layouts from multiplying the maximum inline object size.
+        // Use div_ceil to match the shard_file_size calculation (which also uses
+        // div_ceil), avoiding a 1-byte rounding discrepancy that prevents inline
+        // for objects right at the threshold.
        let inline_block = if self.initialized && self.inline_block_explicit {
            self.inline_block
        } else {
-            (DEFAULT_INLINE_OBJECT_BUDGET / data_shards).min(DEFAULT_INLINE_BLOCK)
+            DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK)
        };

        if versioned {
@@ -21,7 +21,7 @@ use crate::storage_api_contracts::{
    bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
    list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
    multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo},
-    object::{DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
+    object::{DeleteAccounting, DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
    range::HTTPRangeSpec,
 };
 use crate::{
@@ -414,6 +414,66 @@ fn apply_delete_objects_results(
    }
 }

+fn apply_delete_accounting_results(
+    accounting: &mut [Option<DeleteAccounting>],
+    set_objects: &[DelObj],
+    set_accounting: &[Option<DeleteAccounting>],
+) {
+    for (obj, value) in set_objects.iter().zip(set_accounting.iter()) {
+        accounting[obj.orig_idx] = value.clone();
+    }
+}
+
+impl Sets {
+    pub(crate) async fn delete_objects_with_accounting(
+        &self,
+        bucket: &str,
+        objects: Vec<ObjectToDelete>,
+        opts: ObjectOptions,
+    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
+        let mut del_objects = vec![DeletedObject::default(); objects.len()];
+        let mut del_errs = vec![None; objects.len()];
+        let mut accounting = vec![None; objects.len()];
+        let mut set_obj_map = HashMap::new();
+
+        for (i, obj) in objects.iter().enumerate() {
+            let idx = self.get_hashed_set_index(obj.object_name.as_str());
+            set_obj_map.entry(idx).or_insert_with(Vec::new).push(DelObj {
+                orig_idx: i,
+                obj: obj.clone(),
+            });
+        }
+
+        let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
+        let mut futures = FuturesUnordered::new();
+        let bucket = bucket.to_owned();
+
+        for (set_index, set_objects) in set_obj_map {
+            let disks = self.get_disks(set_index);
+            let objects = set_objects.iter().map(|entry| entry.obj.clone()).collect::<Vec<_>>();
+            let bucket = bucket.clone();
+            let opts = opts.clone();
+            let semaphore = semaphore.clone();
+            futures.push(async move {
+                let _permit = semaphore
+                    .acquire_owned()
+                    .await
+                    .expect("delete_objects semaphore should remain open");
+                let (deleted, errors, accounting) = disks.delete_objects_with_accounting(&bucket, objects, opts).await;
+                (set_objects, deleted, errors, accounting)
+            });
+        }
+
+        while let Some((set_objects, deleted, errors, set_accounting)) = futures.next().await {
+            apply_delete_objects_results(&mut del_objects, &mut del_errs, &set_objects, &deleted, errors);
+            apply_delete_accounting_results(&mut accounting, &set_objects, &set_accounting);
+        }
+
+        (del_objects, del_errs, accounting)
+    }
+}
+
 #[async_trait::async_trait]
 impl crate::storage_api_contracts::object::ObjectIO for Sets {
    type Error = Error;
@@ -655,65 +715,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
        objects: Vec<ObjectToDelete>,
        opts: ObjectOptions,
    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
-        // Default return value
-        let mut del_objects = vec![DeletedObject::default(); objects.len()];
-
-        let mut del_errs = Vec::with_capacity(objects.len());
-        for _ in 0..objects.len() {
-            del_errs.push(None)
-        }
-
-        let mut set_obj_map = HashMap::new();
-
-        // hash key
-        for (i, obj) in objects.iter().enumerate() {
-            let idx = self.get_hashed_set_index(obj.object_name.as_str());
-
-            if !set_obj_map.contains_key(&idx) {
-                set_obj_map.insert(
-                    idx,
-                    vec![DelObj {
-                        // set_idx: idx,
-                        orig_idx: i,
-                        obj: obj.clone(),
-                    }],
-                );
-            } else if let Some(val) = set_obj_map.get_mut(&idx) {
-                val.push(DelObj {
-                    // set_idx: idx,
-                    orig_idx: i,
-                    obj: obj.clone(),
-                });
-            }
-        }
-
-        let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
-        let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
-        let mut futures = FuturesUnordered::new();
-        let bucket = bucket.to_string();
-
-        for (k, v) in set_obj_map {
-            let disks = self.get_disks(k);
-            let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
-            let bucket = bucket.clone();
-            let opts = opts.clone();
-            let semaphore = semaphore.clone();
-
-            futures.push(async move {
-                let _permit = semaphore
-                    .acquire_owned()
-                    .await
-                    .expect("delete_objects semaphore should remain open");
-                let (dobjects, errs) = disks.delete_objects(&bucket, objs, opts).await;
-                (v, dobjects, errs)
-            });
-        }
-
-        while let Some((v, dobjects, errs)) = futures.next().await {
-            apply_delete_objects_results(&mut del_objects, &mut del_errs, &v, &dobjects, errs);
-        }
-
-        (del_objects, del_errs)
+        let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
+        (deleted, errors)
    }

    #[tracing::instrument(skip(self))]
@@ -985,14 +988,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for Sets {
    }
 }

-#[async_trait::async_trait]
-impl crate::storage_api_contracts::heal::HealOperations for Sets {
-    type Error = Error;
-    type HealResultItem = HealResultItem;
-    type HealOptions = HealOpts;
-
-    #[tracing::instrument(skip(self))]
-    async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
+impl Sets {
+    pub(crate) async fn heal_format_with_fence<F>(&self, dry_run: bool, fence_lost: F) -> Result<(HealResultItem, Option<Error>)>
+    where
+        F: Fn() -> bool + Send + Sync,
+    {
        let (disks, init_errs) = init_storage_disks_with_errors(
            &self.endpoints.endpoints,
            &DiskOption {
@@ -1065,6 +1065,9 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
            // Save new formats `format.json` on unformatted disks.
            for (index, (fm, disk)) in tmp_new_formats.iter_mut().zip(disks.iter()).enumerate() {
                if fm.is_some() && disk.is_some() {
+                    if fence_lost() {
+                        return Ok((res, Some(StorageError::SlowDown)));
+                    }
                    if let Err(err) = save_format_file(disk, fm).await {
                        if let Some(disk) = disk.as_ref() {
                            let _ = disk.close().await;
@@ -1098,6 +1101,18 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
        }
        Ok((res, None))
    }
+}
+
+#[async_trait::async_trait]
+impl crate::storage_api_contracts::heal::HealOperations for Sets {
+    type Error = Error;
+    type HealResultItem = HealResultItem;
+    type HealOptions = HealOpts;
+
+    #[tracing::instrument(skip(self))]
+    async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
+        self.heal_format_with_fence(dry_run, || false).await
+    }
    #[tracing::instrument(skip(self))]
    async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
        let mut result = HealResultItem {
@@ -26,7 +26,7 @@ use crate::storage_api_contracts::{
    namespace::NamespaceLocking as _,
    object::{HTTPPreconditions, ObjectOperations as _},
 };
-use crate::store::ECStore;
+use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence};
 use bytes::Bytes;
 use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
 use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
@@ -813,7 +813,7 @@ pub(crate) fn is_equivalent_data_movement_metadata(
            .all(|(key, value)| source.user_defined.get(key) == Some(value))
 }

-fn is_equivalent_data_movement_object_identity(
+pub(crate) fn is_equivalent_data_movement_object_identity(
    source: &ObjectInfo,
    target: &ObjectInfo,
    compare_mod_time: bool,
@@ -856,7 +856,6 @@ fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo)
 fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool {
    is_unversioned_data_movement_object(source)
        && is_unversioned_data_movement_object(target)
-        && !target.delete_marker
        && source
            .mod_time
            .zip(target.mod_time)
@@ -1028,6 +1027,7 @@ pub(crate) enum SourceCleanupError {
 pub(crate) struct SourceCleanupBucketFence<'a> {
    pub(crate) expected_incarnation_id: Option<uuid::Uuid>,
    pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
+    pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>,
 }

 fn ensure_source_cleanup_versions_match(
@@ -1065,7 +1065,9 @@ pub(crate) async fn ensure_source_cleanup_versions_unchanged(
 struct SourceCleanupDeleteBarrierState {
    bucket: String,
    object: String,
+    fence_pending: tokio::sync::Notify,
    arrived: tokio::sync::Notify,
+    is_paused: AtomicBool,
    release: tokio::sync::Notify,
 }

@@ -1079,7 +1081,7 @@ pub(crate) struct SourceCleanupDeleteBarrier {
 }

 #[cfg(test)]
-static SOURCE_CLEANUP_DELETE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<SourceCleanupDeleteBarrierState>>>> =
+static SOURCE_CLEANUP_DELETE_BARRIERS: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<SourceCleanupDeleteBarrierState>>>> =
    std::sync::OnceLock::new();

 #[cfg(test)]
@@ -1092,15 +1094,22 @@ impl SourceCleanupDeleteBarrier {
        let state = Arc::new(SourceCleanupDeleteBarrierState {
            bucket: bucket.to_string(),
            object: object.to_string(),
+            fence_pending: tokio::sync::Notify::new(),
            arrived: tokio::sync::Notify::new(),
+            is_paused: AtomicBool::new(false),
            release: tokio::sync::Notify::new(),
        });
-        let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
-            .get_or_init(|| std::sync::Mutex::new(None))
+        let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
+            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
            .lock()
            .expect("source cleanup delete barrier mutex should not poison");
-        assert!(slot.is_none(), "source cleanup delete barrier must be unique");
-        *slot = Some(Arc::clone(&state));
+        assert!(
+            !barriers
+                .iter()
+                .any(|barrier| barrier.bucket == bucket && barrier.object == object),
+            "source cleanup delete barrier must be unique per object"
+        );
+        barriers.push(Arc::clone(&state));
        Self { state }
    }

@@ -1110,35 +1119,58 @@ impl SourceCleanupDeleteBarrier {
            .expect("source cleanup should reach the pre-delete barrier");
    }

+    pub(crate) async fn wait_until_fence_pending(&self) {
+        tokio::time::timeout(StdDuration::from_secs(30), self.state.fence_pending.notified())
+            .await
+            .expect("source cleanup should attempt the fixed mutation fence");
+    }
+
+    pub(crate) fn is_paused(&self) -> bool {
+        self.state.is_paused.load(Ordering::Acquire)
+    }
+
    pub(crate) fn release(&self) {
        self.state.release.notify_one();
    }
 }

+#[cfg(test)]
+pub(crate) fn notify_source_cleanup_mutation_fence_pending(bucket: &str, object: &str) {
+    let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
+        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
+        .lock()
+        .expect("source cleanup delete barrier mutex should not poison")
+        .iter()
+        .find(|barrier| barrier.bucket == bucket && barrier.object == object)
+        .cloned();
+    if let Some(barrier) = barrier {
+        barrier.fence_pending.notify_one();
+    }
+}
+
 #[cfg(test)]
 impl Drop for SourceCleanupDeleteBarrier {
    fn drop(&mut self) {
        self.state.release.notify_one();
-        let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
-            .get_or_init(|| std::sync::Mutex::new(None))
+        let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
+            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
            .lock()
            .expect("source cleanup delete barrier mutex should not poison");
-        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
-            *slot = None;
-        }
+        barriers.retain(|state| !Arc::ptr_eq(state, &self.state));
    }
 }

 #[cfg(test)]
 async fn pause_source_cleanup_before_delete(bucket: &str, object: &str) {
-    let barrier = SOURCE_CLEANUP_DELETE_BARRIER
-        .get_or_init(|| std::sync::Mutex::new(None))
+    let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
+        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
        .lock()
        .expect("source cleanup delete barrier mutex should not poison")
-        .as_ref()
-        .filter(|barrier| barrier.bucket == bucket && barrier.object == object)
+        .iter()
+        .find(|barrier| barrier.bucket == bucket && barrier.object == object)
        .cloned();
    if let Some(barrier) = barrier {
+        barrier.is_paused.store(true, Ordering::Release);
        barrier.arrived.notify_one();
        barrier.release.notified().await;
    }
@@ -1154,11 +1186,20 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
    op_label: &str,
 ) -> std::result::Result<ObjectInfo, SourceCleanupError> {
    let cleanup_key = encode_dir_object(object);
-    let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
-    let _guard = ns_lock
-        .get_write_lock(get_lock_acquire_timeout())
-        .await
-        .map_err(Error::from)?;
+    let source_guard = if bucket_fence
+        .object_mutation_fence
+        .is_some_and(SourceCleanupMutationFence::source_lock_covered)
+    {
+        None
+    } else {
+        let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
+        Some(
+            ns_lock
+                .get_write_lock(get_lock_acquire_timeout())
+                .await
+                .map_err(Error::from)?,
+        )
+    };

    if bucket_fence
        .lifecycle_guard
@@ -1168,6 +1209,14 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
            "{op_label}: bucket incarnation fence was lost before source cleanup"
        ))));
    }
+    if bucket_fence
+        .object_mutation_fence
+        .is_some_and(SourceCleanupMutationFence::is_lock_lost)
+    {
+        return Err(SourceCleanupError::Storage(Error::other(format!(
+            "{op_label}: object mutation fence was lost before source cleanup"
+        ))));
+    }

    ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?;

@@ -1182,7 +1231,12 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
        expected_bucket_incarnation_id: bucket_fence.expected_incarnation_id,
        ..Default::default()
    };
-    opts.add_namespace_lock_guard(&_guard);
+    if let Some(source_guard) = source_guard.as_ref() {
+        opts.add_namespace_lock_guard(source_guard);
+    }
+    if let Some(object_mutation_fence) = bucket_fence.object_mutation_fence {
+        object_mutation_fence.add_namespace_lock_fence(&mut opts);
+    }
    if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard {
        opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard);
    }
@@ -1330,6 +1384,37 @@ fn data_movement_part_upload_failure_stage(err: &Error) -> &'static str {
    }
 }

+pub(crate) async fn migrate_decommission_object(
+    store: Arc<ECStore>,
+    pool_idx: usize,
+    bucket: String,
+    rd: GetObjectReader,
+    source_bucket_incarnation_id: Option<uuid::Uuid>,
+    op_label: &str,
+) -> Result<()> {
+    let source = rd.object_info.clone();
+    let _mutation_fence = store
+        .acquire_decommission_object_mutation_fence(&bucket, &source.name)
+        .await?;
+    let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source)
+        .await?
+        .ok_or(Error::FileNotFound)?;
+    if !is_equivalent_data_movement_object_identity(&source, &current, true, false) {
+        return Err(Error::FileNotFound);
+    }
+
+    migrate_object_inner(
+        store,
+        pool_idx,
+        bucket,
+        rd,
+        source_bucket_incarnation_id,
+        op_label,
+        Some(&_mutation_fence),
+    )
+    .await
+}
+
 pub(crate) async fn migrate_object(
    store: Arc<ECStore>,
    pool_idx: usize,
@@ -1337,6 +1422,18 @@ pub(crate) async fn migrate_object(
    rd: GetObjectReader,
    source_bucket_incarnation_id: Option<uuid::Uuid>,
    op_label: &str,
+) -> Result<()> {
+    migrate_object_inner(store, pool_idx, bucket, rd, source_bucket_incarnation_id, op_label, None).await
+}
+
+async fn migrate_object_inner(
+    store: Arc<ECStore>,
+    pool_idx: usize,
+    bucket: String,
+    rd: GetObjectReader,
+    source_bucket_incarnation_id: Option<uuid::Uuid>,
+    op_label: &str,
+    mutation_fence: Option<&ObjectLockDiagGuard>,
 ) -> Result<()> {
    let object_info = rd.object_info.clone();
    let has_part_checksums = object_info
@@ -1350,7 +1447,7 @@ pub(crate) async fn migrate_object(
        let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx);
        new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
        let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
-            .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts)
+            .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
            .await
        {
            Ok(res) => res,
@@ -1448,7 +1545,7 @@ pub(crate) async fn migrate_object(
            if let Err(err) = store
                .clone()
                .complete_multipart_upload_for_data_movement(
-                    target_pool_idx,
+                    (target_pool_idx, mutation_fence),
                    &bucket,
                    &object_info.name,
                    &res.upload_id,
@@ -1609,7 +1706,7 @@ pub(crate) async fn migrate_object(
    let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx);
    put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
    let (target_pool_idx, put_result) = store
-        .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts)
+        .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
        .await
        .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?;
    if let Err(err) = put_result {
@@ -3541,25 +3638,47 @@ mod tests {
    }

    #[test]
-    fn test_precondition_conflict_rejects_newer_delete_marker() {
-        let source = ObjectInfo {
-            size: 128,
-            etag: Some("etag-source".to_string()),
-            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
-            ..Default::default()
-        };
-        let target = ObjectInfo {
-            delete_marker: true,
-            etag: None,
-            mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
-            ..source.clone()
-        };
+    fn test_precondition_conflict_accepts_only_newer_null_delete_marker() {
+        for version_id in [None, Some(Uuid::nil())] {
+            let source = ObjectInfo {
+                version_id,
+                size: 128,
+                etag: Some("etag-source".to_string()),
+                mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+                ..Default::default()
+            };
+            let target = ObjectInfo {
+                delete_marker: true,
+                etag: None,
+                mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
+                ..source.clone()
+            };

-        let should_resume =
-            resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
-                .expect("delete marker conflict should be evaluated");
+            assert!(
+                resolve_data_movement_overwrite_resume_result(
+                    &Error::PreconditionFailed,
+                    Ok(Some(target.clone())),
+                    &source,
+                    0,
+                    1,
+                )
+                .expect("newer null delete marker should be evaluated")
+            );

-        assert!(!should_resume);
+            let mut same_time = target.clone();
+            same_time.mod_time = source.mod_time;
+            assert!(
+                !resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(same_time)), &source, 0, 1,)
+                    .expect("same-generation null delete marker should be rejected")
+            );
+
+            let mut versioned = target;
+            versioned.version_id = Some(Uuid::new_v4());
+            assert!(
+                !resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(versioned)), &source, 0, 1,)
+                    .expect("a UUID delete marker must not erase a null source version")
+            );
+        }
    }

    #[test]
@@ -73,6 +73,16 @@ struct CachedBucketUsage {
    // mutation. A strictly later generation is required before the mutation
    // evidence can be discarded.
    pending_scanner_position: Option<(u64, u64)>,
+    // Deletes are visible to admin immediately, but quota admission keeps
+    // them pending until a complete scanner generation reconciles the set.
+    // This marker intentionally remains process-local: the delete request
+    // updates this overlay before the scanner writes a durable snapshot. If
+    // the process restarts first, loading the persisted complete snapshot
+    // restores the pre-reconciliation (larger) baseline, which is
+    // conservative for quota admission. A persisted post-delete snapshot is
+    // necessarily a complete scanner reconciliation and therefore creates a
+    // fresh cache entry with no pending hold.
+    pending_negative_delta: u64,
 }

 type UsageMemoryCache = Arc<RwLock<HashMap<String, CachedBucketUsage>>>;
@@ -378,8 +388,12 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
            "nonconverged data usage observations cannot replace the quota-authoritative snapshot",
        ));
    }
+    let Some(expected_publication_epoch) = store.scanner_data_usage_publication_epoch().await else {
+        return Err(Error::other("data usage publication is blocked by data movement"));
+    };
    // Prevent older data from overwriting newer persisted stats
-    if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await
+    let existing_snapshot = load_data_usage_snapshot(store.clone()).await;
+    if let Ok((existing, source)) = existing_snapshot
        && source.is_authoritative()
        && let Some(reason) = stale_data_usage_persist_reason_for_source(&data_usage_info, &existing, source, SystemTime::now())
    {
@@ -390,19 +404,31 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
        return Ok(());
    }

-    save_data_usage_in_backend(data_usage_info, store).await
+    save_data_usage_in_backend(data_usage_info, store, expected_publication_epoch).await
 }

-async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
+async fn save_data_usage_in_backend(
+    data_usage_info: DataUsageInfo,
+    store: Arc<ECStore>,
+    expected_publication_epoch: u64,
+) -> Result<(), Error> {
    let data =
        serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;

    // Save to backend using the same mechanism as original code
+    let Some((publication_guard, publication_epoch)) = store.scanner_data_usage_publication_admission_guard().await else {
+        return Err(Error::other("data usage publication is blocked by data movement"));
+    };
+    if publication_epoch != expected_publication_epoch {
+        return Err(Error::other("data usage publication epoch changed before save"));
+    }
    crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
        .await
        .map_err(Error::other)?;
+    drop(publication_guard);

-    cleanup_observed_data_usage_after_authoritative_save(store.as_ref(), &data_usage_info).await;
+    cleanup_observed_data_usage_after_authoritative_save_with_publication(store.as_ref(), &data_usage_info, Some(store.as_ref()))
+        .await;

    // Invalidate the cached snapshot so readers observe the new save on their
    // next request instead of waiting out the remaining TTL. The next cached
@@ -439,11 +465,24 @@ impl ObservedDataUsageSnapshotCleanup for ECStore {
    }
 }

-async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
-where
+async fn cleanup_observed_data_usage_after_authoritative_save_with_publication<S>(
+    store: &S,
+    authoritative: &DataUsageInfo,
+    publication_store: Option<&ECStore>,
+) where
    S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
 {
-    let (observed, revision) = match load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
+    let observed_read_epoch = match publication_store {
+        Some(publication_store) => {
+            let Some(epoch) = publication_store.scanner_data_usage_publication_epoch().await else {
+                return;
+            };
+            Some(epoch)
+        }
+        None => None,
+    };
+    let observed_snapshot = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await;
+    let (observed, revision) = match observed_snapshot {
        Ok(Some(snapshot)) => snapshot,
        Ok(None) => return,
        Err(err) => {
@@ -459,6 +498,19 @@ where
        return;
    }

+    let publication_guard = match publication_store {
+        Some(publication_store) => {
+            let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
+            else {
+                return;
+            };
+            if observed_read_epoch.is_some_and(|expected| expected != publication_epoch) {
+                return;
+            }
+            Some(guard)
+        }
+        None => None,
+    };
    match store.delete_observed_data_usage_snapshot(&revision).await {
        Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::PreconditionFailed) => {}
        Err(err) => {
@@ -469,6 +521,15 @@ where
            );
        }
    }
+    drop(publication_guard);
+}
+
+#[cfg(test)]
+async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
+where
+    S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
+{
+    cleanup_observed_data_usage_after_authoritative_save_with_publication(store, authoritative, None).await;
 }

 fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) {
@@ -509,7 +570,7 @@ pub async fn remove_bucket_usage_from_backend(store: Arc<ECStore>, bucket: &str)

 pub(crate) async fn remove_bucket_usage_for_namespace_change(store: &ECStore, bucket: &str) -> Result<(), Error> {
    prepare_bucket_usage_for_namespace_change(bucket, None).await?;
-    remove_bucket_usage_from_backend_with_guard(store, bucket, None).await
+    remove_bucket_usage_from_backend_with_guard_fenced(store, bucket, None).await
 }

 pub(crate) async fn prepare_bucket_usage_for_namespace_change(
@@ -532,6 +593,7 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
    Ok(())
 }

+#[cfg(test)]
 pub(crate) async fn remove_bucket_usage_from_backend_with_guard<S>(
    store: &S,
    bucket: &str,
@@ -541,6 +603,24 @@ where
    S: EcstoreObjectIO + ?Sized,
 {
    let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await;
+    invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
+    result
+}
+
+pub(crate) async fn remove_bucket_usage_from_backend_with_guard_fenced(
+    store: &ECStore,
+    bucket: &str,
+    guard: Option<&rustfs_lock::NamespaceLockGuard>,
+) -> Result<(), Error> {
+    let result = remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, Some(store)).await;
+    invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
+    result
+}
+
+async fn invalidate_bucket_usage_snapshot_caches(
+    guard: Option<&rustfs_lock::NamespaceLockGuard>,
+    bucket: &str,
+) -> Result<(), Error> {
    let mut snapshot_cache = data_usage_snapshot_cache().write().await;
    ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
    clear_data_usage_snapshot_cache(&mut snapshot_cache);
@@ -548,7 +628,7 @@ where
    let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
    ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache invalidation")?;
    clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
-    result
+    Ok(())
 }

 async fn load_data_usage_for_bucket_removal<S>(store: &S, object: &str) -> Result<Option<(DataUsageInfo, String)>, Error>
@@ -607,48 +687,83 @@ fn ensure_bucket_namespace_guard(
    Ok(())
 }

+#[cfg(test)]
 async fn remove_bucket_usage_from_backend_with_store_and_guard<S>(
    store: &S,
    bucket: &str,
    guard: Option<&rustfs_lock::NamespaceLockGuard>,
 ) -> Result<(), Error>
+where
+    S: EcstoreObjectIO + ?Sized,
+{
+    remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, None).await
+}
+
+async fn remove_bucket_usage_from_backend_with_store_and_guard_and_publication<S>(
+    store: &S,
+    bucket: &str,
+    guard: Option<&rustfs_lock::NamespaceLockGuard>,
+    publication_store: Option<&ECStore>,
+) -> Result<(), Error>
 where
    S: EcstoreObjectIO + ?Sized,
 {
    ensure_bucket_namespace_guard(guard, bucket, "data usage primary cleanup")?;
+    let primary_seed_epoch = match publication_store {
+        Some(publication_store) => Some(
+            publication_store
+                .scanner_data_usage_publication_epoch()
+                .await
+                .ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
+        ),
+        None => None,
+    };
    let primary_seed = load_data_usage_seed_for_missing_primary(store).await?;
-    remove_bucket_usage_from_object_with_retries(
+    remove_bucket_usage_from_object_with_retries_and_publication(
        store,
        DATA_USAGE_OBJ_NAME_PATH.as_str(),
        bucket,
        DATA_USAGE_REMOVE_CAS_RETRIES,
-        Some(&primary_seed),
+        primary_seed.as_ref(),
        guard,
+        publication_store.map(|store| (store, primary_seed_epoch)),
    )
    .await?;

    ensure_bucket_namespace_guard(guard, bucket, "data usage backup cleanup")?;
+    let backup_seed_epoch = match publication_store {
+        Some(publication_store) => Some(
+            publication_store
+                .scanner_data_usage_publication_epoch()
+                .await
+                .ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
+        ),
+        None => None,
+    };
    let backup_seed = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
        .await?
-        .map_or(primary_seed, |(data_usage_info, _)| data_usage_info);
-    remove_bucket_usage_from_object_with_retries(
+        .map(|(data_usage_info, _)| data_usage_info)
+        .or_else(|| primary_seed.clone());
+    remove_bucket_usage_from_object_with_retries_and_publication(
        store,
        DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
        bucket,
        DATA_USAGE_REMOVE_CAS_RETRIES,
-        Some(&backup_seed),
+        backup_seed.as_ref(),
        guard,
+        publication_store.map(|store| (store, backup_seed_epoch)),
    )
    .await?;

    ensure_bucket_namespace_guard(guard, bucket, "observed data usage cleanup")?;
-    if let Err(err) = remove_bucket_usage_from_object_with_retries(
+    if let Err(err) = remove_bucket_usage_from_object_with_retries_and_publication(
        store,
        DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
        bucket,
        DATA_USAGE_REMOVE_CAS_RETRIES,
        None,
        guard,
+        publication_store.map(|store| (store, None)),
    )
    .await
    {
@@ -662,12 +777,21 @@ where
        LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
        LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
    ] {
-        remove_bucket_usage_from_object_with_retries(store, object, bucket, DATA_USAGE_REMOVE_CAS_RETRIES, None, guard).await?;
+        remove_bucket_usage_from_object_with_retries_and_publication(
+            store,
+            object,
+            bucket,
+            DATA_USAGE_REMOVE_CAS_RETRIES,
+            None,
+            guard,
+            publication_store.map(|store| (store, None)),
+        )
+        .await?;
    }
    Ok(())
 }

-async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<DataUsageInfo, Error>
+async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<Option<DataUsageInfo>, Error>
 where
    S: EcstoreObjectIO + ?Sized,
 {
@@ -680,12 +804,13 @@ where
            if !authoritative {
                data_usage_info.usage_snapshot_complete = false;
            }
-            return Ok(data_usage_info);
+            return Ok(Some(data_usage_info));
        }
    }
-    Ok(DataUsageInfo::default())
+    Ok(None)
 }

+#[cfg(test)]
 async fn remove_bucket_usage_from_object_with_retries<S>(
    store: &S,
    object: &str,
@@ -694,12 +819,42 @@ async fn remove_bucket_usage_from_object_with_retries<S>(
    missing_seed: Option<&DataUsageInfo>,
    guard: Option<&rustfs_lock::NamespaceLockGuard>,
 ) -> Result<(), Error>
+where
+    S: EcstoreObjectIO + ?Sized,
+{
+    remove_bucket_usage_from_object_with_retries_and_publication(store, object, bucket, cas_retries, missing_seed, guard, None)
+        .await
+}
+
+async fn remove_bucket_usage_from_object_with_retries_and_publication<S>(
+    store: &S,
+    object: &str,
+    bucket: &str,
+    cas_retries: usize,
+    missing_seed: Option<&DataUsageInfo>,
+    guard: Option<&rustfs_lock::NamespaceLockGuard>,
+    publication: Option<(&ECStore, Option<u64>)>,
+) -> Result<(), Error>
 where
    S: EcstoreObjectIO + ?Sized,
 {
    for attempt in 0..=cas_retries {
        ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cleanup")?;
-        let (mut data_usage_info, revision) = match load_data_usage_for_bucket_removal(store, object).await? {
+        let read_epoch = match publication {
+            Some((publication_store, expected_publication_epoch)) => {
+                let epoch = publication_store
+                    .scanner_data_usage_publication_epoch()
+                    .await
+                    .ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?;
+                if expected_publication_epoch.is_some_and(|expected| expected != epoch) {
+                    return Err(Error::other("data usage publication epoch changed before snapshot read"));
+                }
+                Some(epoch)
+            }
+            None => None,
+        };
+        let loaded_snapshot = load_data_usage_for_bucket_removal(store, object).await?;
+        let (mut data_usage_info, revision) = match loaded_snapshot {
            Some((data_usage_info, revision)) => (data_usage_info, Some(revision)),
            None => match missing_seed {
                Some(data_usage_info) => (data_usage_info.clone(), None),
@@ -723,6 +878,22 @@ where
            },
        };
        ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot commit")?;
+        let publication_guard = match publication {
+            Some((publication_store, expected_publication_epoch)) => {
+                let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
+                else {
+                    return Err(Error::other("data usage publication is blocked by data movement"));
+                };
+                if expected_publication_epoch
+                    .or(read_epoch)
+                    .is_some_and(|expected| expected != publication_epoch)
+                {
+                    return Err(Error::other("data usage publication epoch changed before snapshot commit"));
+                }
+                Some(guard)
+            }
+            None => None,
+        };
        let save_result = store
            .put_object(
                RUSTFS_META_BUCKET,
@@ -735,6 +906,7 @@ where
                },
            )
            .await;
+        drop(publication_guard);
        match save_result {
            Ok(_) => return Ok(()),
            Err(err) => {
@@ -948,7 +1120,12 @@ async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUs
    };

    match parse_usage_snapshot(&data) {
-        Ok(info) if info.usage_snapshot_converged == Some(false) && info.is_complete_bucket_usage_snapshot() => Some(info),
+        Ok(info)
+            if info.usage_snapshot_converged == Some(false)
+                && (info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()) =>
+        {
+            Some(info)
+        }
        Ok(_) => {
            error!(
                event = "data_usage_snapshot_load_failed",
@@ -993,7 +1170,7 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
 }

 fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
-    if !data_usage_info.is_complete_bucket_usage_snapshot() {
+    if !data_usage_info.is_complete_bucket_usage_snapshot() && !data_usage_info.usage_snapshot_partial {
        data_usage_info.usage_snapshot_complete = false;
        data_usage_info.buckets_usage.clear();
        data_usage_info.bucket_sizes.clear();
@@ -1391,7 +1568,37 @@ impl BucketUsageAccumulator {
 }

 pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
-    let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?;
+    // A compressed object may carry -1 while the transformed size is unknown
+    // (legacy streaming sentinel). In that case the persisted physical size
+    // is still a valid accounting floor; every other negative value is corrupt.
+    // An explicit negative `actual-size` metadata value is corrupt, however:
+    // the sentinel is only valid in the in-memory/object-part field written by
+    // the legacy streaming path, not as a persisted declared size.
+    let compressed = object.is_compressed();
+    if object.actual_size < -1 || (object.actual_size == -1 && !compressed) {
+        return Err(Error::PartMissingOrCorrupt);
+    }
+    if object
+        .parts
+        .iter()
+        .any(|part| part.actual_size < -1 || (part.actual_size < 0 && !compressed))
+    {
+        return Err(Error::PartMissingOrCorrupt);
+    }
+    let declared_actual_size = rustfs_utils::http::get_str(&object.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE)
+        .filter(|value| !value.is_empty());
+    if declared_actual_size
+        .as_deref()
+        .and_then(|value| value.parse::<i64>().ok())
+        .is_some_and(|size| size < 0)
+    {
+        return Err(Error::PartMissingOrCorrupt);
+    }
+    let logical_size = match object.get_actual_size().map_err(Error::other)? {
+        size if size == -1 && compressed && declared_actual_size.is_none() => None,
+        size if size >= 0 => Some(u64::try_from(size).map_err(|_| Error::PartMissingOrCorrupt)?),
+        _ => return Err(Error::PartMissingOrCorrupt),
+    };
    let persisted_part_size = if object.parts.is_empty() {
        u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)?
    } else {
@@ -1399,12 +1606,8 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
            // Compressed streaming objects persist -1 when the transformed
            // part size is unknown. The physical part size remains a valid
            // quota floor; reject only non-negative values that overflow.
-            let actual_size = if part.actual_size < 0 {
-                if object.is_compressed() {
-                    0
-                } else {
-                    return Err(Error::PartMissingOrCorrupt);
-                }
+            let actual_size = if part.actual_size == -1 {
+                0
            } else {
                u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?
            };
@@ -1412,7 +1615,7 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
            total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt)
        })?
    };
-    Ok(logical_size.max(persisted_part_size))
+    Ok(logical_size.unwrap_or(0).max(persisted_part_size))
 }

 type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>;
@@ -1617,6 +1820,7 @@ fn cached_bucket_usage_from_backend(usage: BucketUsageInfo, updated_at: SystemTi
        dirty: false,
        stale_snapshot_pending: false,
        pending_scanner_position: None,
+        pending_negative_delta: 0,
    }
 }

@@ -1630,6 +1834,7 @@ fn cached_bucket_usage_now(usage: BucketUsageInfo) -> CachedBucketUsage {
        dirty: false,
        stale_snapshot_pending: false,
        pending_scanner_position: None,
+        pending_negative_delta: 0,
    }
 }

@@ -1782,6 +1987,7 @@ pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64,
        .or_insert_with(|| cached_bucket_usage_now(BucketUsageInfo::default()));

    entry.usage.size = entry.usage.size.saturating_sub(deleted_size);
+    entry.pending_negative_delta = entry.pending_negative_delta.saturating_add(deleted_size);
    if removed_current_object {
        entry.usage.objects_count = entry.usage.objects_count.saturating_sub(1);
        entry.usage.versions_count = entry.usage.versions_count.saturating_sub(1);
@@ -1837,7 +2043,7 @@ pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
    cache
        .get(bucket)
        .filter(|cached| cached.authoritative)
-        .map(|cached| cached.usage.size)
+        .map(|cached| cached.usage.size.saturating_add(cached.pending_negative_delta))
 }

 async fn update_usage_cache_if_needed() {
@@ -2242,6 +2448,8 @@ pub async fn init_compression_total_memory_from_backend(store: Arc<ECStore>) {
 #[cfg(test)]
 mod tests {
    use super::*;
+    use crate::layout::endpoints::EndpointServerPools;
+    use crate::runtime::instance::InstanceContext;
    use crate::storage_api_contracts::object::ObjectIO as _;
    use rustfs_data_usage::BucketUsageInfo;
    use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
@@ -2264,6 +2472,7 @@ mod tests {
        error_after_commit_put: Option<usize>,
        advance_time_on_put: Option<Duration>,
        advance_time_after_get: Option<(UsageObjectSlot, Duration)>,
+        advance_publication_epoch_after_get: Option<(UsageObjectSlot, Arc<InstanceContext>)>,
        advance_time_before_put: Option<(usize, Duration)>,
        advance_time_after_put: Option<(usize, Duration)>,
        put_count: usize,
@@ -2341,10 +2550,21 @@ mod tests {
                }
                _ => None,
            };
+            let advance_publication_epoch = match state.advance_publication_epoch_after_get {
+                Some((expected_slot, ref ctx)) if expected_slot == slot => {
+                    let ctx = Arc::clone(ctx);
+                    state.advance_publication_epoch_after_get = None;
+                    Some(ctx)
+                }
+                _ => None,
+            };
            drop(state);
            if let Some(duration) = advance {
                tokio::time::advance(duration).await;
            }
+            if let Some(ctx) = advance_publication_epoch {
+                ctx.advance_data_movement_operation_epoch();
+            }
            Ok(crate::object_api::GetObjectReader {
                stream: Box::new(Cursor::new(data)),
                object_info: ObjectInfo {
@@ -2545,6 +2765,23 @@ mod tests {
            .to_string()
    }

+    fn build_publication_store(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
+        let endpoint_pools = EndpointServerPools::default();
+        Arc::new(ECStore {
+            id: uuid::Uuid::new_v4(),
+            disk_map: HashMap::new(),
+            pools: Vec::new(),
+            peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
+            pool_meta: RwLock::new(crate::core::pools::PoolMeta::default()),
+            rebalance_meta: RwLock::new(None),
+            decommission_cancelers: RwLock::new(Vec::new()),
+            start_gate: TokioMutex::new(()),
+            pool_meta_save_gate: TokioMutex::new(()),
+            ctx,
+            bucket_fence_registry: Arc::default(),
+        })
+    }
+
    #[test]
    fn data_usage_cache_absence_covers_the_variants_that_actually_arrive() {
        // `to_object_err` rewrites the raw storage variants before they reach
@@ -2917,6 +3154,45 @@ mod tests {
        assert_eq!(selected.usage_snapshot_converged, Some(true));
    }

+    #[test]
+    fn persisted_authoritative_stalls_but_memory_overlay_remains_visible() {
+        let authoritative = DataUsageInfo {
+            last_update: Some(SystemTime::UNIX_EPOCH),
+            scanner_epoch: Some(4),
+            scanner_cycle: Some(10),
+            usage_snapshot_complete: true,
+            ..Default::default()
+        };
+        let mut partial = authoritative.clone();
+        partial.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1));
+        partial.scanner_cycle = Some(11);
+        partial.usage_snapshot_complete = false;
+        partial.usage_snapshot_partial = true;
+        partial.usage_snapshot_converged = Some(false);
+        partial.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
+        partial.usage_snapshot_set_states = vec![rustfs_data_usage::DataUsageSnapshotSetState {
+            pool_index: 0,
+            set_index: 0,
+            scanner_cycle: Some(10),
+            scanner_epoch: Some(4),
+            scan_plan_digest: Some([1; 32]),
+            complete: false,
+            tombstone: false,
+        }];
+        partial.buckets_usage.insert(
+            "bucket".to_string(),
+            BucketUsageInfo {
+                size: 100,
+                ..Default::default()
+            },
+        );
+        partial.buckets_count = 1;
+
+        let (selected, _) = select_admin_data_usage_snapshot(authoritative, true, Some(partial));
+        assert!(selected.usage_snapshot_partial);
+        assert_eq!(selected.buckets_usage.get("bucket").map(|usage| usage.size), Some(100));
+    }
+
    #[tokio::test]
    async fn authoritative_save_cleanup_removes_observed_snapshot_best_effort() {
        let store = UsageCasStore::default();
@@ -3320,6 +3596,80 @@ mod tests {
        );
    }

+    #[test]
+    fn quota_object_size_accepts_compressed_unknown_actual_size_sentinel() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(
+            &mut metadata,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        let object = ObjectInfo {
+            size: 400,
+            actual_size: -1,
+            user_defined: Arc::new(metadata),
+            ..Default::default()
+        };
+
+        assert_eq!(quota_object_size(&object).expect("compressed sentinel is valid"), 400);
+    }
+
+    #[test]
+    fn quota_object_size_rejects_compressed_part_sum_overflow() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(
+            &mut metadata,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        let object = ObjectInfo {
+            size: 1,
+            user_defined: Arc::new(metadata),
+            parts: Arc::new(vec![
+                rustfs_filemeta::ObjectPartInfo {
+                    actual_size: i64::MAX,
+                    ..Default::default()
+                },
+                rustfs_filemeta::ObjectPartInfo {
+                    actual_size: 1,
+                    ..Default::default()
+                },
+            ]),
+            ..Default::default()
+        };
+
+        assert!(matches!(quota_object_size(&object), Err(Error::Io(_))));
+    }
+
+    #[test]
+    fn quota_object_size_rejects_negative_values_other_than_the_compressed_sentinel() {
+        let mut metadata = HashMap::new();
+        rustfs_utils::http::insert_str(
+            &mut metadata,
+            rustfs_utils::http::SUFFIX_COMPRESSION,
+            "klauspost/compress/s2".to_string(),
+        );
+        let corrupt_object = ObjectInfo {
+            size: 400,
+            actual_size: -2,
+            user_defined: Arc::new(metadata.clone()),
+            ..Default::default()
+        };
+        assert!(matches!(quota_object_size(&corrupt_object), Err(Error::PartMissingOrCorrupt)));
+
+        let corrupt_part = ObjectInfo {
+            size: 400,
+            user_defined: Arc::new(metadata),
+            parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
+                size: 400,
+                actual_size: -2,
+                ..Default::default()
+            }]),
+            ..Default::default()
+        };
+        assert!(matches!(quota_object_size(&corrupt_part), Err(Error::PartMissingOrCorrupt)));
+    }
+
    #[tokio::test]
    #[serial]
    async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() {
@@ -4176,7 +4526,15 @@ mod tests {
                .expect("namespace lock acquisition should not fail")
                .expect("namespace lock should be acquired"),
        );
-        let store = Arc::new(UsageCasStore::default());
+        let snapshot = data_usage_info_for_test(BUCKET, 2, 84, SystemTime::now());
+        let encoded = serde_json::to_vec(&snapshot).expect("usage snapshot should encode");
+        let store = Arc::new(UsageCasStore {
+            state: Mutex::new(UsageCasState {
+                object: Some((encoded.clone(), 1)),
+                backup_object: Some((encoded, 1)),
+                ..Default::default()
+            }),
+        });
        let successor = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now());
        let mut snapshot_cache = data_usage_snapshot_cache().write().await;
        *snapshot_cache = Some(CachedDataUsageSnapshot {
@@ -4275,21 +4633,17 @@ mod tests {
    }

    #[tokio::test]
-    async fn remove_bucket_usage_creates_primary_and_backup_fences_when_missing() {
+    async fn remove_bucket_usage_does_not_synthesize_authoritative_snapshot_when_all_missing() {
        let store = Arc::new(UsageCasStore::default());

        remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a")
            .await
-            .expect("bucket removal should create both usage fences");
+            .expect("bucket removal should remain a no-op without a usage baseline");

        let state = store.state.lock().await;
-        assert_eq!(state.put_count, 2);
-        for (data, revision) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() {
-            let saved = serde_json::from_slice::<DataUsageInfo>(data).expect("saved usage snapshot should decode");
-            assert_eq!(*revision, 1);
-            assert!(saved.last_update.is_some());
-            assert!(!data_usage_contains_bucket(&saved, "bucket-a"));
-        }
+        assert_eq!(state.put_count, 0);
+        assert!(state.object.is_none());
+        assert!(state.backup_object.is_none());
    }

    #[tokio::test]
@@ -4441,6 +4795,39 @@ mod tests {
        assert_eq!(backup_err, Error::PreconditionFailed);
    }

+    #[tokio::test]
+    async fn remove_bucket_usage_rejects_movement_epoch_flip_between_read_and_commit() {
+        let ctx = Arc::new(InstanceContext::new());
+        let publication_store = build_publication_store(ctx.clone());
+        let snapshot = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
+        let store = Arc::new(UsageCasStore {
+            state: Mutex::new(UsageCasState {
+                object: Some((serde_json::to_vec(&snapshot).expect("usage snapshot should encode"), 1)),
+                advance_publication_epoch_after_get: Some((UsageObjectSlot::Primary, ctx)),
+                ..Default::default()
+            }),
+        });
+        let expected_epoch = publication_store
+            .scanner_data_usage_publication_epoch()
+            .await
+            .expect("idle publication store should admit the initial read");
+
+        let err = remove_bucket_usage_from_object_with_retries_and_publication(
+            store.as_ref(),
+            DATA_USAGE_OBJ_NAME_PATH.as_str(),
+            "bucket-a",
+            0,
+            None,
+            None,
+            Some((publication_store.as_ref(), Some(expected_epoch))),
+        )
+        .await
+        .expect_err("a movement epoch flip during the read must fence the commit");
+
+        assert!(err.to_string().contains("epoch changed"));
+        assert_eq!(store.state.lock().await.put_count, 0);
+    }
+
    #[tokio::test]
    async fn remove_bucket_usage_confirms_ambiguous_committed_final_attempt() {
        let initial = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
@@ -4565,6 +4952,55 @@ mod tests {
        );
    }

+    #[tokio::test]
+    #[serial]
+    async fn partial_usage_is_observational_not_authoritative_for_quota() {
+        clear_usage_memory_cache_for_test().await;
+
+        let mut partial = data_usage_info_for_test("bucket-a", 10, 100, SystemTime::now());
+        partial.usage_snapshot_complete = false;
+        partial.usage_snapshot_partial = true;
+        replace_bucket_usage_memory_from_info(&partial).await;
+
+        assert_eq!(get_bucket_usage_memory("bucket-a").await, None);
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn stale_quota_uses_complete_baseline_plus_positive_deltas() {
+        clear_usage_memory_cache_for_test().await;
+
+        let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::now());
+        replace_bucket_usage_memory_from_info(&baseline).await;
+        record_bucket_object_write_memory("bucket-a", None, 25).await;
+
+        assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(125));
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn negative_delta_waits_for_set_reconciliation() {
+        clear_usage_memory_cache_for_test().await;
+
+        let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::UNIX_EPOCH + Duration::from_secs(100));
+        replace_bucket_usage_memory_from_info(&baseline).await;
+        record_bucket_object_delete_memory("bucket-a", 25, true).await;
+
+        assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(100));
+
+        // Simulate a process restart: the request-path overlay is gone, but
+        // the persisted authoritative snapshot is still the pre-reconciliation
+        // baseline. Quota must remain conservative until a complete scanner
+        // result proves the delete.
+        clear_usage_memory_cache_for_test().await;
+        replace_bucket_usage_memory_from_info(&baseline).await;
+        assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(100));
+
+        let reconciled = data_usage_info_for_test("bucket-a", 0, 75, SystemTime::UNIX_EPOCH + Duration::from_secs(101));
+        replace_bucket_usage_memory_from_info(&reconciled).await;
+        assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(75));
+    }
+
    #[tokio::test]
    #[serial]
    async fn memory_overlay_counts_versioned_overwrite_as_new_version() {
@@ -858,6 +858,7 @@ const EVENT_DISK_LOCAL_DIRECT_IO_FALLBACK: &str = "disk_local_direct_io_fallback
 #[cfg(target_os = "linux")]
 const EVENT_DISK_LOCAL_URING_LATCH_OFF: &str = "disk_local_uring_latch_off";
 const EVENT_DISK_LOCAL_DELETE_FAILED: &str = "disk_local_delete_failed";
+const EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED: &str = "disk_local_delete_rollback_failed";
 const EVENT_DISK_LOCAL_CHECK_PARTS: &str = "disk_local_check_parts";
 const EVENT_DISK_LOCAL_ACCESS_FAILED: &str = "disk_local_access_failed";
 const EVENT_DISK_LOCAL_VOLUME_SETUP_FAILED: &str = "disk_local_volume_setup_failed";
@@ -6106,6 +6107,43 @@ impl LocalDisk {
        Ok((bytes, modtime))
    }

+    async fn write_missing_delete_marker(
+        &self,
+        volume: &str,
+        path: &str,
+        fi: FileInfo,
+        object_dir: &Path,
+        xl_path: &Path,
+        rollback_dir: Option<Uuid>,
+    ) -> Result<()> {
+        if let Some(rollback_dir) = rollback_dir {
+            let rollback_path = object_dir.join(rollback_dir.to_string());
+            fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?;
+            fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), [])
+                .await
+                .map_err(to_file_error)?;
+        }
+        if let Err(err) = self.write_metadata("", volume, path, fi).await {
+            if let Some(rollback_dir) = rollback_dir
+                && let Err(restore_err) = restore_delete_rollback(object_dir, xl_path, rollback_dir, &self.publication_root).await
+            {
+                warn!(
+                    event = EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
+                    result = "failed",
+                    volume,
+                    path,
+                    rollback_dir = %rollback_dir,
+                    error = ?restore_err,
+                    "Disk local delete rollback failed"
+                );
+            }
+            return Err(err);
+        }
+        Ok(())
+    }
+
    async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> {
        let volume_dir = self.io_get_bucket_path(volume)?;
        let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?;
@@ -6123,7 +6161,20 @@ impl LocalDisk {
            return restore_metadata_backup(object_dir, &xlpath, rollback_dir, &self.publication_root).await;
        }

-        let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await?;
+        let (data, _) = match self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await {
+            Ok(data) => data,
+            Err(DiskError::FileNotFound) => {
+                // `deleted` alone can be an explicit marker purge; only
+                // `mark_deleted` may create metadata that was not present.
+                let Some(delete_marker) = fis.iter().find(|fi| fi.deleted && fi.mark_deleted).cloned() else {
+                    return Err(DiskError::FileNotFound);
+                };
+                return self
+                    .write_missing_delete_marker(volume, path, delete_marker, object_dir, &xlpath, opts.old_data_dir)
+                    .await;
+            }
+            Err(err) => return Err(err),
+        };

        if data.is_empty() {
            return Err(DiskError::FileNotFound);
@@ -7949,10 +8000,15 @@ impl DiskAPI for LocalDisk {
            use std::io::Write as _;

            let file_path = self.io_get_object_path(volume, path)?;
-            let lock_path = file_path.with_extension("rustfs-cas.lock");
            let path = path.to_string();
            let sync_metadata = effective_durability(volume).syncs_commit_metadata();
            return Ok(tokio::task::spawn_blocking(move || {
+                // A persistent directory lock bounds metadata growth. Removing
+                // per-target lock files can split flock ownership across inodes.
+                let lock_path = file_path
+                    .parent()
+                    .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "conditional file has no parent"))?
+                    .join(".rustfs-cas.lock");
                let lock = std::fs::OpenOptions::new()
                    .create(true)
                    .truncate(false)
@@ -8017,7 +8073,25 @@ impl DiskAPI for LocalDisk {
            .map_err(DiskError::from)??);
        }

-        #[cfg(not(unix))]
+        #[cfg(windows)]
+        {
+            let file_path = self.io_get_object_path(volume, path)?;
+            let sync_metadata = effective_durability(volume).syncs_commit_metadata();
+            let publication_root = self.publication_root.clone();
+            return Ok(tokio::task::spawn_blocking(move || {
+                os::compare_and_update_control_file(
+                    &file_path,
+                    expected.as_deref(),
+                    replacement.as_deref(),
+                    sync_metadata,
+                    &publication_root,
+                )
+            })
+            .await
+            .map_err(DiskError::from)??);
+        }
+
+        #[cfg(not(any(unix, windows)))]
        {
            let _ = (volume, path, expected, replacement);
            Err(DiskError::MethodNotAllowed)
@@ -8889,6 +8963,7 @@ impl DiskAPI for LocalDisk {
        )
        .await?;

+        out.close().await?;
        Ok(())
    }

@@ -10422,29 +10497,9 @@ impl DiskAPI for LocalDisk {
                }

                if fi.deleted && force_del_marker {
-                    if let Some(rollback_dir) = rollback_dir {
-                        let rollback_path = file_path.join(rollback_dir.to_string());
-                        fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?;
-                        fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), [])
-                            .await
-                            .map_err(to_file_error)?;
-                    }
-                    if let Err(err) = self.write_metadata("", volume, path, fi).await {
-                        if let Some(rollback_dir) = rollback_dir
-                            && let Err(restore_err) =
-                                restore_delete_rollback(file_path.as_path(), &xl_path, rollback_dir, &self.publication_root).await
-                        {
-                            warn!(
-                                volume,
-                                path,
-                                rollback_dir = %rollback_dir,
-                                error = ?restore_err,
-                                "failed to restore metadata after delete marker commit error"
-                            );
-                        }
-                        return Err(err);
-                    }
-                    return Ok(());
+                    return self
+                        .write_missing_delete_marker(volume, path, fi, file_path.as_path(), &xl_path, rollback_dir)
+                        .await;
                }

                return if fi.version_id.is_some() {
@@ -21792,9 +21847,9 @@ mod test {
        assert!(matches!(results[1].as_ref().unwrap_err(), DiskError::Io(_)));
    }

-    #[cfg(unix)]
+    #[cfg(any(unix, windows))]
    #[tokio::test]
-    async fn conditional_file_update_never_deletes_a_new_owner() {
+    async fn windows_and_unix_conditional_file_update_never_deletes_a_new_owner() {
        use tempfile::tempdir;

        let dir = tempdir().expect("temp dir should be created");
@@ -21825,8 +21880,18 @@ mod test {
            disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
                .await
                .expect("new owner marker should remain"),
-            owner_b
+            owner_b.clone()
        );
+        assert_eq!(
+            disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, Some(owner_b), None)
+                .await
+                .expect("current owner should remove marker"),
+            ConditionalFileUpdate::Updated
+        );
+        assert!(matches!(
+            disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH).await,
+            Err(DiskError::FileNotFound)
+        ));
    }

    #[cfg(unix)]
@@ -21841,7 +21906,10 @@ mod test {
        let marker_path = disk
            .get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
            .expect("marker path should resolve");
-        let lock_path = marker_path.with_extension("rustfs-cas.lock");
+        let lock_path = marker_path
+            .parent()
+            .expect("marker path should have a parent")
+            .join(".rustfs-cas.lock");
        let lock = std::fs::OpenOptions::new()
            .create(true)
            .truncate(false)
@@ -21862,6 +21930,40 @@ mod test {
        assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock));
    }

+    #[cfg(windows)]
+    #[tokio::test]
+    async fn windows_conditional_file_update_returns_would_block_when_marker_lock_is_contended() {
+        let dir = tempfile::tempdir().expect("temp dir should be created");
+        let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
+        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
+        ensure_test_volume(&disk, RUSTFS_META_BUCKET).await;
+        let marker_path = disk
+            .get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH)
+            .expect("marker path should resolve");
+        let lock_path = marker_path
+            .parent()
+            .expect("marker path should have a parent")
+            .join(".rustfs-cas.lock");
+        let lock = std::fs::OpenOptions::new()
+            .create(true)
+            .truncate(false)
+            .read(true)
+            .write(true)
+            .open(lock_path)
+            .expect("marker lock should open");
+        lock.try_lock().expect("marker lock should be held");
+
+        let err = tokio::time::timeout(
+            Duration::from_secs(1),
+            disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, None, Some(Bytes::from_static(b"owner"))),
+        )
+        .await
+        .expect("contended conditional update must not block")
+        .expect_err("contended conditional update must retry");
+
+        assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock));
+    }
+
    #[cfg(target_os = "linux")]
    #[tokio::test]
    async fn replacement_io_paths_stay_under_the_mount_lease() {
@@ -44,6 +44,8 @@ pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
 const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_DISK: &str = "disk";
 const EVENT_DISK_PART_ERR_UNCLASSIFIED: &str = "disk_part_err_unclassified";
+const ENV_BATCH_READ_VERSION_SERVER_PARALLELISM: &str = "RUSTFS_BATCH_READ_VERSION_SERVER_PARALLELISM";
+const BATCH_READ_VERSION_SERVER_PARALLELISM: usize = 4;

 pub fn part_transaction_path(part_path: &str) -> String {
    match part_path.rsplit_once('/') {
@@ -62,6 +64,7 @@ use bytes::Bytes;
 use endpoint::Endpoint;
 use error::DiskError;
 use error::{Error, Result};
+use futures::stream::{self, StreamExt};
 use local::LocalDisk;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use rustfs_madmin::info_commands::DiskMetrics;
@@ -417,6 +420,14 @@ impl DiskAPI for Disk {
        }
    }

+    #[tracing::instrument(level = "trace", skip_all)]
+    async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
+        match self {
+            Disk::Local(local_disk) => local_disk.batch_read_version(req).await,
+            Disk::Remote(remote_disk) => remote_disk.batch_read_version(req).await,
+        }
+    }
+
    #[tracing::instrument(level = "trace", skip_all)]
    async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
        match self {
@@ -1028,36 +1039,47 @@ where
    D: DiskAPI + ?Sized,
 {
    validate_batch_read_version_item_count(req.items.len())?;
+    let parallelism = batch_read_version_server_parallelism();

-    let mut responses = Vec::with_capacity(req.items.len());
-    for (index, item) in req.items.iter().enumerate() {
-        let response = match disk
-            .read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
-            .await
-        {
-            Ok(file_info) => BatchReadVersionResp {
-                index,
-                path: item.path.clone(),
-                version_id: item.version_id.clone(),
-                success: true,
-                file_info,
-                error: String::new(),
-            },
-            Err(err) => BatchReadVersionResp {
-                index,
-                path: item.path.clone(),
-                version_id: item.version_id.clone(),
-                success: false,
-                file_info: FileInfo::default(),
-                error: err.to_string(),
-            },
-        };
-        responses.push(response);
-    }
+    let mut responses = stream::iter(req.items.into_iter().enumerate())
+        .map(|(index, item)| async move {
+            match disk
+                .read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
+                .await
+            {
+                Ok(file_info) => BatchReadVersionResp {
+                    index,
+                    path: item.path,
+                    version_id: item.version_id,
+                    success: true,
+                    file_info,
+                    error: String::new(),
+                    error_code: 0,
+                },
+                Err(err) => BatchReadVersionResp {
+                    index,
+                    path: item.path,
+                    version_id: item.version_id,
+                    success: false,
+                    file_info: FileInfo::default(),
+                    error: err.to_string(),
+                    error_code: err.to_u32(),
+                },
+            }
+        })
+        .buffer_unordered(parallelism)
+        .collect::<Vec<_>>()
+        .await;
+    responses.sort_unstable_by_key(|response| response.index);

    Ok(responses)
 }

+fn batch_read_version_server_parallelism() -> usize {
+    rustfs_utils::get_env_usize(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, BATCH_READ_VERSION_SERVER_PARALLELISM)
+        .clamp(1, BATCH_READ_VERSION_MAX_ITEMS)
+}
+
 #[derive(Debug, Default, Serialize, Deserialize)]
 pub struct CheckPartsResp {
    pub results: Vec<usize>,
@@ -1322,6 +1344,8 @@ pub struct BatchReadVersionResp {
    pub success: bool,
    pub file_info: FileInfo,
    pub error: String,
+    #[serde(default)]
+    pub error_code: u32,
 }

 pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> {
@@ -1417,6 +1441,26 @@ mod tests {
        assert!(!partial_valid_location.valid());
    }

+    #[test]
+    fn batch_read_version_server_parallelism_defaults_to_conservative_four() {
+        temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, None::<&str>, || {
+            assert_eq!(batch_read_version_server_parallelism(), 4);
+        });
+    }
+
+    #[test]
+    fn batch_read_version_server_parallelism_honors_env_with_bounds() {
+        temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("8"), || {
+            assert_eq!(batch_read_version_server_parallelism(), 8);
+        });
+        temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("0"), || {
+            assert_eq!(batch_read_version_server_parallelism(), 1);
+        });
+        temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("9999"), || {
+            assert_eq!(batch_read_version_server_parallelism(), BATCH_READ_VERSION_MAX_ITEMS);
+        });
+    }
+
    /// Test FileInfoVersions find_version_index
    #[test]
    fn test_file_info_versions_find_version_index() {
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+#[cfg(windows)]
+use crate::disk::ConditionalFileUpdate;
 use crate::disk::error::DiskError;
 use crate::disk::error::Result;
 use crate::disk::error_conv::to_file_error;
@@ -3458,6 +3460,89 @@ fn read_windows_relative_file(file_path: &Path, parent_guard: &ExistingBaseDirec
    Ok(Some(data))
 }

+#[cfg(windows)]
+pub(crate) fn compare_and_update_control_file(
+    file_path: &Path,
+    expected: Option<&[u8]>,
+    replacement: Option<&[u8]>,
+    sync_metadata: bool,
+    publication_root: &PublicationRoot,
+) -> io::Result<ConditionalFileUpdate> {
+    use windows_sys::{
+        Wdk::Storage::FileSystem::{
+            FILE_NON_DIRECTORY_FILE, FILE_OPEN, FILE_OPEN_IF, FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT,
+        },
+        Win32::Storage::FileSystem::{
+            DELETE, FILE_ATTRIBUTE_NORMAL, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_WRITE_DATA, SYNCHRONIZE,
+        },
+    };
+
+    let parent = file_path
+        .parent()
+        .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file has no parent"))?;
+    let parent_guard = lock_windows_directory_tree(parent, Some(parent), publication_root)?;
+    let lock = open_windows_relative(
+        parent_guard.last_handle()?,
+        std::ffi::OsStr::new(".rustfs-cas.lock"),
+        SYNCHRONIZE | FILE_READ_ATTRIBUTES | FILE_WRITE_DATA,
+        FILE_SHARE_READ | FILE_SHARE_WRITE,
+        FILE_OPEN_IF,
+        FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT,
+        FILE_ATTRIBUTE_NORMAL,
+        true,
+    )?;
+    validate_windows_owned_file(&lock)?;
+    match lock.as_file().try_lock() {
+        Ok(()) => {}
+        Err(std::fs::TryLockError::WouldBlock) => return Err(io::Error::from(io::ErrorKind::WouldBlock)),
+        Err(std::fs::TryLockError::Error(err)) => return Err(err),
+    }
+
+    let current = read_windows_relative_file(file_path, &parent_guard)?;
+    let matches = match (&current, expected) {
+        (None, None) => true,
+        (Some(current), Some(expected)) => current.as_slice() == expected,
+        _ => false,
+    };
+    if !matches {
+        return Ok(match current {
+            None => ConditionalFileUpdate::Missing,
+            Some(_) => ConditionalFileUpdate::Mismatch,
+        });
+    }
+
+    match replacement {
+        Some(replacement) => RenameDestinationPathGuard {
+            directory: parent.to_path_buf(),
+            _directory_guard: parent_guard,
+        }
+        .write_file_for_path_access(file_path, replacement, sync_metadata, sync_metadata)?,
+        None => {
+            let file_name = file_path
+                .file_name()
+                .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file must have a name"))?;
+            let file = open_windows_relative(
+                parent_guard.last_handle()?,
+                file_name,
+                DELETE | SYNCHRONIZE | FILE_READ_ATTRIBUTES,
+                FILE_SHARE_READ,
+                FILE_OPEN,
+                FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT,
+                0,
+                true,
+            )?;
+            validate_windows_owned_file(&file)?;
+            set_windows_file_delete_on_close(&file, true)?;
+            drop(file);
+            if sync_metadata {
+                fsync_dir_std(parent)?;
+            }
+        }
+    }
+
+    Ok(ConditionalFileUpdate::Updated)
+}
+
 #[cfg(windows)]
 fn open_windows_directory_component(
    parent: &WindowsDirectoryHandle,
@@ -784,6 +784,24 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle(
 ///
 /// # Returns
 /// A Result containing the BitrotWriterWrapper or an error
+/// Size hint handed to `DiskAPI::create_file` for a bitrot-wrapped shard.
+///
+/// A known length is grown by one checksum per shard so the on-disk file size
+/// matches what the bitrot writer emits. A negative length is the
+/// unknown-size sentinel (`HashReader::SIZE_PRESERVE_LAYER`, used by SSE and
+/// compression) and must be preserved: `RemoteDisk::create_file` forwards it
+/// in the `put_file_stream` query, and the receiver only treats `size > 0` as
+/// a fixed body length when locating the authenticated trailer. Clamping it
+/// to `0` would claim an empty body and misframe the stream. `0` stays `0`
+/// because a genuinely empty object still means an empty body.
+fn bitrot_create_file_size(length: i64, shard_size: usize, checksum_algo: &HashAlgorithm) -> i64 {
+    if length <= 0 {
+        return length;
+    }
+    let length = length as usize;
+    (length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
+}
+
 pub async fn create_bitrot_writer(
    is_inline_buffer: bool,
    disk: Option<&DiskStore>,
@@ -796,12 +814,7 @@ pub async fn create_bitrot_writer(
    let writer = if is_inline_buffer {
        CustomWriter::new_inline_buffer()
    } else if let Some(disk) = disk {
-        let length = if length > 0 {
-            let length = length as usize;
-            (length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
-        } else {
-            0
-        };
+        let length = bitrot_create_file_size(length, shard_size, &checksum_algo);

        let file = disk.create_file("", volume, path, length).await?;
        #[cfg(feature = "hotpath")]
@@ -820,6 +833,25 @@ mod tests {
    use rustfs_rio::ChunkReader;
    use std::collections::VecDeque;

+    #[test]
+    fn bitrot_create_file_size_grows_known_length_by_checksums() {
+        // 10 bytes over 4-byte shards = 3 shards, each followed by a 32-byte hash.
+        assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::HighwayHash256), 10 + 3 * 32);
+        assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::None), 10);
+    }
+
+    #[test]
+    fn bitrot_create_file_size_keeps_empty_and_unknown_distinct() {
+        assert_eq!(bitrot_create_file_size(0, 4, &HashAlgorithm::HighwayHash256), 0);
+        // SSE/compression streams advertise SIZE_PRESERVE_LAYER (-1); the remote
+        // put_file_stream receiver relies on a non-positive size to parse the auth
+        // trailer from the stream tail, so the sentinel must survive untouched.
+        assert_eq!(
+            bitrot_create_file_size(rustfs_rio::HashReader::SIZE_PRESERVE_LAYER, 4, &HashAlgorithm::HighwayHash256),
+            rustfs_rio::HashReader::SIZE_PRESERVE_LAYER
+        );
+    }
+
    struct TestChunkReader {
        chunks: VecDeque<Bytes>,
    }
@@ -81,6 +81,14 @@ pub fn shutdown_background_monitors() {
    cluster::rpc::shutdown_background_monitors();
 }

+/// Publish that the process is ready to serve user-object GET traffic.
+///
+/// Experimental metadata coalescing is allowed to run only after this point so
+/// startup and internal metadata reads keep the original per-disk path.
+pub fn mark_get_metadata_read_version_coalescing_service_ready() {
+    runtime::global::mark_get_metadata_read_version_coalescing_service_ready();
+}
+
 #[cfg(test)]
 mod rio_tests {
    #[test]
@@ -24,7 +24,7 @@ use crate::storage_api_contracts::{
 pub struct NamespaceLockFence {
    signals: Arc<Vec<Arc<rustfs_lock::distributed_lock::LockLostSignal>>>,
    #[cfg(test)]
-    forced_lost: Arc<std::sync::atomic::AtomicBool>,
+    forced_lost: Arc<Vec<Arc<std::sync::atomic::AtomicBool>>>,
 }

 impl Debug for NamespaceLockFence {
@@ -40,13 +40,17 @@ impl NamespaceLockFence {
        Self {
            signals: Arc::default(),
            #[cfg(test)]
-            forced_lost: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+            forced_lost: Arc::new(vec![Arc::new(std::sync::atomic::AtomicBool::new(false))]),
        }
    }

    pub(crate) fn is_lock_lost(&self) -> bool {
        #[cfg(test)]
-        if self.forced_lost.load(std::sync::atomic::Ordering::Acquire) {
+        if self
+            .forced_lost
+            .iter()
+            .any(|lost| lost.load(std::sync::atomic::Ordering::Acquire))
+        {
            return true;
        }
        self.signals.iter().any(|signal| signal.is_lost())
@@ -57,27 +61,26 @@ impl NamespaceLockFence {
    }

    fn extend(&mut self, other: &Self) {
-        if Arc::ptr_eq(&self.signals, &other.signals) {
-            return;
+        if !Arc::ptr_eq(&self.signals, &other.signals) {
+            Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
        }
-        Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
        #[cfg(test)]
-        if other.forced_lost.load(std::sync::atomic::Ordering::Acquire) {
-            self.forced_lost.store(true, std::sync::atomic::Ordering::Release);
+        if !Arc::ptr_eq(&self.forced_lost, &other.forced_lost) {
+            Arc::make_mut(&mut self.forced_lost).extend(other.forced_lost.iter().cloned());
        }
    }

    #[cfg(test)]
    pub(crate) fn lost_for_test() -> Self {
        let fence = Self::new();
-        fence.forced_lost.store(true, std::sync::atomic::Ordering::Release);
+        fence.forced_lost[0].store(true, std::sync::atomic::Ordering::Release);
        fence
    }

    #[cfg(test)]
    pub(crate) fn loss_handle_for_test() -> (Self, Arc<std::sync::atomic::AtomicBool>) {
        let fence = Self::new();
-        (fence.clone(), Arc::clone(&fence.forced_lost))
+        (fence.clone(), Arc::clone(&fence.forced_lost[0]))
    }
 }

@@ -411,6 +414,13 @@ impl ObjectOptions {
        self.namespace_lock_fence.get_or_insert_with(NamespaceLockFence::new);
    }

+    #[cfg(test)]
+    pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) {
+        self.namespace_lock_fence
+            .get_or_insert_with(NamespaceLockFence::new)
+            .extend(fence);
+    }
+
    pub(crate) fn ensure_lifecycle_delete_all_journal(&mut self) {
        self.lifecycle_delete_all_journal
            .get_or_insert_with(|| Arc::new(parking_lot::Mutex::new(LifecycleDeleteAllJournalState::default())));
@@ -689,6 +699,9 @@ impl ObjectInfo {
    }

    pub fn get_actual_size(&self) -> std::io::Result<i64> {
+        if self.actual_size < -1 || (self.actual_size == -1 && !self.is_compressed()) {
+            return Err(std::io::Error::other("invalid negative actual size"));
+        }
        if self.actual_size > 0 {
            return Ok(self.actual_size);
        }
@@ -700,10 +713,25 @@ impl ObjectInfo {
                let size = size_str.parse::<i64>().map_err(|e| std::io::Error::other(e.to_string()))?;
                return Ok(size);
            }
-            let mut actual_size = 0;
-            self.parts.iter().for_each(|part| {
-                actual_size += part.actual_size;
-            });
+            if self.actual_size == -1 && self.parts.is_empty() {
+                return Ok(-1);
+            }
+            let mut actual_size = 0_i64;
+            let mut unknown = false;
+            for part in self.parts.iter() {
+                match part.actual_size {
+                    -1 => unknown = true,
+                    size if size >= 0 => {
+                        actual_size = actual_size
+                            .checked_add(size)
+                            .ok_or_else(|| std::io::Error::other("compressed actual size overflow"))?;
+                    }
+                    _ => return Err(std::io::Error::other("invalid negative compressed part size")),
+                }
+            }
+            if unknown {
+                return Ok(-1);
+            }
            if actual_size == 0 && actual_size != self.size {
                return Err(std::io::Error::other(format!("invalid decompressed size {} {}", actual_size, self.size)));
            }
@@ -718,6 +746,18 @@ impl ObjectInfo {
        Ok(self.size)
    }

+    /// Returns a non-negative size for client and replication boundaries.
+    ///
+    /// Compressed legacy metadata can retain the internal `-1` unknown-size
+    /// sentinel. Those boundaries cannot emit a negative length, so they use
+    /// the persisted physical size while quota accounting keeps the sentinel
+    /// distinction in [`crate::data_usage::quota_object_size`].
+    pub fn get_actual_size_or_physical(&self) -> i64 {
+        self.get_actual_size()
+            .map(|size| if size >= 0 { size } else { self.size.max(0) })
+            .unwrap_or_else(|_| self.size.max(0))
+    }
+
    pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
        let mut version_id = fi.version_id;

@@ -25,7 +25,10 @@ use lazy_static::lazy_static;
 use rustfs_lock::client::LockClient;
 use std::{
    collections::HashMap,
-    sync::{Arc, OnceLock},
+    sync::{
+        Arc, OnceLock,
+        atomic::{AtomicBool, Ordering},
+    },
    time::SystemTime,
 };
 use tokio::sync::{OnceCell, RwLock};
@@ -37,6 +40,16 @@ pub const DISK_MIN_INODES: u64 = 1000;
 pub const DISK_FILL_FRACTION: f64 = 0.99;
 pub const DISK_RESERVE_FRACTION: f64 = 0.15;

+static GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY: AtomicBool = AtomicBool::new(false);
+
+pub(crate) fn mark_get_metadata_read_version_coalescing_service_ready() {
+    GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.store(true, Ordering::Release);
+}
+
+pub(crate) fn get_metadata_read_version_coalescing_service_ready() -> bool {
+    GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.load(Ordering::Acquire)
+}
+
 // Global singletons for backward compatibility with MinIO port.
 // These should be migrated to AppContext over time.
 // See issue #730 for migration plan.
@@ -52,11 +52,18 @@ use crate::services::tier::tier::TierConfigMgr;
 use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
 use s3s::region::Region;
 use std::collections::{HashMap, HashSet};
-use std::sync::{Arc, OnceLock};
+use std::sync::{
+    Arc, OnceLock,
+    atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
+};
 use tokio::sync::{OnceCell, RwLock};
 use tokio_util::sync::CancellationToken;
 use uuid::Uuid;

+const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
+const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1;
+const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
+
 /// Runtime state owned by a single `ECStore` instance.
 ///
 /// This is intentionally minimal in the first migration slice; subsequent
@@ -160,6 +167,22 @@ pub struct InstanceContext {
    /// workers (scanner/heal/tier/lifecycle) without touching another instance.
    /// Replaces the process-global cancel-token static.
    background_cancel_token: OnceLock<CancellationToken>,
+    /// Serializes data-movement transitions with scanner publication commits.
+    /// Readers are held across one publication commit; movement transitions
+    /// take the writer at their durable state commit boundary.
+    data_movement_operation_gate: Arc<RwLock<()>>,
+    /// Monotonic admission epoch paired with the operation gate. A
+    /// publication admitted before a movement transition must never be
+    /// mistaken for one admitted after the transition.
+    data_movement_operation_epoch: AtomicU64,
+    /// Once the admission epoch reaches its reserved terminal value, no new
+    /// publication may be admitted. Keeping this state separate from the
+    /// saturating counter prevents an unchanged `u64::MAX` value from being
+    /// mistaken for a fresh epoch after overflow.
+    data_movement_operation_epoch_exhausted: AtomicBool,
+    /// Last storage-owned movement snapshot observed under the operation
+    /// gate. SetDisks cache writers fail closed until ECStore refreshes it.
+    scanner_publication_state: AtomicU8,
    /// Resolves object-encryption material at the application boundary.
    object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
    tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
@@ -200,6 +223,10 @@ impl InstanceContext {
            local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
            bucket_metadata_sys: std::sync::Mutex::new(None),
            background_cancel_token: OnceLock::new(),
+            data_movement_operation_gate: Arc::new(RwLock::new(())),
+            data_movement_operation_epoch: AtomicU64::new(0),
+            data_movement_operation_epoch_exhausted: AtomicBool::new(false),
+            scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
            object_encryption_resolver: OnceLock::new(),
            tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
            transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -218,6 +245,56 @@ impl InstanceContext {
        self.lock_manager.clone()
    }

+    pub(crate) fn data_movement_operation_gate(&self) -> Arc<RwLock<()>> {
+        Arc::clone(&self.data_movement_operation_gate)
+    }
+
+    pub(crate) fn data_movement_operation_epoch(&self) -> u64 {
+        self.data_movement_operation_epoch.load(Ordering::Acquire)
+    }
+
+    pub(crate) fn data_movement_operation_epoch_exhausted(&self) -> bool {
+        self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire)
+    }
+
+    pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
+        !self.data_movement_operation_epoch_exhausted()
+            && self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
+    }
+
+    pub(crate) fn set_scanner_publication_state(&self, blocked: bool) {
+        self.scanner_publication_state.store(
+            if blocked {
+                SCANNER_PUBLICATION_STATE_BLOCKED
+            } else {
+                SCANNER_PUBLICATION_STATE_ALLOWED
+            },
+            Ordering::Release,
+        );
+    }
+
+    pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
+        self.scanner_publication_state
+            .store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
+        let _ = self
+            .data_movement_operation_epoch
+            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
+        let result = self.data_movement_operation_epoch.load(Ordering::Acquire);
+        if result == u64::MAX {
+            self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
+        }
+        result
+    }
+
+    #[cfg(test)]
+    pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) {
+        self.data_movement_operation_epoch.store(epoch, Ordering::Release);
+        self.data_movement_operation_epoch_exhausted
+            .store(epoch == u64::MAX, Ordering::Release);
+        self.scanner_publication_state
+            .store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
+    }
+
    /// Install the application-owned object-encryption resolver once.
    pub fn set_object_encryption_resolver(
        &self,
@@ -220,6 +220,14 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
            last_usage_save_unix_secs: metrics.usage_freshness.last_usage_save_unix_secs,
            last_usage_save_result: metrics.usage_freshness.last_usage_save_result,
            last_usage_save_result_code: metrics.usage_freshness.last_usage_save_result_code,
+            last_durable_success_unix_secs: metrics.usage_freshness.last_durable_success_unix_secs,
+            last_publication_unix_secs: metrics.usage_freshness.last_publication_unix_secs,
+            last_publication_state: metrics.usage_freshness.last_publication_state,
+            last_publication_reason: metrics.usage_freshness.last_publication_reason,
+            deferred_pending: metrics.usage_freshness.deferred_pending,
+            deferred_total: metrics.usage_freshness.deferred_total,
+            last_deferred_unix_secs: metrics.usage_freshness.last_deferred_unix_secs,
+            last_deferred_reason: metrics.usage_freshness.last_deferred_reason,
        },
        maintenance_control: MadminScannerMaintenanceControlSnapshot {
            primary_control: metrics.maintenance_control.primary_control,
@@ -256,6 +264,10 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
        cycle_max_duration_seconds: metrics.cycle_max_duration_seconds,
        cycle_max_objects: metrics.cycle_max_objects,
        cycle_max_directories: metrics.cycle_max_directories,
+        cycle_timeout_total: metrics.cycle_timeout_total,
+        cycle_recovery_required_total: metrics.cycle_recovery_required_total,
+        cycle_last_progress_age: metrics.cycle_last_progress_age,
+        leader_lease_without_progress: metrics.leader_lease_without_progress,
        bitrot_cycle_enabled: metrics.bitrot_cycle_enabled,
        bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
        scan_checkpoint: metrics.scan_checkpoint.map(|checkpoint| MadminScannerCheckpointReport {
@@ -611,6 +623,10 @@ mod test {
            current_started: chrono_to_jiff_timestamp(current_started),
            last_cycle_partial_source: "usage".to_string(),
            last_cycle_partial_source_code: 1,
+            cycle_timeout_total: 3,
+            cycle_recovery_required_total: 2,
+            cycle_last_progress_age: 17,
+            leader_lease_without_progress: true,
            partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
                source: "usage".to_string(),
                cycles: 2,
@@ -622,6 +638,10 @@ mod test {
        assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
        assert_eq!(scanner.last_cycle_partial_source, "usage");
        assert_eq!(scanner.last_cycle_partial_source_code, 1);
+        assert_eq!(scanner.cycle_timeout_total, 3);
+        assert_eq!(scanner.cycle_recovery_required_total, 2);
+        assert_eq!(scanner.cycle_last_progress_age, 17);
+        assert!(scanner.leader_lease_without_progress);
        let usage = scanner
            .partial_cycles_by_source
            .iter()
@@ -804,6 +824,14 @@ mod test {
                last_usage_save_unix_secs: 12,
                last_usage_save_result: "success".to_string(),
                last_usage_save_result_code: 1,
+                last_durable_success_unix_secs: 13,
+                last_publication_unix_secs: 14,
+                last_publication_state: "published".to_string(),
+                last_publication_reason: "complete".to_string(),
+                deferred_pending: true,
+                deferred_total: 15,
+                last_deferred_unix_secs: 16,
+                last_deferred_reason: "data_movement".to_string(),
            },
            ..Default::default()
        });
@@ -816,6 +844,14 @@ mod test {
        assert_eq!(scanner.usage_freshness.last_usage_save_unix_secs, 12);
        assert_eq!(scanner.usage_freshness.last_usage_save_result, "success");
        assert_eq!(scanner.usage_freshness.last_usage_save_result_code, 1);
+        assert_eq!(scanner.usage_freshness.last_durable_success_unix_secs, 13);
+        assert_eq!(scanner.usage_freshness.last_publication_unix_secs, 14);
+        assert_eq!(scanner.usage_freshness.last_publication_state, "published");
+        assert_eq!(scanner.usage_freshness.last_publication_reason, "complete");
+        assert!(scanner.usage_freshness.deferred_pending);
+        assert_eq!(scanner.usage_freshness.deferred_total, 15);
+        assert_eq!(scanner.usage_freshness.last_deferred_unix_secs, 16);
+        assert_eq!(scanner.usage_freshness.last_deferred_reason, "data_movement");
    }

    #[test]
--- a/Show More
+++ b/Show More