feat(ci): publish functional reports and keep workflows non-blocking

The transition_matrix_tests use #[serial_test::serial] which has no
effect under nextest (each test runs in a separate process). When running
alongside thousands of other ecstore tests, the shared metadata cache
generation counter can race, causing intermittent 'metadata read should
publish the generation under test' panics.

Add both tests to the ecstore-serial-flaky test group in both default
and ci nextest profiles so they run single-threaded.

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
 sha256-darwin=d6aa36cfaae2c4d8590482c7e47138c5965b335b34a75f50d11ffc3366e9021e
-sha256-linux=96db8060fce98addda4f69092d297ca236bec4892d820617a26a261eedac61b0
+sha256-linux=e3eb4ab7fc72224abf58c546ac0706d6605d3bd26bac7d8ce338829fd3daecc2

.config/nextest.toml

@@ -100,6 +100,16 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'
 
+# Serialize the transition matrix tests. They build a 4-disk hermetic erasure
+# set, populate the get_object_metadata_cache, and assert generation lifecycle
+# semantics. serial_test's #[serial] has no effect across nextest's process
+# boundary, so concurrent execution races the shared metadata-cache generation
+# counter and causes spurious "metadata read should publish the generation"
+# panics. Preventive serialization, no retries.
+[[profile.default.overrides]]
+filter = 'package(rustfs-ecstore) & test(set_disk::transition_matrix_tests::)'
+test-group = 'ecstore-serial-flaky'
+
 # The durable ILM decommission regressions build isolated multi-pool stores and
 # deliberately take source or target disks offline while checking fencing.
 [[profile.default.overrides]]
@@ -232,6 +242,12 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'
 
+# Serialize the transition matrix tests under the ci profile too (see the
+# matching default-profile override near the top). No retries.
+[[profile.ci.overrides]]
+filter = 'package(rustfs-ecstore) & test(set_disk::transition_matrix_tests::)'
+test-group = 'ecstore-serial-flaky'
+
 [[profile.ci.overrides]]
 filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
 test-group = 'ecstore-serial-flaky'

.github/actionlint.yaml

@@ -6,3 +6,4 @@ self-hosted-runner:
     - sm-standard-4
     - dind-sm-standard-2
     - smoke-testing
+    - pf-testing

.github/workflows/build.yml

@@ -1065,7 +1065,7 @@ jobs:
           while IFS= read -r preview_tag; do
             [[ -n "$preview_tag" ]] || continue
             echo "🧹 Deleting preview release $preview_tag (tag kept)"
-            gh release delete "$preview_tag" --repo "${GITHUB_REPOSITORY}" --yes
+            gh release delete "$preview_tag" --yes
             DELETED=$((DELETED + 1))
           done < <(
             jq -r --arg tag "$TAG" '

.github/workflows/nix.yml

@@ -27,6 +27,7 @@ on:
     paths:
       - 'flake.nix'
       - 'flake.lock'
+      - 'nix/**'
       - 'Cargo.toml'
       - 'Cargo.lock'
       - '.github/workflows/nix.yml'
@@ -36,6 +37,7 @@ on:
     paths:
       - 'flake.nix'
       - 'flake.lock'
+      - 'nix/**'
       - 'Cargo.toml'
       - 'Cargo.lock'
       - '.github/workflows/nix.yml'

.github/workflows/package.yml

@@ -224,7 +224,7 @@ jobs:
           ZIP_FILE=$(find ./binary-artifact -name "*.zip" -type f | head -1)
           if [[ -z "$ZIP_FILE" ]]; then
             echo "❌ No binary artifact found"
-            ls -la ./binary-artifact/ || true
+            find ./binary-artifact -mindepth 1 -maxdepth 1 -print 2>/dev/null || true
             exit 1
           fi
 
@@ -239,7 +239,7 @@ jobs:
           fi
 
           chmod +x ./bin/rustfs
-          ls -lh ./bin/rustfs
+          stat --printf='%n %s bytes\n' ./bin/rustfs
           echo "✅ Binary extracted"
 
       - name: Build DEB package
@@ -336,7 +336,7 @@ jobs:
           fakeroot dpkg-deb --build "${PKG_DIR}"
 
           DEB_FILE="${PKG_DIR}.deb"
-          ls -lh "$DEB_FILE"
+          stat --printf='%n %s bytes\n' "$DEB_FILE"
           echo "deb_file=$DEB_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ DEB built: $DEB_FILE"
 
@@ -410,13 +410,14 @@ jobs:
             LICENSE=/usr/share/doc/rustfs/LICENSE \
             README.md=/usr/share/doc/rustfs/README.md
 
-          RPM_FILE=$(ls -1 rustfs-*.rpm 2>/dev/null | head -1)
+            RPM_FILE=$(find . -maxdepth 1 -type f -name 'rustfs-*.rpm' -print | head -1)
+            RPM_FILE="${RPM_FILE#./}"
           if [[ -z "$RPM_FILE" ]]; then
             echo "❌ RPM build failed"
             exit 1
           fi
 
-          ls -lh "$RPM_FILE"
+          stat --printf='%n %s bytes\n' "$RPM_FILE"
           echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ RPM built: $RPM_FILE"
 
@@ -552,11 +553,13 @@ jobs:
       - name: Print summary
         shell: bash
         run: |
-          echo "## 📦 Package Summary" >> "$GITHUB_STEP_SUMMARY"
-          echo "" >> "$GITHUB_STEP_SUMMARY"
-          echo "| Item | Value |" >> "$GITHUB_STEP_SUMMARY"
-          echo "|------|-------|" >> "$GITHUB_STEP_SUMMARY"
-          echo "| Version | \`${{ needs.resolve.outputs.version }}\` |" >> "$GITHUB_STEP_SUMMARY"
-          echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |" >> "$GITHUB_STEP_SUMMARY"
-          echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |" >> "$GITHUB_STEP_SUMMARY"
-          echo "| Package Status | ${{ needs.package.result }} |" >> "$GITHUB_STEP_SUMMARY"
+          {
+            echo "## 📦 Package Summary"
+            echo ""
+            echo "| Item | Value |"
+            echo "|------|-------|"
+            echo "| Version | \`${{ needs.resolve.outputs.version }}\` |"
+            echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |"
+            echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |"
+            echo "| Package Status | ${{ needs.package.result }} |"
+          } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/rustfs-heal-test.yml

@@ -23,12 +23,6 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  workflow_run:
-    # Run after the nightly build completes: the nightly deb is what the test
-    # installs. Heal also runs inside the functional suite; this standalone
-    # workflow enables manual single-suite runs as well.
-    workflows: ["Nightly GNU Build"]
-    types: [completed]
 
 permissions:
   contents: read
@@ -36,7 +30,7 @@ permissions:
 # Only one test at a time: both this and the pool-expansion workflow mutate
 # the same test environment, so they share one concurrency group.
 concurrency:
-  group: rustfs-pool-expansion-test
+  group: rustfs-shared-functional-tests
   cancel-in-progress: false
 
 defaults:
@@ -55,9 +49,9 @@ jobs:
   heal-test:
     runs-on: smoke-testing
     timeout-minutes: 480
-    # Run on manual dispatch, or when the nightly build completed successfully.
-    # Skipped when nightly failed.
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    # Manual-only standalone run. Nightly chain already runs heal in
+    # rustfs-pool-expand-test.yml to avoid duplicate heal executions.
+    if: ${{ github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout auto-testing scripts
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7

.github/workflows/rustfs-kms-test.yml

@@ -0,0 +1,347 @@
+name: RustFS KMS Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      rustfs_version:
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        required: false
+        default: '1.0.0-rc.4-preview.1'
+      package_url:
+        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
+        required: false
+        type: string
+  workflow_run:
+    # Strict shared-environment order: run after S3 compatibility test succeeds.
+    workflows: ["RustFS S3 Compatibility Test"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-shared-functional-tests
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  kms-test:
+    runs-on: smoke-testing
+    continue-on-error: true
+    timeout-minutes: 420
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          openssl version
+          docker --version || true
+
+      - name: Cleanup environment (before)
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
+            '
+          done
+
+      - name: Ensure docker (Vault container)
+        run: |
+          if ! command -v docker >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y docker.io
+          fi
+          sudo systemctl enable --now docker
+          docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1
+
+      - name: Run KMS suite
+        id: test
+        continue-on-error: true
+        env:
+          LOG_FILE: /tmp/rustfs-kms.log
+        run: |
+          set -euo pipefail
+          chmod +x auto-testing/rustfs-kms-test.sh
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          ARGS=(--all-topologies --backends "local,vault-kv2" -y --log-file "${LOG_FILE}")
+          if [ -n "${PACKAGE_URL}" ]; then
+            ARGS+=(--package-url "${PACKAGE_URL}")
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            ARGS+=(--version "${RUSTFS_VERSION}")
+          else
+            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
+          fi
+          ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-kms.log
+          REPORT_FILE: /tmp/rustfs-kms-report.md
+        run: |
+          set -euo pipefail
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          CASE_TABLE="/tmp/rustfs-kms-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          try:
+            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+              for raw in fh:
+                line = ansi.sub('', raw).strip()
+                m = start_re.match(line)
+                if m:
+                  case_id, name = m.group(1), m.group(2)
+                  if case_id not in index:
+                    index[case_id] = len(rows)
+                    rows.append([case_id, name, 'RUNNING'])
+                  continue
+                m = done_re.match(line)
+                if m:
+                  status, case_id = m.group(1), m.group(2)
+                  if case_id in index:
+                    rows[index[case_id]][2] = status
+                  else:
+                    rows.append([case_id, case_id, status])
+                    index[case_id] = len(rows) - 1
+          except FileNotFoundError:
+            rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+            counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+            out.write('## Case Summary\n\n')
+            out.write(f"- Total: {len(rows)}\\n")
+            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+            out.write('\\n')
+            out.write('| Case | Name | Status |\\n')
+            out.write('| --- | --- | --- |\\n')
+            for case_id, name, status in rows:
+              out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
+          {
+            echo "# RustFS KMS test report"
+            echo ""
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Package: ${PACKAGE_SOURCE}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo ""
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
+          } | tee "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: always()
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-kms-report.md
+          SUITE: kms
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+
+          cat > /tmp/rustfs-functional-index.html <<'EOF'
+          <!doctype html>
+          <html lang="en">
+          <head>
+            <meta charset="utf-8" />
+            <meta name="viewport" content="width=device-width, initial-scale=1" />
+            <title>RustFS Functional Test Reports</title>
+            <style>
+              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
+              * { box-sizing: border-box; }
+              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
+              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
+              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
+              h1 { margin: 0 0 8px; font-size: 26px; }
+              p { margin: 0 0 14px; color: var(--muted); }
+              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
+              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
+              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
+              ul { list-style: none; margin: 0; padding: 0; }
+              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
+              a { color: var(--accent); text-decoration: none; }
+              a:hover { text-decoration: underline; }
+            </style>
+          </head>
+          <body>
+            <div class="wrap">
+              <div class="card">
+                <h1>RustFS Functional Test Reports</h1>
+                <p>S3, KMS, Tier report tabs. Each tab lists reports by date.</p>
+                <div class="tabs" id="tabs"></div>
+                <ul id="list"></ul>
+              </div>
+            </div>
+            <script>
+              const suites = [
+                { key: 's3', label: 'S3 Compatibility' },
+                { key: 'kms', label: 'KMS' },
+                { key: 'tier', label: 'Tier' },
+              ];
+              const tabs = document.getElementById('tabs');
+              const list = document.getElementById('list');
+
+              async function loadSuite(suite) {
+                list.innerHTML = '<li>Loading...</li>';
+                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
+                try {
+                  const res = await fetch(api);
+                  if (!res.ok) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    return;
+                  }
+                  const data = await res.json();
+                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
+                  if (!files.length) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    return;
+                  }
+                  list.innerHTML = files.map(f => `<li><a href="${f.html_url}" target="_blank" rel="noreferrer">${f.name.replace('.md','')}</a></li>`).join('');
+                } catch (_e) {
+                  list.innerHTML = '<li>Failed to load reports.</li>';
+                }
+              }
+
+              function setActive(key) {
+                for (const btn of tabs.querySelectorAll('button')) {
+                  btn.classList.toggle('active', btn.dataset.key === key);
+                }
+                loadSuite(key);
+              }
+
+              for (const suite of suites) {
+                const btn = document.createElement('button');
+                btn.textContent = suite.label;
+                btn.dataset.key = suite.key;
+                btn.addEventListener('click', () => setActive(suite.key));
+                tabs.appendChild(btn);
+              }
+              setActive('s3');
+            </script>
+          </body>
+          </html>
+          EOF
+
+          INDEX_PATH="functional/index.html"
+          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
+          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${INDEX_SHA}" ]; then
+            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          fi
+
+      - name: Upload report and logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-kms-test-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-kms.log
+            /tmp/rustfs-kms-report.md
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        if: always()
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
+            '
+          done
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS KMS suite failed"
+          echo "See the uploaded report and log artifacts for details."

.github/workflows/rustfs-performance-test.yml

@@ -78,7 +78,7 @@ env:
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
   # Fixed benchmark result directory so later steps can read summary.md
   RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results
-  # Cross-repo token for writing to rustfs/backlog (set in repo settings)
+  # Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings)
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
 jobs:
@@ -149,38 +149,67 @@ jobs:
         run: |
           ./auto-testing/rustfs_performance_test.sh --step 6 -y
 
-      - name: Post results to backlog issue
+      - name: Collect RustFS version info
+        if: ${{ steps.benchmark.conclusion == 'success' }}
+        env:
+          VERSION_FILE: /tmp/rustfs-version.txt
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES}"
+          [ "${#NODES[@]}" -gt 0 ] || { echo "RUSTFS_NODES is empty"; exit 1; }
+          NODE="${NODES[0]}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          {
+            echo "Node: ${NODE}"
+            echo "Command: rustfs --version"
+            echo ""
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
+              "${SSH_USER}@${NODE}" 'rustfs --version'
+          } > "${VERSION_FILE}"
+
+      - name: Upload report to dashboard (reports/YYYY-MM-DD.md)
         if: ${{ steps.benchmark.conclusion == 'success' }}
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
           RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
+          VERSION_FILE: /tmp/rustfs-version.txt
         run: |
           set -euo pipefail
           if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping issue post"
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping report upload"
             exit 0
           fi
           SUMMARY="${RESULT_DIR}/summary.md"
           [ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
           DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="reports/${DATE}.md"
           {
-            echo "## RustFS nightly build performance testing report"
+            echo "# RustFS nightly build performance testing report"
             echo ""
-            echo "- **日期**: ${DATE}"
+            echo "- **Date**: ${DATE}"
             echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- **触发方式**: ${{ github.event_name }}"
+            echo "- **Trigger**: ${{ github.event_name }}"
+            echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
             echo ""
             cat "${SUMMARY}"
-          } > /tmp/rustfs-perf-issue-body.md
-          TITLE="RustFS nightly build performance testing report"
-          EXISTING="$(gh issue list --repo rustfs/backlog \
-            --search "in:title \"${TITLE}\"" --state all --limit 5 \
-            --json number --jq '.[0].number // empty')"
-          if [ -n "${EXISTING}" ]; then
-            gh issue comment "${EXISTING}" --repo rustfs/backlog --body-file /tmp/rustfs-perf-issue-body.md
-            echo "commented on existing issue #${EXISTING}"
+            echo ""
+            echo "## RustFS version"
+            echo '```text'
+            cat "${VERSION_FILE}"
+            echo '```'
+          } > /tmp/rustfs-perf-report.md
+          CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+            echo "updated ${REPORT_PATH} in rustfs/dashboard"
           else
-            gh issue create --repo rustfs/backlog --title "${TITLE}" --body-file /tmp/rustfs-perf-issue-body.md
+            jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+            echo "created ${REPORT_PATH} in rustfs/dashboard"
           fi
 
       - name: Upload test logs & results
@@ -191,6 +220,7 @@ jobs:
           path: |
             /tmp/rustfs-perf-test*.log
             /tmp/rustfs-perf-results/**
+            /tmp/rustfs-version.txt
           if-no-files-found: warn
 
       - name: Reset test environment (after)

.github/workflows/rustfs-pool-expand-test.yml

@@ -1,12 +1,11 @@
-name: RustFS Functional Test Suite (S3/KMS/tier/pool/heal)
+name: RustFS Pool Expansion / Heal Test
 
 on:
   workflow_dispatch:
     inputs:
       rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.3)'
+        description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
         required: false
-        default: '1.0.0-rc.3'
       package_url:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
@@ -47,8 +46,8 @@ on:
         type: boolean
         default: true
   workflow_run:
-    # Run after the nightly build completes: S3 -> KMS -> tier -> pool -> heal.
-    workflows: ["Nightly GNU Build"]
+    # Strict shared-environment order: run after tier test succeeds.
+    workflows: ["RustFS Tier Test"]
     types: [completed]
 
 permissions:
@@ -58,7 +57,7 @@ permissions:
 # environment (vm000/vm001/vm002), so concurrent runs must not clobber each
 # other. Jobs inside a run are chained with needs to serialize them.
 concurrency:
-  group: rustfs-pool-expansion-test
+  group: rustfs-shared-functional-tests
   cancel-in-progress: false
 
 defaults:
@@ -76,167 +75,11 @@ env:
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
 
 jobs:
-  # All test scripts live in rustfs/auto-testing; rustfs stores only this
-  # workflow. Each job checks out auto-testing before running.
-
-  s3-compat-test:
-    name: S3 compatibility test
-    runs-on: smoke-testing
-    timeout-minutes: 240
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
-    steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
-      - name: Run S3 compatibility suite
-        run: |
-          chmod +x auto-testing/rustfs-s3-compat-test.sh
-          ARGS=(--all-topologies -y --log-file /tmp/rustfs-s3-compat.log)
-          if [ -n "${{ inputs.package_url }}" ]; then
-            ARGS+=(--package-url "${{ inputs.package_url }}")
-          elif [ -n "${{ inputs.rustfs_version }}" ]; then
-            ARGS+=(--version "${{ inputs.rustfs_version }}")
-          else
-            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
-          fi
-          ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
-
-      - name: Upload test logs
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: rustfs-s3-compat-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-s3-compat*.log
-          if-no-files-found: warn
-
-      - name: Notify on failure
-        if: failure()
-        run: |
-          echo "RustFS S3 compatibility suite failed"
-          echo "See the uploaded log artifact for details."
-
-  kms-test:
-    name: KMS test (after S3)
-    runs-on: smoke-testing
-    timeout-minutes: 360
-    needs: s3-compat-test
-    if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
-    steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
-      - name: Ensure docker (Vault container)
-        run: |
-          if ! command -v docker >/dev/null 2>&1; then
-            sudo apt-get update
-            sudo apt-get install -y docker.io
-          fi
-          sudo systemctl enable --now docker
-          docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1
-
-      - name: Run KMS suite
-        run: |
-          chmod +x auto-testing/rustfs-kms-test.sh
-          ARGS=(--all-topologies --backends local,vault-kv2 -y --log-file /tmp/rustfs-kms.log)
-          if [ -n "${{ inputs.package_url }}" ]; then
-            ARGS+=(--package-url "${{ inputs.package_url }}")
-          elif [ -n "${{ inputs.rustfs_version }}" ]; then
-            ARGS+=(--version "${{ inputs.rustfs_version }}")
-          else
-            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
-          fi
-          ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
-
-      - name: Upload test logs
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: rustfs-kms-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-kms*.log
-          if-no-files-found: warn
-
-      - name: Notify on failure
-        if: failure()
-        run: |
-          echo "RustFS KMS suite failed"
-          echo "See the uploaded log artifact for details."
-
-  tier-test:
-    name: Tier / event / audit test (after KMS)
-    runs-on: smoke-testing
-    timeout-minutes: 360
-    needs: kms-test
-    if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
-    steps:
-      - name: Checkout auto-testing scripts
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: main
-          path: auto-testing
-          persist-credentials: false
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
-      - name: Ensure MQTT broker + clients (event notification tests)
-        run: |
-          if ! command -v mosquitto_sub >/dev/null 2>&1; then
-            sudo apt-get update
-            sudo apt-get install -y mosquitto mosquitto-clients
-          fi
-          sudo mkdir -p /etc/mosquitto/conf.d
-          printf 'listener 1883 0.0.0.0\nallow_anonymous true\n' | sudo tee /etc/mosquitto/conf.d/rustfs-test.conf >/dev/null
-          sudo systemctl restart mosquitto
-          sleep 2
-          ss -tln 2>/dev/null | grep -q ':1883' || { echo "mosquitto not listening on 1883"; exit 1; }
-
-      - name: Run tier / event / audit suite
-        run: |
-          chmod +x auto-testing/rustfs-tier-test.sh
-          ARGS=(--all-topologies -y --log-file /tmp/rustfs-tier.log)
-          if [ -n "${{ inputs.package_url }}" ]; then
-            ARGS+=(--package-url "${{ inputs.package_url }}")
-          elif [ -n "${{ inputs.rustfs_version }}" ]; then
-            ARGS+=(--version "${{ inputs.rustfs_version }}")
-          else
-            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
-          fi
-          ./auto-testing/rustfs-tier-test.sh "${ARGS[@]}"
-
-      - name: Upload test logs
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: rustfs-tier-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-tier*.log
-          if-no-files-found: warn
-
-      - name: Notify on failure
-        if: failure()
-        run: |
-          echo "RustFS tier/event/audit suite failed"
-          echo "See the uploaded log artifact for details."
-
   pool-expansion-test:
-    name: Pool expansion / decommission test (after tier)
+    name: Pool expansion / decommission test
     runs-on: smoke-testing
     timeout-minutes: 360
-    needs: tier-test
-    if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     steps:
       - name: Checkout auto-testing scripts
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -296,12 +139,19 @@ jobs:
               STEPS="$STEPS,9"
             fi
           fi
-          ./auto-testing/rustfs_pool_expand.sh \
-            --steps "$STEPS" --with-warp -y \
+          ARGS=(--steps "$STEPS" --with-warp -y \
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
             --storage-threshold "${{ inputs.storage_threshold || '50' }}" \
             --warp-duration "${{ inputs.warp_duration || '10m' }}" \
-            --log-file /tmp/rustfs-pool-test.log
+            --log-file /tmp/rustfs-pool-test.log)
+          if [ -n "${{ inputs.package_url }}" ]; then
+            ARGS+=(--package-url "${{ inputs.package_url }}")
+          elif [ -n "${{ inputs.rustfs_version }}" ]; then
+            ARGS+=(--version "${{ inputs.rustfs_version }}")
+          else
+            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
+          fi
+          ./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
 
       - name: Upload test logs
         if: always()
@@ -371,12 +221,17 @@ jobs:
 
       - name: Run heal test (write -> outage -> heal -> verify)
         run: |
-          ./auto-testing/rustfs_heal_test.sh \
-            --steps 3,4,5,6,7 -y \
+          ARGS=(--steps "3,4,5,6,7" -y \
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
             --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
             --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
-            --log-file /tmp/rustfs-heal-test.log
+            --log-file /tmp/rustfs-heal-test.log)
+          if [ -n "${{ inputs.package_url }}" ]; then
+            ARGS+=(--package-url "${{ inputs.package_url }}")
+          else
+            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
+          fi
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
 
       - name: Upload test logs
         if: always()

.github/workflows/rustfs-s3-compat-test.yml

@@ -0,0 +1,341 @@
+name: RustFS S3 Compatibility Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      rustfs_version:
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        required: false
+        default: '1.0.0-rc.4-preview.1'
+      package_url:
+        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
+        required: false
+        type: string
+  workflow_run:
+    # Run after the nightly build completes; the nightly deb is what the test installs.
+    workflows: ["Nightly GNU Build"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-shared-functional-tests
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  s3-compat-test:
+    runs-on: smoke-testing
+    continue-on-error: true
+    timeout-minutes: 360
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          openssl version
+          df -h /data | tail -1
+
+      - name: Cleanup environment (before)
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs
+            '
+          done
+
+      - name: Run S3 compatibility suite
+        id: test
+        continue-on-error: true
+        env:
+          LOG_FILE: /tmp/rustfs-s3-compat.log
+        run: |
+          set -euo pipefail
+          chmod +x auto-testing/rustfs-s3-compat-test.sh
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          ARGS=(--all-topologies -y --log-file "${LOG_FILE}")
+          if [ -n "${PACKAGE_URL}" ]; then
+            ARGS+=(--package-url "${PACKAGE_URL}")
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            ARGS+=(--version "${RUSTFS_VERSION}")
+          else
+            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
+          fi
+          ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-s3-compat.log
+          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
+        run: |
+          set -euo pipefail
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          current = None
+          try:
+            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+              for raw in fh:
+                line = ansi.sub('', raw).strip()
+                m = start_re.match(line)
+                if m:
+                  case_id, name = m.group(1), m.group(2)
+                  current = case_id
+                  if case_id not in index:
+                    index[case_id] = len(rows)
+                    rows.append([case_id, name, 'RUNNING'])
+                  continue
+                m = done_re.match(line)
+                if m:
+                  status, case_id = m.group(1), m.group(2)
+                  if case_id in index:
+                    rows[index[case_id]][2] = status
+                  else:
+                    rows.append([case_id, case_id, status])
+                    index[case_id] = len(rows) - 1
+                  current = None
+          except FileNotFoundError:
+            rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+            counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+            out.write('## Case Summary\n\n')
+            out.write(f"- Total: {len(rows)}\\n")
+            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+            out.write('\\n')
+            out.write('| Case | Name | Status |\\n')
+            out.write('| --- | --- | --- |\\n')
+            for case_id, name, status in rows:
+              out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
+          {
+            echo "# RustFS S3 compatibility test report"
+            echo ""
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Package: ${PACKAGE_SOURCE}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo ""
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
+          } | tee "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: always()
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
+          SUITE: s3
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+
+          cat > /tmp/rustfs-functional-index.html <<'EOF'
+          <!doctype html>
+          <html lang="en">
+          <head>
+            <meta charset="utf-8" />
+            <meta name="viewport" content="width=device-width, initial-scale=1" />
+            <title>RustFS Functional Test Reports</title>
+            <style>
+              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
+              * { box-sizing: border-box; }
+              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
+              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
+              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
+              h1 { margin: 0 0 8px; font-size: 26px; }
+              p { margin: 0 0 14px; color: var(--muted); }
+              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
+              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
+              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
+              ul { list-style: none; margin: 0; padding: 0; }
+              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
+              a { color: var(--accent); text-decoration: none; }
+              a:hover { text-decoration: underline; }
+            </style>
+          </head>
+          <body>
+            <div class="wrap">
+              <div class="card">
+                <h1>RustFS Functional Test Reports</h1>
+                <p>S3, KMS, Tier report tabs. Each tab lists reports by date.</p>
+                <div class="tabs" id="tabs"></div>
+                <ul id="list"></ul>
+              </div>
+            </div>
+            <script>
+              const suites = [
+                { key: 's3', label: 'S3 Compatibility' },
+                { key: 'kms', label: 'KMS' },
+                { key: 'tier', label: 'Tier' },
+              ];
+              const tabs = document.getElementById('tabs');
+              const list = document.getElementById('list');
+
+              async function loadSuite(suite) {
+                list.innerHTML = '<li>Loading...</li>';
+                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
+                try {
+                  const res = await fetch(api);
+                  if (!res.ok) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    return;
+                  }
+                  const data = await res.json();
+                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
+                  if (!files.length) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    return;
+                  }
+                  list.innerHTML = files.map(f => `<li><a href="${f.html_url}" target="_blank" rel="noreferrer">${f.name.replace('.md','')}</a></li>`).join('');
+                } catch (_e) {
+                  list.innerHTML = '<li>Failed to load reports.</li>';
+                }
+              }
+
+              function setActive(key) {
+                for (const btn of tabs.querySelectorAll('button')) {
+                  btn.classList.toggle('active', btn.dataset.key === key);
+                }
+                loadSuite(key);
+              }
+
+              for (const suite of suites) {
+                const btn = document.createElement('button');
+                btn.textContent = suite.label;
+                btn.dataset.key = suite.key;
+                btn.addEventListener('click', () => setActive(suite.key));
+                tabs.appendChild(btn);
+              }
+              setActive('s3');
+            </script>
+          </body>
+          </html>
+          EOF
+
+          INDEX_PATH="functional/index.html"
+          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
+          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${INDEX_SHA}" ]; then
+            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          fi
+
+      - name: Upload report and logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-s3-compat-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-s3-compat.log
+            /tmp/rustfs-s3-compat-report.md
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        if: always()
+        run: |
+          set -euo pipefail
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs
+            '
+          done
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS S3 compatibility suite failed"
+          echo "See the uploaded report and log artifacts for details."

.github/workflows/rustfs-tier-test.yml

@@ -0,0 +1,370 @@
+name: RustFS Tier Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      rustfs_version:
+        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
+        required: false
+        default: '1.0.0-rc.4-preview.1'
+      package_url:
+        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
+        required: false
+        type: string
+  workflow_run:
+    # Strict shared-environment order: run after KMS test succeeds.
+    workflows: ["RustFS KMS Test"]
+    types: [completed]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-shared-functional-tests
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
+  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
+  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
+  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
+  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+jobs:
+  tier-test:
+    runs-on: smoke-testing
+    continue-on-error: true
+    timeout-minutes: 420
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Show environment
+        run: |
+          uname -a
+          jq --version
+          openssl version
+          df -h /data | tail -1
+
+      - name: Cleanup environment (before)
+        run: |
+          set -euo pipefail
+          sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
+          sudo rm -f /tmp/rustfs-mosquitto.conf
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
+            '
+          done
+
+      - name: Ensure MQTT broker + clients
+        run: |
+          set -euo pipefail
+          if ! command -v mosquitto_sub >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y mosquitto-clients
+          fi
+          command -v docker >/dev/null 2>&1 || { echo 'docker not found on runner'; exit 1; }
+          sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
+          cat <<'EOF' | sudo tee /tmp/rustfs-mosquitto.conf >/dev/null
+          listener 1883 0.0.0.0
+          allow_anonymous true
+          EOF
+          sudo docker run -d --name rustfs-test-mqtt -p 1883:1883 \
+            -v /tmp/rustfs-mosquitto.conf:/mosquitto/config/mosquitto.conf:ro \
+            eclipse-mosquitto:2 >/dev/null
+          for _ in {1..10}; do
+            if ss -tln 2>/dev/null | grep -q ':1883'; then
+              break
+            fi
+            sleep 1
+          done
+          ss -tln 2>/dev/null | grep -q ':1883' || {
+            echo 'mosquitto container is not listening on 1883'
+            sudo docker logs rustfs-test-mqtt || true
+            exit 1
+          }
+
+      - name: Run tier suite
+        id: test
+        continue-on-error: true
+        env:
+          LOG_FILE: /tmp/rustfs-tier.log
+        run: |
+          set -euo pipefail
+          chmod +x auto-testing/rustfs-tier-test.sh
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          ARGS=(--all-topologies -y --log-file "${LOG_FILE}")
+          if [ -n "${PACKAGE_URL}" ]; then
+            ARGS+=(--package-url "${PACKAGE_URL}")
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            ARGS+=(--version "${RUSTFS_VERSION}")
+          else
+            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
+          fi
+          ./auto-testing/rustfs-tier-test.sh "${ARGS[@]}"
+
+      - name: Generate report
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-tier.log
+          REPORT_FILE: /tmp/rustfs-tier-report.md
+        run: |
+          set -euo pipefail
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          if [ -n "${PACKAGE_URL}" ]; then
+            PACKAGE_SOURCE="${PACKAGE_URL}"
+          elif [ -n "${RUSTFS_VERSION}" ]; then
+            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
+          else
+            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
+          fi
+          CASE_TABLE="/tmp/rustfs-tier-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          try:
+            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+              for raw in fh:
+                line = ansi.sub('', raw).strip()
+                m = start_re.match(line)
+                if m:
+                  case_id, name = m.group(1), m.group(2)
+                  if case_id not in index:
+                    index[case_id] = len(rows)
+                    rows.append([case_id, name, 'RUNNING'])
+                  continue
+                m = done_re.match(line)
+                if m:
+                  status, case_id = m.group(1), m.group(2)
+                  if case_id in index:
+                    rows[index[case_id]][2] = status
+                  else:
+                    rows.append([case_id, case_id, status])
+                    index[case_id] = len(rows) - 1
+          except FileNotFoundError:
+            rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+            counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+            out.write('## Case Summary\n\n')
+            out.write(f"- Total: {len(rows)}\\n")
+            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+            out.write('\\n')
+            out.write('| Case | Name | Status |\\n')
+            out.write('| --- | --- | --- |\\n')
+            for case_id, name, status in rows:
+              out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
+          {
+            echo "# RustFS tier test report"
+            echo ""
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Trigger: ${{ github.event_name }}"
+            echo "- Package: ${PACKAGE_SOURCE}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
+            echo ""
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
+          } | tee "${REPORT_FILE}"
+          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload functional report to dashboard
+        if: always()
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-tier-report.md
+          SUITE: tier
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
+            exit 0
+          fi
+          DATE="$(date -u +%Y-%m-%d)"
+          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${SHA}" ]; then
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
+          fi
+
+          cat > /tmp/rustfs-functional-index.html <<'EOF'
+          <!doctype html>
+          <html lang="en">
+          <head>
+            <meta charset="utf-8" />
+            <meta name="viewport" content="width=device-width, initial-scale=1" />
+            <title>RustFS Functional Test Reports</title>
+            <style>
+              :root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
+              * { box-sizing: border-box; }
+              body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
+              .wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
+              .card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
+              h1 { margin: 0 0 8px; font-size: 26px; }
+              p { margin: 0 0 14px; color: var(--muted); }
+              .tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
+              button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
+              button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
+              ul { list-style: none; margin: 0; padding: 0; }
+              li { padding: 10px 0; border-bottom: 1px solid var(--line); }
+              a { color: var(--accent); text-decoration: none; }
+              a:hover { text-decoration: underline; }
+            </style>
+          </head>
+          <body>
+            <div class="wrap">
+              <div class="card">
+                <h1>RustFS Functional Test Reports</h1>
+                <p>S3, KMS, Tier report tabs. Each tab lists reports by date.</p>
+                <div class="tabs" id="tabs"></div>
+                <ul id="list"></ul>
+              </div>
+            </div>
+            <script>
+              const suites = [
+                { key: 's3', label: 'S3 Compatibility' },
+                { key: 'kms', label: 'KMS' },
+                { key: 'tier', label: 'Tier' },
+              ];
+              const tabs = document.getElementById('tabs');
+              const list = document.getElementById('list');
+
+              async function loadSuite(suite) {
+                list.innerHTML = '<li>Loading...</li>';
+                const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
+                try {
+                  const res = await fetch(api);
+                  if (!res.ok) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    return;
+                  }
+                  const data = await res.json();
+                  const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
+                  if (!files.length) {
+                    list.innerHTML = '<li>No reports yet.</li>';
+                    return;
+                  }
+                  list.innerHTML = files.map(f => `<li><a href="${f.html_url}" target="_blank" rel="noreferrer">${f.name.replace('.md','')}</a></li>`).join('');
+                } catch (_e) {
+                  list.innerHTML = '<li>Failed to load reports.</li>';
+                }
+              }
+
+              function setActive(key) {
+                for (const btn of tabs.querySelectorAll('button')) {
+                  btn.classList.toggle('active', btn.dataset.key === key);
+                }
+                loadSuite(key);
+              }
+
+              for (const suite of suites) {
+                const btn = document.createElement('button');
+                btn.textContent = suite.label;
+                btn.dataset.key = suite.key;
+                btn.addEventListener('click', () => setActive(suite.key));
+                tabs.appendChild(btn);
+              }
+              setActive('s3');
+            </script>
+          </body>
+          </html>
+          EOF
+
+          INDEX_PATH="functional/index.html"
+          INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
+          INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
+          if [ -n "${INDEX_SHA}" ]; then
+            jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          else
+            jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
+              '{message:$msg, content:$content}' \
+              | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+          fi
+
+      - name: Upload report and logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-tier-test-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-tier.log
+            /tmp/rustfs-tier-report.md
+          if-no-files-found: warn
+
+      - name: Cleanup environment (after)
+        if: always()
+        run: |
+          set -euo pipefail
+          sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
+          sudo rm -f /tmp/rustfs-mosquitto.conf
+          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
+          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
+          for node in "${NODES[@]}"; do
+            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
+              set -euo pipefail
+              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
+              ${SUDO} systemctl stop rustfs 2>/dev/null || true
+              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
+                ${SUDO} dpkg -P rustfs
+              fi
+              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
+            '
+          done
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS tier suite failed"
+          echo "See the uploaded report and log artifacts for details."

Cargo.lock

@@ -56,9 +56,9 @@ dependencies = [
 
 [[package]]
 name = "aes"
-version = "0.9.2"
+version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58"
+checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32"
 dependencies = [
  "cipher 0.5.2",
  "cpubits",
@@ -73,7 +73,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f"
 dependencies = [
  "aead",
- "aes 0.9.2",
+ "aes 0.9.3",
  "cipher 0.5.2",
  "ctr",
  "ctutils",
@@ -491,7 +491,7 @@ dependencies = [
  "arrow-select",
  "chrono",
  "half",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "itoa",
  "lexical-core",
  "memchr",
@@ -626,9 +626,9 @@ dependencies = [
 
 [[package]]
 name = "astral-tokio-tar"
-version = "0.6.4"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b18457efd137254e016bbde5e1d88df61c4e1a5ae2223746e56123bac6af2463"
+checksum = "6f2e989b33246fe9240d39accf4dd9a01e0b6c1f3ce9dd095e0a47fa02505523"
 dependencies = [
  "futures-core",
  "libc",
@@ -638,6 +638,7 @@ dependencies = [
  "tokio",
  "tokio-stream",
  "xattr",
+ "zerocopy",
 ]
 
 [[package]]
@@ -1220,7 +1221,7 @@ dependencies = [
  "hyper",
  "hyper-rustls",
  "hyper-util",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "pin-project-lite",
  "rustls",
  "rustls-native-certs",
@@ -2831,7 +2832,7 @@ dependencies = [
  "datafusion-session",
  "datafusion-sql",
  "futures",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "itertools 0.15.0",
  "log",
  "object_store",
@@ -2906,7 +2907,7 @@ dependencies = [
  "foldhash 0.2.0",
  "half",
  "hashbrown 0.17.1",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "itertools 0.15.0",
  "libc",
  "log",
@@ -3111,7 +3112,7 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-functions-window-common",
  "datafusion-physical-expr-common",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "itertools 0.15.0",
  "recursive",
  "serde_json",
@@ -3126,7 +3127,7 @@ checksum = "2604994999d5aeca1d1df645ffc98bc787447aaff05dde27aad0342b48fc1fe0"
 dependencies = [
  "arrow",
  "datafusion-common",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "itertools 0.15.0",
 ]
 
@@ -3282,7 +3283,7 @@ dependencies = [
  "datafusion-expr",
  "datafusion-expr-common",
  "datafusion-physical-expr",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "itertools 0.15.0",
  "log",
  "recursive",
@@ -3304,7 +3305,7 @@ dependencies = [
  "datafusion-physical-expr-common",
  "half",
  "hashbrown 0.17.1",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "itertools 0.15.0",
  "parking_lot",
  "petgraph 0.8.3",
@@ -3338,7 +3339,7 @@ dependencies = [
  "datafusion-common",
  "datafusion-expr-common",
  "hashbrown 0.17.1",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "itertools 0.15.0",
  "parking_lot",
  "pin-project",
@@ -3389,7 +3390,7 @@ dependencies = [
  "futures",
  "half",
  "hashbrown 0.17.1",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "itertools 0.15.0",
  "log",
  "num-traits",
@@ -3442,7 +3443,7 @@ dependencies = [
  "datafusion-common",
  "datafusion-expr",
  "datafusion-functions-nested",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "log",
  "recursive",
  "regex",
@@ -4207,7 +4208,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -4342,12 +4343,12 @@ dependencies = [
 
 [[package]]
 name = "flate2"
-version = "1.1.9"
+version = "1.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
+checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
 dependencies = [
  "crc32fast",
- "miniz_oxide 0.8.9",
+ "miniz_oxide 0.9.1",
  "zlib-rs",
 ]
 
@@ -4617,7 +4618,7 @@ checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c"
 dependencies = [
  "fnv",
  "hashbrown 0.16.1",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "stable_deref_trait",
 ]
 
@@ -4895,7 +4896,7 @@ dependencies = [
  "futures-core",
  "futures-sink",
  "http 1.5.0",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "slab",
  "tokio",
  "tokio-util",
@@ -5338,9 +5339,9 @@ dependencies = [
 
 [[package]]
 name = "hyper"
-version = "1.11.0"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72"
+checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -5560,9 +5561,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.14.0"
+version = "2.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
+checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb"
 dependencies = [
  "equivalent",
  "hashbrown 0.17.1",
@@ -5665,7 +5666,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
 dependencies = [
  "hermit-abi",
  "libc",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -5855,7 +5856,7 @@ dependencies = [
  "base64 0.22.1",
  "getrandom 0.2.17",
  "js-sys",
- "pem",
+ "pem 3.0.6",
  "serde",
  "serde_json",
  "signature 2.2.0",
@@ -5873,7 +5874,7 @@ dependencies = [
  "bytes",
  "crc",
  "crc32c",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "paste",
  "uuid",
 ]
@@ -5904,7 +5905,7 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d"
 dependencies = [
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
 ]
 
 [[package]]
@@ -6384,7 +6385,7 @@ dependencies = [
  "crossbeam-epoch",
  "crossbeam-utils",
  "hashbrown 0.16.1",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "metrics",
  "ordered-float 5.5.0",
  "quanta",
@@ -6528,7 +6529,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
 dependencies = [
  "adler2",
- "simd-adler32",
 ]
 
 [[package]]
@@ -6538,6 +6538,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
 dependencies = [
  "adler2",
+ "simd-adler32",
 ]
 
 [[package]]
@@ -7630,6 +7631,16 @@ dependencies = [
  "serde_core",
 ]
 
+[[package]]
+name = "pem"
+version = "4.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120"
+dependencies = [
+ "base64 0.23.1",
+ "serde_core",
+]
+
 [[package]]
 name = "pem-rfc7468"
 version = "0.7.0"
@@ -7682,7 +7693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772"
 dependencies = [
  "fixedbitset",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
 ]
 
 [[package]]
@@ -7693,7 +7704,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455"
 dependencies = [
  "fixedbitset",
  "hashbrown 0.15.5",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "serde",
 ]
 
@@ -7727,6 +7738,16 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "phf"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6"
+dependencies = [
+ "phf_shared 0.14.0",
+ "serde",
+]
+
 [[package]]
 name = "phf_shared"
 version = "0.12.1"
@@ -7745,6 +7766,15 @@ dependencies = [
  "siphasher",
 ]
 
+[[package]]
+name = "phf_shared"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89"
+dependencies = [
+ "siphasher",
+]
+
 [[package]]
 name = "pin-project"
 version = "1.1.13"
@@ -7845,7 +7875,7 @@ version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca"
 dependencies = [
- "aes 0.9.2",
+ "aes 0.9.3",
  "aes-gcm",
  "cbc 0.2.1",
  "der 0.8.1",
@@ -8343,7 +8373,7 @@ dependencies = [
  "log",
  "murmur3",
  "nom 7.1.3",
- "pem",
+ "pem 3.0.6",
  "prost 0.13.5",
  "prost-build 0.13.5",
  "prost-derive 0.13.5",
@@ -8578,7 +8608,7 @@ dependencies = [
  "once_cell",
  "socket2",
  "tracing",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -8774,12 +8804,12 @@ dependencies = [
 
 [[package]]
 name = "rcgen"
-version = "0.14.9"
+version = "0.14.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "091e7a8e7d86e6feb87a27ce8e2cba29d49eff9507afeebefab7eeb2ca667fb4"
+checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8"
 dependencies = [
  "aws-lc-rs",
- "pem",
+ "pem 4.0.0",
  "rustls-pki-types",
  "time",
  "x509-parser",
@@ -9185,7 +9215,7 @@ version = "0.63.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "35bab1b87d915817d5d9cc352637cd40d5f0b298a48c6309af9156a4addc3031"
 dependencies = [
- "aes 0.9.2",
+ "aes 0.9.3",
  "aws-lc-rs",
  "bitflags 2.13.1",
  "block-padding 0.4.2",
@@ -9825,6 +9855,7 @@ name = "rustfs-heal-contracts"
 version = "1.0.0-rc.4"
 dependencies = [
  "serde",
+ "serde_json",
  "tokio",
  "uuid",
 ]
@@ -9915,7 +9946,7 @@ dependencies = [
  "bytes",
  "fnv",
  "hmac 0.13.0",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "kafka-protocol",
  "metrics",
  "pbkdf2 0.13.0",
@@ -10903,7 +10934,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -10976,7 +11007,7 @@ dependencies = [
  "security-framework",
  "security-framework-sys",
  "webpki-root-certs",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -11033,7 +11064,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 [[package]]
 name = "s3s"
 version = "0.15.0"
-source = "git+https://github.com/rustfs/s3s.git?rev=6e7b41252c7ba218a90886f58d297716ddf68acf#6e7b41252c7ba218a90886f58d297716ddf68acf"
+source = "git+https://github.com/rustfs/s3s.git?rev=9c4690d8e73fc8d184031a19b2c4539ebc77d180#9c4690d8e73fc8d184031a19b2c4539ebc77d180"
 dependencies = [
  "arc-swap",
  "arrayvec",
@@ -11060,6 +11091,7 @@ dependencies = [
  "mime",
  "nom 8.0.0",
  "numeric_cast",
+ "phf 0.14.0",
  "pin-project-lite",
  "quick-xml",
  "regex",
@@ -11089,7 +11121,7 @@ dependencies = [
 [[package]]
 name = "s3s-sigv2"
 version = "0.16.0-alpha.1"
-source = "git+https://github.com/rustfs/s3s.git?rev=6e7b41252c7ba218a90886f58d297716ddf68acf#6e7b41252c7ba218a90886f58d297716ddf68acf"
+source = "git+https://github.com/rustfs/s3s.git?rev=9c4690d8e73fc8d184031a19b2c4539ebc77d180#9c4690d8e73fc8d184031a19b2c4539ebc77d180"
 dependencies = [
  "jiff",
  "thiserror 2.0.20",
@@ -11098,7 +11130,7 @@ dependencies = [
 [[package]]
 name = "s3s-sigv4"
 version = "0.16.0-alpha.1"
-source = "git+https://github.com/rustfs/s3s.git?rev=6e7b41252c7ba218a90886f58d297716ddf68acf#6e7b41252c7ba218a90886f58d297716ddf68acf"
+source = "git+https://github.com/rustfs/s3s.git?rev=9c4690d8e73fc8d184031a19b2c4539ebc77d180#9c4690d8e73fc8d184031a19b2c4539ebc77d180"
 dependencies = [
  "arrayvec",
  "base64-simd",
@@ -11345,7 +11377,7 @@ version = "1.0.151"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
 dependencies = [
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "itoa",
  "memchr",
  "serde",
@@ -11416,7 +11448,7 @@ dependencies = [
  "chrono",
  "hex",
  "indexmap 1.9.3",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "jiff",
  "schemars 0.9.0",
  "schemars 1.2.2",
@@ -11840,7 +11872,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d801accda99469cde6d73da741422610fdf6508a72d9a69d1b55cb241c720597"
 dependencies = [
  "aead",
- "aes 0.9.2",
+ "aes 0.9.3",
  "aes-gcm",
  "chacha20",
  "cipher 0.5.2",
@@ -12195,7 +12227,7 @@ dependencies = [
  "getrandom 0.3.4",
  "once_cell",
  "rustix",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -12563,7 +12595,7 @@ version = "0.25.13+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b"
 dependencies = [
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "toml_datetime",
  "toml_parser",
  "winnow",
@@ -12657,7 +12689,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
 dependencies = [
  "futures-core",
  "futures-util",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "pin-project-lite",
  "slab",
  "sync_wrapper",
@@ -13314,7 +13346,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -13785,7 +13817,7 @@ version = "8.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
 dependencies = [
- "aes 0.9.2",
+ "aes 0.9.3",
  "bzip2",
  "constant_time_eq",
  "crc32fast",
@@ -13793,7 +13825,7 @@ dependencies = [
  "flate2",
  "getrandom 0.4.3",
  "hmac 0.13.0",
- "indexmap 2.14.0",
+ "indexmap 2.14.1",
  "lzma-rust2",
  "memchr",
  "pbkdf2 0.13.0",

Cargo.toml

@@ -155,7 +155,7 @@ futures-util = "0.3.34"
 pollster = "1.0.1"
 pulsar = { default-features = false, version = "6.9.0" }
 lapin = { default-features = false, version = "4.10.0" }
-hyper = { version = "1.11.0" }
+hyper = { version = "1.11.1" }
 hyper-rustls = { default-features = false, version = "0.27.9" }
 hyper-util = { version = "0.1.20" }
 http = "1.5.0"
@@ -232,7 +232,7 @@ tokio-postgres-rustls = "0.14.0"
 # Utilities and Tools
 anyhow = "1.0.104"
 arc-swap = "1.9.2"
-astral-tokio-tar = "0.6.4"
+astral-tokio-tar = "0.7.0"
 atoi = "3.1.0"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.11.0" }
@@ -257,7 +257,7 @@ datafusion = { default-features = false, version = "55.0.0" }
 derive_builder = "0.20.2"
 enumset = "1.1.14"
 faster-hex = "0.10.0"
-flate2 = "1.1.9"
+flate2 = "1.1.10"
 glob = "0.3.4"
 google-cloud-storage = "1.18.0"
 google-cloud-auth = "1.16.0"
@@ -304,7 +304,7 @@ rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.4" }
 rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
-s3s = { git = "https://github.com/rustfs/s3s.git", rev = "6e7b41252c7ba218a90886f58d297716ddf68acf", version = "0.15.0", features = ["minio"] }
+s3s = { git = "https://github.com/rustfs/s3s.git", rev = "9c4690d8e73fc8d184031a19b2c4539ebc77d180", version = "0.15.0", features = ["minio"] }
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.3"
@@ -355,7 +355,7 @@ pyroscope = { version = "2.1.1" }
 libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "10.0.2" }
-rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
+rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
 russh = { version = "0.63.1" }
 russh-sftp = "2.4.0"
 

README.md

@@ -245,6 +245,26 @@ nix build
 nix run
 ```
 
+The flake also exports a NixOS module and the RustFS `rc` client. Add the
+module to your system and provide credentials through runtime files (for
+example, sops-nix or agenix) so secrets are never stored in the Nix store:
+
+```nix
+imports = [ inputs.rustfs.nixosModules.rustfs ];
+
+services.rustfs = {
+  enable = true;
+  accessKeyFile = "/run/secrets/rustfs-access-key";
+  secretKeyFile = "/run/secrets/rustfs-secret-key";
+  volumes = [ "/var/lib/rustfs" ];
+};
+```
+
+Install the S3-compatible client with
+`nix profile install github:rustfs/rustfs#rustfs-client` (the executable is named
+`rc`), or use `inputs.rustfs.packages.${pkgs.system}.rustfs-client` in a system
+configuration.
+
 ### 6\. X-CMD (Option 6)
 
 If you are an [x-cmd](https://www.x-cmd.com/install/rustfs) user:

README_ZH.md

@@ -191,6 +191,12 @@ nix build
 nix run
 ```
 
+该 Flake 同时提供 NixOS 模块和 RustFS `rc` 客户端。将
+`inputs.rustfs.nixosModules.rustfs` 加入 `imports`，并通过运行时密钥文件
+（例如 sops-nix 或 agenix）配置 `accessKeyFile` 与 `secretKeyFile`，避免密钥
+进入 Nix store。客户端包为
+`inputs.rustfs.packages.${pkgs.system}.rustfs-client`，安装后的命令名为 `rc`。
+
 ### 6\. X-CMD (Option 6)
 
 如果你是 [x-cmd](https://www.x-cmd.com/install/rustfs) 用户：

crates/e2e_test/src/common.rs

@@ -217,7 +217,37 @@ pub(crate) async fn signed_s3_request(
     access_key: &str,
     secret_key: &str,
 ) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
-    signed_s3_request_with_session_token(method, url, body, content_type, access_key, secret_key, None).await
+    signed_s3_request_with_headers(method, url, body, content_type, access_key, secret_key, &http::HeaderMap::new()).await
+}
+
+pub(crate) async fn signed_s3_request_with_headers(
+    method: http::Method,
+    url: &str,
+    body: Option<String>,
+    content_type: Option<&str>,
+    access_key: &str,
+    secret_key: &str,
+    extra_headers: &http::HeaderMap,
+) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
+    signed_s3_request_with_session_token(
+        method,
+        url,
+        body,
+        content_type,
+        SigningCredentials {
+            access_key,
+            secret_key,
+            session_token: None,
+        },
+        extra_headers,
+    )
+    .await
+}
+
+struct SigningCredentials<'a> {
+    access_key: &'a str,
+    secret_key: &'a str,
+    session_token: Option<&'a str>,
 }
 
 async fn signed_s3_request_with_session_token(
@@ -225,9 +255,8 @@ async fn signed_s3_request_with_session_token(
     url: &str,
     body: Option<String>,
     content_type: Option<&str>,
-    access_key: &str,
-    secret_key: &str,
-    session_token: Option<&str>,
+    credentials: SigningCredentials<'_>,
+    extra_headers: &http::HeaderMap,
 ) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
     let uri = url.parse::<http::Uri>()?;
     let authority = uri.authority().ok_or("S3 URL missing authority")?.to_string();
@@ -239,14 +268,17 @@ async fn signed_s3_request_with_session_token(
     if let Some(content_type) = content_type {
         request = request.header(CONTENT_TYPE, content_type);
     }
+    for (name, value) in extra_headers {
+        request = request.header(name, value);
+    }
 
     let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "S3 request body is too large")?;
     let signed = sign_v4(
         request.body(Body::empty())?,
         content_length,
-        access_key,
-        secret_key,
-        session_token.unwrap_or_default(),
+        credentials.access_key,
+        credentials.secret_key,
+        credentials.session_token.unwrap_or_default(),
         "us-east-1",
     );
 
@@ -283,8 +315,19 @@ pub(crate) async fn admin_request_with_session_token(
 ) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
     let url = format!("{base_url}{path_and_query}");
     let content_type = body.as_ref().map(|_| "application/json");
-    let response =
-        signed_s3_request_with_session_token(method, &url, body, content_type, access_key, secret_key, session_token).await?;
+    let response = signed_s3_request_with_session_token(
+        method,
+        &url,
+        body,
+        content_type,
+        SigningCredentials {
+            access_key,
+            secret_key,
+            session_token,
+        },
+        &http::HeaderMap::new(),
+    )
+    .await?;
     let status = response.status();
     let body = response.text().await?;
     Ok((status, body))

crates/e2e_test/src/kms/configured_roundtrip_test.rs

@@ -17,6 +17,7 @@
 
 use super::common::{
     LocalKMSTestEnvironment, VAULT_KEY_NAME, VaultTestEnvironment, configure_kms, get_kms_status, kms_admin_request, start_kms,
+    test_sse_kms_encryption,
 };
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{BucketVersioningStatus, ServerSideEncryption, VersioningConfiguration};
@@ -431,6 +432,38 @@ async fn test_configured_local_kms_admin_and_versioned_cleanup() -> TestResult {
     Ok(())
 }
 
+#[tokio::test]
+async fn test_admin_configured_local_kms_is_restored_after_restart() -> TestResult {
+    let mut env = LocalKMSTestEnvironment::new().await?;
+    env.base_env.start_rustfs_server(Vec::new()).await?;
+
+    let default_key_id = env.configure_local_kms().await?;
+    start_kms(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
+
+    env.base_env.restart_server_preserving_data(Vec::new(), &[]).await?;
+    assert_configured_status(
+        &env.base_env.url,
+        &env.base_env.access_key,
+        &env.base_env.secret_key,
+        "local",
+        &default_key_id,
+    )
+    .await?;
+
+    let bucket = format!("kms-restart-{}", Uuid::new_v4());
+    env.base_env.create_test_bucket(&bucket).await?;
+    let client = env.base_env.create_s3_client();
+    test_sse_kms_encryption(&client, &bucket).await?;
+    client
+        .delete_object()
+        .bucket(&bucket)
+        .key("test-sse-kms-object")
+        .send()
+        .await?;
+    env.base_env.delete_test_bucket(&bucket).await?;
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_configured_vault_kms_admin_and_versioned_cleanup() -> TestResult {
     let mut env = VaultTestEnvironment::new().await?;

crates/e2e_test/src/kms/mod.rs

@@ -57,6 +57,9 @@ mod copy_object_version_restore_sse_test;
 #[cfg(test)]
 mod configured_roundtrip_test;
 
+#[cfg(test)]
+mod select_sse_response_test;
+
 #[cfg(test)]
 mod kms_anonymous_enforcement_test;
 

crates/e2e_test/src/kms/select_sse_response_test.rs

@@ -0,0 +1,241 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! SelectObjectContent SSE response-header compatibility (backlog#1625).
+
+use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64, start_kms};
+use crate::common::signed_s3_request_with_headers;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::ServerSideEncryption;
+use base64_simd::STANDARD as BASE64;
+use http::{HeaderMap, Method};
+use std::error::Error;
+use uuid::Uuid;
+
+type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
+
+const CSV_BODY: &[u8] = b"name\nalice\n";
+const SELECT_BODY: &str = r#"<SelectObjectContentRequest xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+<Expression>SELECT * FROM S3Object</Expression>
+<ExpressionType>SQL</ExpressionType>
+<InputSerialization><CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV></InputSerialization>
+<OutputSerialization><CSV/></OutputSerialization>
+</SelectObjectContentRequest>"#;
+const KMS_CONTEXT: &str = "eyJ0ZW5hbnQiOiJzMy1zZWxlY3QifQ==";
+const SSE_ALGORITHM: &str = "x-amz-server-side-encryption";
+const SSE_KMS_KEY_ID: &str = "x-amz-server-side-encryption-aws-kms-key-id";
+const SSE_KMS_CONTEXT: &str = "x-amz-server-side-encryption-context";
+const SSE_C_ALGORITHM: &str = "x-amz-server-side-encryption-customer-algorithm";
+const SSE_C_KEY: &str = "x-amz-server-side-encryption-customer-key";
+const SSE_C_KEY_MD5: &str = "x-amz-server-side-encryption-customer-key-md5";
+const LOG_FLUSH_SENTINEL: &str = "select-sse-log-flush-sentinel.csv";
+
+async fn raw_select(
+    env: &crate::common::RustFSTestEnvironment,
+    bucket: &str,
+    object: &str,
+    request_headers: &HeaderMap,
+) -> TestResult<reqwest::Response> {
+    let url = format!("{}/{bucket}/{object}?select&select-type=2", env.url);
+    signed_s3_request_with_headers(
+        Method::POST,
+        &url,
+        Some(SELECT_BODY.to_string()),
+        Some("application/xml"),
+        &env.access_key,
+        &env.secret_key,
+        request_headers,
+    )
+    .await
+}
+
+async fn assert_success_headers(response: reqwest::Response, expected: &[(&str, &str)], absent: &[&str]) -> TestResult {
+    if response.status() != reqwest::StatusCode::OK {
+        let status = response.status();
+        let url = response.url().clone();
+        let body = response.text().await?;
+        panic!("Select request to {url} failed with {status}: {body}");
+    }
+    for (name, value) in expected {
+        assert_eq!(response.headers().get(*name).and_then(|header| header.to_str().ok()), Some(*value));
+    }
+    for name in absent {
+        assert!(response.headers().get(*name).is_none(), "successful Select response must omit {name}");
+    }
+    let body = response.bytes().await?;
+    assert!(
+        body.windows(b"alice".len()).any(|window| window == b"alice"),
+        "successful Select response must contain a Records event with the selected row"
+    );
+    assert!(
+        body.windows(b"End".len()).any(|window| window == b"End"),
+        "successful Select response must contain the terminal End event"
+    );
+    Ok(())
+}
+
+async fn assert_pre_stream_failure(response: reqwest::Response) -> TestResult {
+    assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
+    let body = response.text().await?;
+    assert!(body.contains("<Error>"), "pre-stream failure must return an S3 XML error: {body}");
+    assert!(
+        body.contains("<Code>InvalidRequest</Code>"),
+        "invalid SSE-C parameters must preserve the S3 error code: {body}"
+    );
+    Ok(())
+}
+
+fn put_object(
+    client: &aws_sdk_s3::Client,
+    bucket: &str,
+    object: &str,
+) -> aws_sdk_s3::operation::put_object::builders::PutObjectFluentBuilder {
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(object)
+        .body(ByteStream::from_static(CSV_BODY))
+}
+
+#[tokio::test]
+async fn select_projects_encryption_headers_and_rejects_invalid_sse_c_before_streaming() -> TestResult {
+    let mut kms = LocalKMSTestEnvironment::new().await?;
+    let log_path = format!("{}/server.log", kms.base_env.temp_dir);
+    kms.base_env.capture_log_path = Some(log_path.clone());
+    kms.base_env
+        .start_rustfs_server_with_env(Vec::new(), &[("RUST_LOG", "s3s=debug,rustfs=info")])
+        .await?;
+    let key_id = kms.configure_local_kms().await?;
+    start_kms(&kms.base_env.url, &kms.base_env.access_key, &kms.base_env.secret_key).await?;
+
+    let client = kms.base_env.create_s3_client();
+    let bucket = format!("select-sse-{}", Uuid::new_v4().simple());
+    client.create_bucket().bucket(&bucket).send().await?;
+
+    put_object(&client, &bucket, "plain.csv").send().await?;
+    put_object(&client, &bucket, "sse-s3.csv")
+        .server_side_encryption(ServerSideEncryption::Aes256)
+        .send()
+        .await?;
+    put_object(&client, &bucket, "sse-kms.csv")
+        .server_side_encryption(ServerSideEncryption::AwsKms)
+        .ssekms_key_id(&key_id)
+        .ssekms_encryption_context(KMS_CONTEXT)
+        .send()
+        .await?;
+
+    let customer_key = "01234567890123456789012345678901";
+    let customer_key_b64 = BASE64.encode_to_string(customer_key);
+    let customer_key_md5 = sse_customer_key_md5_base64(customer_key);
+    put_object(&client, &bucket, "sse-c.csv")
+        .sse_customer_algorithm("AES256")
+        .sse_customer_key(&customer_key_b64)
+        .sse_customer_key_md5(&customer_key_md5)
+        .send()
+        .await?;
+
+    assert_success_headers(
+        raw_select(&kms.base_env, &bucket, "plain.csv", &HeaderMap::new()).await?,
+        &[],
+        &[
+            SSE_ALGORITHM,
+            SSE_KMS_KEY_ID,
+            SSE_KMS_CONTEXT,
+            SSE_C_ALGORITHM,
+            SSE_C_KEY,
+            SSE_C_KEY_MD5,
+        ],
+    )
+    .await?;
+    assert_success_headers(
+        raw_select(&kms.base_env, &bucket, "sse-s3.csv", &HeaderMap::new()).await?,
+        &[(SSE_ALGORITHM, "AES256")],
+        &[SSE_KMS_KEY_ID, SSE_KMS_CONTEXT, SSE_C_ALGORITHM, SSE_C_KEY, SSE_C_KEY_MD5],
+    )
+    .await?;
+    assert_success_headers(
+        raw_select(&kms.base_env, &bucket, "sse-kms.csv", &HeaderMap::new()).await?,
+        &[
+            (SSE_ALGORITHM, "aws:kms"),
+            (SSE_KMS_KEY_ID, &key_id),
+            (SSE_KMS_CONTEXT, KMS_CONTEXT),
+        ],
+        &[SSE_C_ALGORITHM, SSE_C_KEY, SSE_C_KEY_MD5],
+    )
+    .await?;
+
+    let mut sse_c_headers = HeaderMap::new();
+    sse_c_headers.insert(SSE_C_ALGORITHM, "AES256".parse()?);
+    sse_c_headers.insert(SSE_C_KEY, customer_key_b64.parse()?);
+    sse_c_headers.insert(SSE_C_KEY_MD5, customer_key_md5.parse()?);
+    assert_success_headers(
+        raw_select(&kms.base_env, &bucket, "sse-c.csv", &sse_c_headers).await?,
+        &[(SSE_C_ALGORITHM, "AES256"), (SSE_C_KEY_MD5, &customer_key_md5)],
+        &[SSE_ALGORITHM, SSE_KMS_KEY_ID, SSE_KMS_CONTEXT, SSE_C_KEY],
+    )
+    .await?;
+
+    assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &HeaderMap::new()).await?).await?;
+
+    let mut missing_algorithm_headers = HeaderMap::new();
+    missing_algorithm_headers.insert(SSE_C_KEY, customer_key_b64.parse()?);
+    missing_algorithm_headers.insert(SSE_C_KEY_MD5, customer_key_md5.parse()?);
+    assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &missing_algorithm_headers).await?).await?;
+
+    let mut wrong_algorithm_headers = sse_c_headers.clone();
+    wrong_algorithm_headers.insert(SSE_C_ALGORITHM, "AES128".parse()?);
+    assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_algorithm_headers).await?).await?;
+
+    let wrong_md5 = sse_customer_key_md5_base64("99999999999999999999999999999999");
+    let mut wrong_md5_headers = sse_c_headers.clone();
+    wrong_md5_headers.insert(SSE_C_KEY_MD5, wrong_md5.parse()?);
+    assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_md5_headers).await?).await?;
+
+    let wrong_key = "99999999999999999999999999999999";
+    let wrong_key_b64 = BASE64.encode_to_string(wrong_key);
+    let mut wrong_key_headers = HeaderMap::new();
+    wrong_key_headers.insert(SSE_C_ALGORITHM, "AES256".parse()?);
+    wrong_key_headers.insert(SSE_C_KEY, wrong_key_b64.parse()?);
+    wrong_key_headers.insert(SSE_C_KEY_MD5, wrong_md5.parse()?);
+    assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_key_headers).await?).await?;
+
+    put_object(&client, &bucket, LOG_FLUSH_SENTINEL).send().await?;
+    assert_success_headers(
+        raw_select(&kms.base_env, &bucket, LOG_FLUSH_SENTINEL, &HeaderMap::new()).await?,
+        &[],
+        &[
+            SSE_ALGORITHM,
+            SSE_KMS_KEY_ID,
+            SSE_KMS_CONTEXT,
+            SSE_C_ALGORITHM,
+            SSE_C_KEY,
+            SSE_C_KEY_MD5,
+        ],
+    )
+    .await?;
+    let mut logs = String::new();
+    for _ in 0..100 {
+        logs = tokio::fs::read_to_string(&log_path).await?;
+        if logs.contains(LOG_FLUSH_SENTINEL) {
+            break;
+        }
+        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
+    }
+    assert!(logs.contains(LOG_FLUSH_SENTINEL), "timed out waiting for the log sink to flush");
+    for secret in [customer_key, customer_key_b64.as_str(), wrong_key, wrong_key_b64.as_str()] {
+        assert!(!logs.contains(secret), "Select request logging leaked SSE-C customer key material");
+    }
+
+    Ok(())
+}

crates/e2e_test/src/lib.rs

@@ -61,6 +61,9 @@ mod get_codec_streaming_compat_test;
 #[cfg(test)]
 mod version_id_regression_test;
 
+#[cfg(test)]
+mod select_request_root_alias_test;
+
 // Pinned previous-release -> current-build on-disk compatibility.
 #[cfg(test)]
 mod upgrade_compatibility_test;
@@ -164,6 +167,10 @@ mod delete_objects_versioning_test;
 #[cfg(test)]
 mod delete_object_no_content_length_test;
 
+// Regression test for signed empty PutObject requests without Content-Length.
+#[cfg(test)]
+mod put_object_no_content_length_test;
+
 // Delete-marker visibility baseline for data-movement migration proof.
 #[cfg(test)]
 mod delete_marker_migration_semantics_test;

crates/e2e_test/src/put_object_no_content_length_test.rs

@@ -0,0 +1,152 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression coverage for rustfs#6830: a signed empty `PutObject` request
+//! without `Content-Length` and without `Transfer-Encoding` is still a
+//! zero-length object upload.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use http::header::{CONTENT_LENGTH, HOST, TRANSFER_ENCODING};
+    use rustfs_signer::sign_v4;
+    use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+    use s3s::Body;
+    use std::error::Error;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::TcpStream;
+    use tokio::time::{Duration, timeout};
+    use tracing::info;
+
+    const RAW_RESPONSE_TIMEOUT: Duration = Duration::from_secs(10);
+
+    fn parse_status(raw_response: &str) -> Option<u16> {
+        raw_response.lines().next()?.split_whitespace().nth(1)?.parse().ok()
+    }
+
+    async fn send_raw_signed_put(
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+        transfer_encoding: Option<&str>,
+        raw_body: &[u8],
+    ) -> Result<String, Box<dyn Error + Send + Sync>> {
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+        let path_and_query = uri.path_and_query().ok_or("request URL missing path")?.as_str().to_string();
+
+        let mut request = http::Request::builder()
+            .method(http::Method::PUT)
+            .uri(uri)
+            .header(HOST, authority.clone())
+            .header("x-amz-content-sha256", EMPTY_STRING_SHA256_HASH);
+        if let Some(value) = transfer_encoding {
+            request = request.header(TRANSFER_ENCODING, value);
+        }
+
+        let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
+
+        let mut raw_request = format!("PUT {path_and_query} HTTP/1.1\r\nHost: {authority}\r\nConnection: close\r\n");
+        for (name, value) in signed.headers() {
+            if name == HOST || name == CONTENT_LENGTH {
+                continue;
+            }
+            raw_request.push_str(name.as_str());
+            raw_request.push_str(": ");
+            raw_request.push_str(value.to_str()?);
+            raw_request.push_str("\r\n");
+        }
+        raw_request.push_str("\r\n");
+
+        assert!(
+            !raw_request.to_ascii_lowercase().contains("\r\ncontent-length:"),
+            "raw regression request must omit Content-Length; request was:\n{raw_request}"
+        );
+
+        let mut stream = TcpStream::connect(&authority).await?;
+        stream.write_all(raw_request.as_bytes()).await?;
+        stream.write_all(raw_body).await?;
+        stream.flush().await?;
+
+        let mut response = Vec::new();
+        timeout(RAW_RESPONSE_TIMEOUT, stream.read_to_end(&mut response))
+            .await
+            .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out reading raw PUT response"))??;
+        Ok(String::from_utf8_lossy(&response).into_owned())
+    }
+
+    #[tokio::test]
+    async fn test_put_object_without_content_length_boundaries() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        info!("TEST: PutObject without Content-Length boundaries");
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let empty_bucket = "put-no-content-length";
+        let empty_key = "empty.bin";
+        let chunked_bucket = "put-chunked-no-length";
+        let chunked_key = "chunked.bin";
+
+        client.create_bucket().bucket(empty_bucket).send().await?;
+        client.create_bucket().bucket(chunked_bucket).send().await?;
+
+        let url = format!("{}/{}/{}", env.url, empty_bucket, empty_key);
+        let raw_response = send_raw_signed_put(&url, &env.access_key, &env.secret_key, None, b"").await?;
+        info!("raw empty PUT response:\n{}", raw_response);
+
+        assert_eq!(
+            parse_status(&raw_response),
+            Some(200),
+            "empty PutObject without Content-Length should succeed, got:\n{raw_response}"
+        );
+        assert!(
+            raw_response.to_ascii_lowercase().contains("\r\netag:"),
+            "successful PutObject should return an ETag header: {raw_response}"
+        );
+
+        let head = client.head_object().bucket(empty_bucket).key(empty_key).send().await?;
+        assert_eq!(head.content_length(), Some(0), "stored object must be zero length");
+
+        let url = format!("{}/{}/{}", env.url, chunked_bucket, chunked_key);
+        let raw_response = send_raw_signed_put(&url, &env.access_key, &env.secret_key, Some("chunked"), b"0\r\n\r\n").await?;
+        info!("raw chunked PUT response:\n{}", raw_response);
+
+        assert_eq!(
+            parse_status(&raw_response),
+            Some(411),
+            "unknown-length chunked PutObject must stay rejected, got:\n{raw_response}"
+        );
+        assert!(
+            raw_response.contains("<Code>MissingContentLength</Code>"),
+            "expected MissingContentLength, got:\n{raw_response}"
+        );
+
+        let missing = client
+            .head_object()
+            .bucket(chunked_bucket)
+            .key(chunked_key)
+            .send()
+            .await
+            .expect_err("rejected unknown-length PUT must not create an object");
+        assert_eq!(
+            missing.raw_response().map(|response| response.status().as_u16()),
+            Some(404),
+            "rejected unknown-length PUT absence probe must return HTTP 404, got {missing:?}"
+        );
+
+        Ok(())
+    }
+}

crates/e2e_test/src/reliant/tiering.rs

@@ -210,19 +210,27 @@ async fn add_rustfs_tier(hot: &RustFSTestEnvironment, cold: &RustFSTestEnvironme
     }
 }
 
-async fn remove_rustfs_tier_force(hot: &RustFSTestEnvironment) -> TestResult {
-    let path = format!("/rustfs/admin/v3/tier/{TIER_NAME}?force=true");
+fn clear_tiers_confirmation_token(now: OffsetDateTime) -> String {
+    let mut rand = "AGD1R25GI3I1GJGUGJFD7FBS4DFAASDF".to_string();
+    rand.insert_str(3, &now.day().to_string());
+    rand.insert_str(17, &now.month().to_string());
+    rand.insert_str(23, &now.year().to_string());
+    rand
+}
+
+async fn clear_rustfs_tiers_force(hot: &RustFSTestEnvironment) -> TestResult {
     let deadline = Instant::now() + StdDuration::from_secs(30);
     loop {
-        let (status, resp) =
-            signed_admin_request(&hot.url, Method::DELETE, &path, None, &hot.access_key, &hot.secret_key).await?;
+        let rand = clear_tiers_confirmation_token(OffsetDateTime::now_utc());
+        let path = format!("/rustfs/admin/v3/tier/clear?rand={rand}&force=true");
+        let (status, resp) = signed_admin_request(&hot.url, Method::POST, &path, None, &hot.access_key, &hot.secret_key).await?;
         if status.is_success() {
             return Ok(());
         }
         if (!resp.contains("TierNameBackendInUse") && !resp.contains(TIER_MUTATION_RECOVERY_CHANGED))
             || Instant::now() >= deadline
         {
-            return Err(format!("RemoveTier(RustFS) failed: status={status}, body={resp}").into());
+            return Err(format!("ClearTier(RustFS) failed: status={status}, body={resp}").into());
         }
         // Tier mutation cleanup and startup recovery are asynchronous.
         tokio::time::sleep(StdDuration::from_millis(100)).await;
@@ -1708,7 +1716,7 @@ async fn test_manual_transition_async_tier_failure_reports_terminal_partial() ->
         0,
     )
     .await?;
-    remove_rustfs_tier_force(&hot).await?;
+    clear_rustfs_tiers_force(&hot).await?;
 
     let due_mtime = OffsetDateTime::now_utc() - time::Duration::hours(25);
     put_backdated_single_part_object(

crates/e2e_test/src/select_request_root_alias_test.rs

@@ -0,0 +1,84 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Raw HTTP regression coverage for the Select request root alias (backlog#1626).
+
+use crate::common::{RustFSTestEnvironment, signed_s3_request};
+use aws_sdk_s3::primitives::ByteStream;
+use http::Method;
+use std::error::Error;
+use uuid::Uuid;
+
+type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
+
+const CSV_BODY: &[u8] = b"name\nGatewayJ-root-alias\nignored\n";
+const EXPECTED_RECORD: &[u8] = b"GatewayJ-root-alias";
+
+fn select_request(root: &str) -> String {
+    format!(
+        r#"<{root} xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+<Expression>SELECT s.name FROM S3Object s WHERE s.name = 'GatewayJ-root-alias'</Expression>
+<ExpressionType>SQL</ExpressionType>
+<InputSerialization><CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV></InputSerialization>
+<OutputSerialization><CSV/></OutputSerialization>
+</{root}>"#
+    )
+}
+
+async fn raw_select(env: &RustFSTestEnvironment, bucket: &str, object: &str, root: &str) -> TestResult {
+    let response = signed_s3_request(
+        Method::POST,
+        &format!("{}/{bucket}/{object}?select&select-type=2", env.url),
+        Some(select_request(root)),
+        Some("application/xml"),
+        &env.access_key,
+        &env.secret_key,
+    )
+    .await?;
+    let status = response.status();
+    let body = response.bytes().await?.to_vec();
+    assert_eq!(
+        status,
+        reqwest::StatusCode::OK,
+        "{root} root was rejected: {}",
+        String::from_utf8_lossy(&body)
+    );
+    assert!(
+        body.windows(EXPECTED_RECORD.len()).any(|window| window == EXPECTED_RECORD),
+        "{root} root did not return the projected record"
+    );
+    Ok(())
+}
+
+#[tokio::test]
+async fn select_request_root_alias_reaches_select_endpoint() -> TestResult {
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(Vec::new()).await?;
+    let client = env.create_s3_client();
+    let bucket = format!("select-root-{}", Uuid::new_v4().simple());
+    let object = "input.csv";
+
+    client.create_bucket().bucket(&bucket).send().await?;
+    client
+        .put_object()
+        .bucket(&bucket)
+        .key(object)
+        .body(ByteStream::from_static(CSV_BODY))
+        .send()
+        .await?;
+
+    raw_select(&env, &bucket, object, "SelectObjectContentRequest").await?;
+    raw_select(&env, &bucket, object, "SelectRequest").await?;
+    Ok(())
+}

crates/ecstore/src/api/mod.rs

@@ -194,9 +194,10 @@ pub mod bucket {
             BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats,
             DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric,
             MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, OperatorRuleContract,
-            REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
-            REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
-            REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
+            REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS,
+            REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE,
+            REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
+            ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
             ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
             ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
             ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
@@ -405,7 +406,7 @@ pub mod notification {
     pub use crate::services::notification_sys::{
         CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
         acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
-        new_global_notification_sys, start_remote_version_state_fleet_probe,
+        new_global_notification_sys, scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
     };
 }
 
@@ -415,9 +416,10 @@ pub mod object {
         GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver,
         ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
         RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
-        SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, StreamConsumer, get_object_body_cache_plaintext_len,
-        lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
-        unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
+        SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
+        ScannerPublicationCommitState, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
+        register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
+        unregister_object_mutation_hook,
     };
     pub use crate::store::{
         PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -22,6 +22,7 @@ use crate::bucket::target::{self, BucketTarget, BucketTargets, Credentials};
 use crate::bucket::versioning_sys::BucketVersioningSys;
 use crate::runtime::sources as runtime_sources;
 use aws_credential_types::Credentials as SdkCredentials;
+use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future};
 use aws_sdk_s3::config::Region as SdkRegion;
 use aws_sdk_s3::config::SharedHttpClient;
 use aws_sdk_s3::error::ProvideErrorMetadata;
@@ -77,7 +78,7 @@ use std::str::FromStr as _;
 use std::sync::Arc;
 use std::sync::OnceLock;
 use std::sync::Weak;
-use std::time::{Duration, Instant};
+use std::time::{Duration, Instant, SystemTime};
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};
 use tokio::sync::Mutex;
 use tokio::sync::RwLock;
@@ -89,6 +90,71 @@ use uuid::Uuid;
 
 const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
 const REDACTED_CREDENTIAL: &str = "<redacted>";
+const EXPIRED_REMOTE_TARGET_CREDENTIALS: &str = "remote target credentials have expired";
+
+#[derive(Clone)]
+struct RemoteTargetCredentialsProvider {
+    credentials: SdkCredentials,
+}
+
+impl RemoteTargetCredentialsProvider {
+    fn resolve_at(&self, now: SystemTime) -> aws_credential_types::provider::Result {
+        if self.credentials.expiry().is_some_and(|expiration| expiration <= now) {
+            return Err(CredentialsError::provider_error(std::io::Error::other(EXPIRED_REMOTE_TARGET_CREDENTIALS)));
+        }
+        Ok(self.credentials.clone())
+    }
+}
+
+impl fmt::Debug for RemoteTargetCredentialsProvider {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("RemoteTargetCredentialsProvider")
+            .field("temporary", &self.credentials.session_token().is_some())
+            .field("expiration", &self.credentials.expiry())
+            .finish()
+    }
+}
+
+impl ProvideCredentials for RemoteTargetCredentialsProvider {
+    fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a>
+    where
+        Self: 'a,
+    {
+        future::ProvideCredentials::ready(self.resolve_at(SystemTime::now()))
+    }
+
+    fn fallback_on_interrupt(&self) -> Option<SdkCredentials> {
+        self.resolve_at(SystemTime::now()).ok()
+    }
+}
+
+fn remote_target_sdk_credentials(
+    credentials: &Credentials,
+    account_id: &str,
+    now: SystemTime,
+) -> Result<SdkCredentials, &'static str> {
+    let session_token = credentials.effective_session_token();
+    let expiration = credentials.effective_expiration().map(SystemTime::from);
+    if expiration.is_some() && session_token.is_none() {
+        return Err("remote target credential expiration requires a session token");
+    }
+    if expiration.is_some_and(|expiration| expiration <= now) {
+        return Err(EXPIRED_REMOTE_TARGET_CREDENTIALS);
+    }
+
+    let mut builder = SdkCredentials::builder()
+        .access_key_id(credentials.access_key.clone())
+        .secret_access_key(credentials.secret_key.clone())
+        .account_id(account_id.to_string())
+        .provider_name("bucket_target_sys");
+    if let Some(session_token) = session_token {
+        builder = builder.session_token(session_token.to_string());
+    }
+    if let Some(expiration) = expiration {
+        builder = builder.expiry(expiration);
+    }
+    Ok(builder.build())
+}
 
 pub type HeadObjectSdkError = Box<SdkError<HeadObjectError>>;
 pub type GetObjectSdkError = Box<SdkError<GetObjectError>>;
@@ -845,13 +911,26 @@ impl BucketTargetSys {
         Ok(BucketTargets { targets: new_targets })
     }
 
+    async fn mark_refresh_attempt(&self, arn: &str) {
+        // Rate-limit a failed config fetch as well as a failed client build.
+        // A successful rebuild replaces this timestamp during publication.
+        self.arn_remotes_map
+            .write()
+            .await
+            .entry(arn.to_string())
+            .or_default()
+            .last_refresh = OffsetDateTime::now_utc();
+    }
+
     pub async fn mark_refresh_in_progress(&self, bucket: &str, arn: &str) {
         let mut arn_errs = self.arn_errs_map.write().await;
-        arn_errs.entry(arn.to_string()).or_insert_with(|| ArnErrs {
-            bucket: bucket.to_string(),
-            update_in_progress: true,
+        let err = arn_errs.entry(arn.to_string()).or_insert_with(|| ArnErrs {
             count: 1,
+            bucket: bucket.to_string(),
+            ..Default::default()
         });
+        err.update_in_progress = true;
+        err.bucket = bucket.to_string();
     }
 
     pub async fn mark_refresh_done(&self, bucket: &str, arn: &str) {
@@ -863,15 +942,21 @@ impl BucketTargetSys {
     }
 
     pub async fn is_reloading_target(&self, _bucket: &str, arn: &str) -> bool {
-        let arn_errs = self.arn_errs_map.read().await;
-        arn_errs.get(arn).map(|err| err.update_in_progress).unwrap_or(false)
+        self.arn_errs_map
+            .read()
+            .await
+            .get(arn)
+            .is_some_and(|err| err.update_in_progress)
     }
 
-    pub async fn inc_arn_errs(&self, _bucket: &str, arn: &str) {
+    pub async fn inc_arn_errs(&self, bucket: &str, arn: &str) {
         let mut arn_errs = self.arn_errs_map.write().await;
-        if let Some(err) = arn_errs.get_mut(arn) {
-            err.count += 1;
-        }
+        let err = arn_errs.entry(arn.to_string()).or_insert_with(|| ArnErrs {
+            bucket: bucket.to_string(),
+            ..Default::default()
+        });
+        err.count += 1;
+        err.bucket = bucket.to_string();
     }
 
     pub async fn get_remote_target_client(&self, bucket: &str, arn: &str) -> Option<Arc<TargetClient>> {
@@ -884,15 +969,15 @@ impl BucketTargetSys {
                 .unwrap_or((None, None))
         };
 
-        if let Some(cli) = cli {
+        let credentials_expired = cli
+            .as_ref()
+            .is_some_and(|client| client.credentials_expired_at(jiff::Timestamp::now()));
+        if let Some(cli) = cli
+            && !credentials_expired
+        {
             return Some(cli);
         }
 
-        // TODO(backlog): spawn an async task to proactively reload the replication target
-        if self.is_reloading_target(bucket, arn).await {
-            return None;
-        }
-
         if let Some(last_refresh) = last_refresh {
             let now = OffsetDateTime::now_utc();
             if now - last_refresh < Duration::from_secs(60 * 5) {
@@ -900,16 +985,24 @@ impl BucketTargetSys {
             }
         }
 
+        // The existing per-bucket publication lock is also the reload claim:
+        // try-locking keeps the request path non-blocking, is cancellation-safe,
+        // and prevents a stale reload from publishing after a credential update.
+        let update_mutex = self.target_update_mutex(bucket).await;
+        let Ok(update_guard) = update_mutex.try_lock() else {
+            return None;
+        };
+        self.mark_refresh_attempt(arn).await;
+
         match get_bucket_targets_config(bucket).await {
             Ok(bucket_targets) => {
-                self.mark_refresh_in_progress(bucket, arn).await;
-                self.update_all_targets(bucket, Some(&bucket_targets)).await;
-                self.mark_refresh_done(bucket, arn).await;
+                self.update_all_targets_locked(bucket, Some(&bucket_targets)).await;
             }
             Err(e) => {
                 error!("get bucket targets config error:{}", e);
             }
         };
+        drop(update_guard);
 
         let cli = self
             .arn_remotes_map
@@ -917,8 +1010,10 @@ impl BucketTargetSys {
             .await
             .get(arn)
             .and_then(|target| target.client.clone());
-        if cli.is_some() {
-            return cli;
+        if let Some(cli) = cli
+            && !cli.credentials_expired_at(jiff::Timestamp::now())
+        {
+            return Some(cli);
         }
 
         self.inc_arn_errs(bucket, arn).await;
@@ -948,12 +1043,13 @@ impl BucketTargetSys {
             });
         };
 
-        let creds = SdkCredentials::builder()
-            .access_key_id(credentials.access_key.clone())
-            .secret_access_key(credentials.secret_key.clone())
-            .account_id(target.reset_id.clone())
-            .provider_name("bucket_target_sys")
-            .build();
+        let creds = remote_target_sdk_credentials(credentials, &target.reset_id, SystemTime::now()).map_err(|error| {
+            BucketTargetError::RemoteTargetConnectionErr {
+                bucket: target.target_bucket.clone(),
+                access_key: credentials.access_key.clone(),
+                error: error.to_string(),
+            }
+        })?;
 
         let endpoint = if target.secure {
             format!("https://{}", target.endpoint)
@@ -973,7 +1069,7 @@ impl BucketTargetSys {
 
         let mut config_builder = S3Config::builder()
             .endpoint_url(endpoint.clone())
-            .credentials_provider(SharedCredentialsProvider::new(creds))
+            .credentials_provider(SharedCredentialsProvider::new(RemoteTargetCredentialsProvider { credentials: creds }))
             .region(SdkRegion::new(target.region.clone()))
             .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest());
 
@@ -1047,6 +1143,13 @@ impl BucketTargetSys {
         let update_mutex = self.target_update_mutex(bucket).await;
         let _update_guard = update_mutex.lock().await;
 
+        self.update_all_targets_locked(bucket, targets).await;
+    }
+
+    /// Builds and publishes one bucket snapshot while its update mutex is held.
+    /// Keeping persisted-config reads under the same mutex prevents a stale
+    /// reload from overwriting a concurrent credential rotation.
+    async fn update_all_targets_locked(&self, bucket: &str, targets: Option<&BucketTargets>) {
         let mut clients = Vec::new();
         if let Some(new_targets) = targets {
             for target in &new_targets.targets {
@@ -1078,6 +1181,17 @@ impl BucketTargetSys {
             && !new_targets.is_empty()
         {
             for (target, client) in clients {
+                // Keep a timestamped placeholder for configured targets whose
+                // client cannot be built. Replication records these attempts as
+                // failed, while the placeholder prevents every object from
+                // triggering another metadata reload/client build for five minutes.
+                arn_remotes_map.insert(
+                    target.arn.clone(),
+                    ArnTarget {
+                        client: None,
+                        last_refresh: OffsetDateTime::now_utc(),
+                    },
+                );
                 match client {
                     Ok(client) => {
                         arn_remotes_map.insert(
@@ -1090,11 +1204,6 @@ impl BucketTargetSys {
                         health_map.insert(client.arn.clone(), target_health(&client));
                         self.update_bandwidth_limit(bucket, &target.arn, target.bandwidth_limit);
                     }
-                    // The target stays in `targets_map`, so it keeps showing up in
-                    // `bucket remote ls` while no client exists to replicate through it —
-                    // replication then drops every object for this ARN. Without this the
-                    // rejection (loopback endpoint, bad CA, unparseable URL) left no trace
-                    // anywhere.
                     Err(err) => warn!(
                         bucket = %bucket,
                         arn = %target.arn,
@@ -1962,6 +2071,13 @@ pub struct TargetClient {
 }
 
 impl TargetClient {
+    fn credentials_expired_at(&self, now: jiff::Timestamp) -> bool {
+        self.credentials
+            .as_ref()
+            .and_then(Credentials::effective_expiration)
+            .is_some_and(|expiration| expiration <= now)
+    }
+
     pub fn to_url(&self) -> Url {
         Url::parse(&self.endpoint).unwrap()
     }
@@ -2366,6 +2482,21 @@ impl TargetClient {
         }
     }
 
+    pub async fn abort_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str) -> Result<(), S3ClientError> {
+        match self
+            .client
+            .abort_multipart_upload()
+            .bucket(bucket)
+            .key(object)
+            .upload_id(upload_id)
+            .send()
+            .await
+        {
+            Ok(_) => Ok(()),
+            Err(e) => Err(e.into()),
+        }
+    }
+
     pub async fn remove_object(
         &self,
         bucket: &str,
@@ -2542,6 +2673,26 @@ mod tests {
         }
     }
 
+    #[derive(Clone, Debug)]
+    struct RecordingAuthConnector {
+        signed_requests: Arc<std::sync::Mutex<Vec<(bool, bool)>>>,
+    }
+
+    impl SmithyHttpConnector for RecordingAuthConnector {
+        fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
+            let has_expected_token = request.headers().get("x-amz-security-token") == Some("temporary-session-token");
+            let has_authorization = request.headers().contains_key("authorization");
+            self.signed_requests
+                .lock()
+                .expect("recorded auth request lock should not be poisoned")
+                .push((has_expected_token, has_authorization));
+            HttpConnectorFuture::ready(Ok(HttpResponse::new(
+                aws_smithy_runtime_api::http::StatusCode::try_from(200_u16).expect("200 should be a valid response status"),
+                SdkBody::empty(),
+            )))
+        }
+    }
+
     fn recording_target_client() -> (TargetClient, Arc<std::sync::Mutex<Vec<String>>>) {
         let request_uris = Arc::new(std::sync::Mutex::new(Vec::new()));
         let connector = SharedHttpConnector::new(RecordingHttpConnector {
@@ -2567,6 +2718,150 @@ mod tests {
         )
     }
 
+    #[test]
+    fn remote_target_sdk_credentials_preserve_temporary_credential_fields() {
+        let now = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000);
+        let expiration = SystemTime::UNIX_EPOCH + Duration::from_secs(2_000);
+        let credentials = Credentials {
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            session_token: Some("temporary-session-token".to_string()),
+            expiration: Some(jiff::Timestamp::try_from(expiration).expect("test expiration should convert")),
+        };
+
+        let sdk_credentials =
+            remote_target_sdk_credentials(&credentials, "account", now).expect("unexpired temporary credentials should build");
+
+        assert_eq!(sdk_credentials.session_token(), Some("temporary-session-token"));
+        assert_eq!(sdk_credentials.expiry(), Some(expiration));
+        assert_eq!(sdk_credentials.account_id().map(|id| id.as_str()), Some("account"));
+    }
+
+    #[test]
+    fn remote_target_sdk_credentials_normalize_go_zero_expiration() {
+        let credentials = Credentials {
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            session_token: None,
+            expiration: Some("0001-01-01T00:00:00Z".parse().expect("Go zero time should parse")),
+        };
+
+        let sdk_credentials = remote_target_sdk_credentials(&credentials, "", SystemTime::now())
+            .expect("Go zero expiration should remain compatible with static credentials");
+
+        assert!(sdk_credentials.session_token().is_none());
+        assert!(sdk_credentials.expiry().is_none());
+    }
+
+    #[test]
+    fn remote_target_sdk_credentials_reject_invalid_expiration_boundaries() {
+        let expiration = SystemTime::UNIX_EPOCH + Duration::from_secs(2_000);
+        let mut credentials = Credentials {
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            session_token: None,
+            expiration: Some(jiff::Timestamp::try_from(expiration).expect("test expiration should convert")),
+        };
+
+        assert_eq!(
+            remote_target_sdk_credentials(&credentials, "", SystemTime::UNIX_EPOCH + Duration::from_secs(1_000))
+                .expect_err("expiration without a session token must fail"),
+            "remote target credential expiration requires a session token"
+        );
+
+        credentials.session_token = Some("temporary-session-token".to_string());
+        assert_eq!(
+            remote_target_sdk_credentials(&credentials, "", expiration)
+                .expect_err("credentials expire at the exact expiration boundary"),
+            EXPIRED_REMOTE_TARGET_CREDENTIALS
+        );
+    }
+
+    #[test]
+    fn remote_target_credentials_provider_fails_closed_after_expiration() {
+        let expiration = SystemTime::UNIX_EPOCH + Duration::from_secs(2_000);
+        let provider = RemoteTargetCredentialsProvider {
+            credentials: SdkCredentials::new(
+                "access",
+                "secret",
+                Some("temporary-session-token".to_string()),
+                Some(expiration),
+                "test",
+            ),
+        };
+
+        assert!(provider.resolve_at(expiration - Duration::from_nanos(1)).is_ok());
+        let err = provider
+            .resolve_at(expiration)
+            .expect_err("expired credentials must not be returned");
+        assert_eq!(err.source().map(ToString::to_string).as_deref(), Some(EXPIRED_REMOTE_TARGET_CREDENTIALS));
+        assert!(!format!("{provider:?}").contains("temporary-session-token"));
+        assert!(!format!("{provider:?}").contains("secret"));
+    }
+
+    #[test]
+    fn target_client_detects_expiration_for_cache_refresh() {
+        let expiration: jiff::Timestamp = "2099-01-01T00:00:00Z".parse().expect("expiration should parse");
+        let (mut client, _) = recording_target_client();
+        client.credentials = Some(Credentials {
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            session_token: Some("temporary-session-token".to_string()),
+            expiration: Some(expiration),
+        });
+
+        assert!(!client.credentials_expired_at("2098-12-31T23:59:59Z".parse().expect("pre-expiration timestamp should parse")));
+        assert!(client.credentials_expired_at(expiration));
+
+        client.credentials.as_mut().expect("credentials should exist").expiration =
+            Some("0001-01-01T00:00:00Z".parse().expect("Go zero time should parse"));
+        assert!(!client.credentials_expired_at(jiff::Timestamp::now()));
+    }
+
+    #[tokio::test]
+    async fn temporary_credentials_add_security_token_to_sigv4_requests() {
+        let signed_requests = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let connector = SharedHttpConnector::new(RecordingAuthConnector {
+            signed_requests: Arc::clone(&signed_requests),
+        });
+        let http_client = http_client_fn(move |_settings, _components| connector.clone());
+        let credentials = Credentials {
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            session_token: Some("temporary-session-token".to_string()),
+            expiration: Some("2099-01-01T00:00:00Z".parse().expect("future expiration should parse")),
+        };
+        let sdk_credentials = remote_target_sdk_credentials(&credentials, "", SystemTime::now())
+            .expect("unexpired temporary credentials should build");
+        let client = S3Client::from_conf(
+            S3Config::builder()
+                .endpoint_url("https://target.example")
+                .credentials_provider(SharedCredentialsProvider::new(RemoteTargetCredentialsProvider {
+                    credentials: sdk_credentials,
+                }))
+                .region(SdkRegion::new("us-east-1"))
+                .http_client(http_client)
+                .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
+                .build(),
+        );
+
+        client
+            .head_bucket()
+            .bucket("target-bucket")
+            .send()
+            .await
+            .expect("recording connector should accept the signed request");
+
+        assert_eq!(
+            signed_requests
+                .lock()
+                .expect("recorded auth request lock should not be poisoned")
+                .as_slice(),
+            &[(true, true)],
+            "SigV4 request must include both authorization and the session-token header"
+        );
+    }
+
     fn spawn_https_server(cert: &rcgen::CertifiedKey<rcgen::KeyPair>, requests: usize) -> (u16, std::thread::JoinHandle<()>) {
         use std::io::{Read, Write};
 
@@ -3498,6 +3793,29 @@ mod tests {
         assert!(mutexes.contains_key("second"));
     }
 
+    #[tokio::test]
+    async fn target_refresh_attempt_updates_retry_timestamp_and_error_count() {
+        let sys = BucketTargetSys::default();
+
+        sys.mark_refresh_attempt("arn:reload").await;
+        let last_refresh = sys.arn_remotes_map.read().await["arn:reload"].last_refresh;
+        assert!(OffsetDateTime::now_utc() - last_refresh < Duration::from_secs(5));
+
+        sys.inc_arn_errs("bucket", "arn:reload").await;
+        sys.inc_arn_errs("bucket", "arn:reload").await;
+        let errors = sys.arn_errs_map.read().await;
+        assert_eq!(errors["arn:reload"].count, 2);
+        assert_eq!(errors["arn:reload"].bucket, "bucket");
+        drop(errors);
+
+        sys.mark_refresh_in_progress("bucket", "arn:reload").await;
+        assert!(sys.is_reloading_target("bucket", "arn:reload").await);
+        sys.mark_refresh_done("bucket", "arn:reload").await;
+        assert!(!sys.is_reloading_target("bucket", "arn:reload").await);
+        sys.mark_refresh_in_progress("bucket", "arn:reload").await;
+        assert!(sys.is_reloading_target("bucket", "arn:reload").await);
+    }
+
     #[tokio::test]
     async fn update_all_targets_publishes_disable_proxy_on_target_client() {
         // The read-proxy selector (replication_proxy::get_proxy_targets) skips
@@ -3536,6 +3854,88 @@ mod tests {
         assert!(opted_out.disable_proxy, "disable_proxy must reach the published TargetClient");
     }
 
+    #[tokio::test]
+    async fn update_all_targets_keeps_failed_client_placeholder() {
+        let sys = BucketTargetSys::default();
+        let target = BucketTarget {
+            arn: "arn:expired".to_string(),
+            endpoint: "192.168.1.10:9000".to_string(),
+            target_bucket: "target-bucket".to_string(),
+            region: "us-east-1".to_string(),
+            credentials: Some(Credentials {
+                access_key: "access".to_string(),
+                secret_key: "secret".to_string(),
+                session_token: Some("temporary-session-token".to_string()),
+                expiration: Some("2000-01-01T00:00:00Z".parse().expect("expired timestamp should parse")),
+            }),
+            ..Default::default()
+        };
+        let targets = BucketTargets { targets: vec![target] };
+
+        sys.update_all_targets("bucket", Some(&targets)).await;
+
+        let remotes = sys.arn_remotes_map.read().await;
+        let placeholder = remotes
+            .get("arn:expired")
+            .expect("configured target should retain a cache entry");
+        assert!(placeholder.client.is_none());
+        assert!(OffsetDateTime::now_utc() - placeholder.last_refresh < Duration::from_secs(5));
+        drop(remotes);
+        assert!(sys.get_remote_target_client("bucket", "arn:expired").await.is_none());
+    }
+
+    #[tokio::test]
+    async fn credential_rotation_atomically_replaces_published_client() {
+        let sys = BucketTargetSys::default();
+        let target = |session_token: &str| BucketTarget {
+            arn: "arn:rotating".to_string(),
+            endpoint: "192.168.1.10:9000".to_string(),
+            target_bucket: "target-bucket".to_string(),
+            region: "us-east-1".to_string(),
+            credentials: Some(Credentials {
+                access_key: "access".to_string(),
+                secret_key: "secret".to_string(),
+                session_token: Some(session_token.to_string()),
+                expiration: None,
+            }),
+            ..Default::default()
+        };
+
+        sys.update_all_targets(
+            "bucket",
+            Some(&BucketTargets {
+                targets: vec![target("old-session-token")],
+            }),
+        )
+        .await;
+        let old_client = sys
+            .get_remote_target_client("bucket", "arn:rotating")
+            .await
+            .expect("initial client should be published");
+
+        sys.update_all_targets(
+            "bucket",
+            Some(&BucketTargets {
+                targets: vec![target("new-session-token")],
+            }),
+        )
+        .await;
+        let new_client = sys
+            .get_remote_target_client("bucket", "arn:rotating")
+            .await
+            .expect("rotated client should be published");
+
+        assert!(!Arc::ptr_eq(&old_client, &new_client));
+        assert_eq!(
+            old_client.credentials.as_ref().and_then(Credentials::effective_session_token),
+            Some("old-session-token")
+        );
+        assert_eq!(
+            new_client.credentials.as_ref().and_then(Credentials::effective_session_token),
+            Some("new-session-token")
+        );
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn target_updates_serialize_client_build_through_publication_per_bucket() {
         let sys = Arc::new(BucketTargetSys::default());

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -490,7 +490,7 @@ impl ExpiryStats {
     }
 
     fn add_nonnegative(counter: &AtomicI64, delta: i64) {
-        let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(delta).max(0)));
+        let _ = counter.try_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(delta).max(0)));
     }
 
     fn increment_missed_expiry_tasks(&self) {

crates/ecstore/src/bucket/replication/mod.rs

@@ -44,14 +44,14 @@ mod replication_versioning_boundary;
 mod runtime_boundary;
 
 pub use replication_config_boundary::{
-    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
-    REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
-    REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
-    assign_site_replication_rule_priorities, invalid_replication_config_status_field, is_site_replication_role,
-    is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
-    replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
-    site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
-    validate_replication_config_target_arns,
+    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS,
+    REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION,
+    REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError,
+    ReplicationConfigurationExt, ReplicationTargetValidationError, assign_site_replication_rule_priorities,
+    invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule,
+    merge_incoming_replication_config, merge_user_replication_config, replication_target_arn_deployment_id,
+    replication_target_arns, should_remove_replication_target, site_replication_rule_deployment_id,
+    unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
 };
 pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
 pub use replication_filemeta_boundary::{

crates/ecstore/src/bucket/replication/replication_config_boundary.rs

@@ -13,12 +13,12 @@
 // limitations under the License.
 
 pub use rustfs_replication::{
-    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
-    REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
-    REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt,
-    ReplicationTargetValidationError, assign_site_replication_rule_priorities, invalid_replication_config_status_field,
-    is_site_replication_role, is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
-    replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
-    site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
-    validate_replication_config_target_arns,
+    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS,
+    REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION,
+    REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError,
+    ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError, assign_site_replication_rule_priorities,
+    invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule,
+    merge_incoming_replication_config, merge_user_replication_config, replication_target_arn_deployment_id,
+    replication_target_arns, should_remove_replication_target, site_replication_rule_deployment_id,
+    unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
 };

crates/ecstore/src/bucket/replication/replication_object_config.rs

@@ -436,16 +436,21 @@ pub(crate) async fn check_replicate_delete_strict(
     }
 
     for target in decision.targets_map.values_mut() {
-        if let Some(client) = ReplicationTargetStore::remote_target_client(bucket, &target.arn).await {
-            target.synchronous = client.replicate_sync;
-        } else {
-            target.replicate = false;
-            target.synchronous = false;
-        }
+        let replicate_sync = ReplicationTargetStore::remote_target_client(bucket, &target.arn)
+            .await
+            .map(|client| client.replicate_sync);
+        apply_target_delivery_mode(target, replicate_sync);
     }
     Ok(decision)
 }
 
+fn apply_target_delivery_mode(target: &mut ReplicateTargetDecision, replicate_sync: Option<bool>) {
+    // A missing runtime client is a delivery failure, not a rule mismatch.
+    // Preserve admission and fall back to the asynchronous worker, which can
+    // persist FAILED state for the heal/retry path.
+    target.synchronous = replicate_sync.unwrap_or(false);
+}
+
 pub(crate) fn check_replicate_delete_with_snapshot(
     dobj: &ObjectToDelete,
     oi: &ObjectInfo,
@@ -629,6 +634,23 @@ mod tests {
         }));
     }
 
+    #[test]
+    fn missing_target_client_preserves_delete_admission_as_async() {
+        let mut target = ReplicateTargetDecision::new("arn:target".to_string(), true, true);
+
+        apply_target_delivery_mode(&mut target, None);
+
+        assert!(target.replicate, "a runtime client miss must not erase the replication rule decision");
+        assert!(
+            !target.synchronous,
+            "unavailable synchronous targets must fall back to the async retry path"
+        );
+
+        apply_target_delivery_mode(&mut target, Some(true));
+        assert!(target.replicate);
+        assert!(target.synchronous);
+    }
+
     #[test]
     fn must_replicate_options_preserve_request_flag() {
         let user_defined = HashMap::new();

crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs

@@ -19,9 +19,9 @@ pub use rustfs_replication::{
 };
 pub(crate) use rustfs_replication::{
     ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, delete_marker_purge_mrf_entry,
-    delete_marker_purge_version_id, delete_replication_missing_source_decision, delete_replication_object_opts,
-    heal_uses_delete_replication_path, is_retryable_delete_replication_head_error, is_version_delete_replication,
-    replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
+    delete_marker_purge_version_id, delete_replication_creates_marker, delete_replication_missing_source_decision,
+    delete_replication_object_opts, heal_uses_delete_replication_path, is_retryable_delete_replication_head_error,
+    is_version_delete_replication, replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
     replication_multipart_part_plan, resync_existing_delete_replication_info, resync_target_for_object,
     should_retry_delete_marker_purge, target_delete_version_id,
 };

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -1048,7 +1048,6 @@ pub fn resync_start_conflict_id(error: &EcstoreError) -> Option<&str> {
 }
 
 /// Main replication pool structure
-#[derive(Debug)]
 pub struct ReplicationPool<S: ReplicationStorage> {
     // Atomic counters for active workers
     active_workers: Arc<AtomicI32>,
@@ -1094,6 +1093,16 @@ pub struct ReplicationPool<S: ReplicationStorage> {
     resyncer: Arc<ReplicationResyncer>,
 }
 
+impl<S: ReplicationStorage> std::fmt::Debug for ReplicationPool<S> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ReplicationPool")
+            .field("active_workers", &self.active_workers.load(Ordering::Relaxed))
+            .field("active_lrg_workers", &self.active_lrg_workers.load(Ordering::Relaxed))
+            .field("active_mrf_workers", &self.active_mrf_workers.load(Ordering::Relaxed))
+            .finish_non_exhaustive()
+    }
+}
+
 impl<S: ReplicationStorage> ReplicationPool<S> {
     /// Creates a new replication pool with specified options
     pub async fn new(opts: ReplicationPoolOpts, stats: Arc<ReplicationStats>, storage: Arc<S>) -> Arc<Self> {
@@ -2132,7 +2141,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
     }
 
     /// Load bucket replication resync statuses into memory
-    #[instrument(skip(_cancellation_token))]
+    #[instrument(skip(self, buckets, _cancellation_token), fields(bucket_count = buckets.len()))]
     async fn load_resync(
         self: Arc<Self>,
         buckets: &[String],

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -30,8 +30,8 @@ use super::replication_msgp_boundary::ReplicationMsgpCodec;
 use super::replication_object_config::{ReplicationConfig, get_replication_config, must_replicate};
 use super::replication_object_decision_boundary::{
     MustReplicateOptions, ReplicationMultipartPartInput, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
-    heal_uses_delete_replication_path, is_retryable_delete_replication_head_error, is_version_delete_replication,
-    replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
+    delete_replication_creates_marker, heal_uses_delete_replication_path, is_retryable_delete_replication_head_error,
+    is_version_delete_replication, replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
     replication_multipart_part_plan, resync_existing_delete_replication_info, should_retry_delete_marker_purge,
     target_delete_version_id,
 };
@@ -54,11 +54,12 @@ use super::replication_storage_boundary::{
 };
 use super::replication_target_boundary::{
     ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions,
-    ReplicationTargetStore, SsecPassthroughCapability, SsecPassthroughGate, TargetClient, is_replication_target_offline_error,
-    replication_action_for_target_head, replication_complete_multipart_options, replication_delete_marker_purge_remove_options,
-    replication_delete_remove_options, replication_force_delete_remove_options, replication_object_is_ssec_encrypted,
-    replication_put_object_header_size, replication_put_object_options, replication_target_head_is_newer_null_version,
-    resolve_read_api_version_id, ssec_passthrough_evidence_present, ssec_passthrough_gate, version_identity_drifted,
+    ReplicationTargetStore, S3ClientError, SsecPassthroughCapability, SsecPassthroughGate, TargetClient,
+    is_replication_target_offline_error, replication_action_for_target_head, replication_complete_multipart_options,
+    replication_delete_marker_purge_remove_options, replication_delete_remove_options, replication_force_delete_remove_options,
+    replication_object_is_ssec_encrypted, replication_put_object_header_size, replication_put_object_options,
+    replication_target_head_is_newer_null_version, resolve_read_api_version_id, ssec_passthrough_evidence_present,
+    ssec_passthrough_gate, version_identity_drifted,
 };
 use super::replication_versioning_boundary::ReplicationVersioningStore;
 use super::runtime_boundary as runtime_sources;
@@ -101,6 +102,7 @@ const BACKGROUND_WALKDIR_TIMEOUT: TokioDuration = TokioDuration::from_secs(60);
 const ENV_REPL_RESYNC_MAX_JOBS: &str = "RUSTFS_REPL_RESYNC_MAX_JOBS";
 const DEFAULT_REPL_RESYNC_MAX_JOBS: usize = 2;
 const MAX_REPL_RESYNC_MAX_JOBS: usize = 32;
+const TARGET_CLIENT_UNAVAILABLE_ERROR: &str = "replication target client is unavailable";
 use uuid::Uuid;
 
 const EVENT_RESYNC_STATUS_UPDATE_SKIPPED: &str = "replication_resync_status_update_skipped";
@@ -1617,7 +1619,6 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
     }
 
     let bucket = dobj.bucket.clone();
-    let mut source_state_verified = true;
     let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
         Some(version_id.to_owned())
     } else {
@@ -1675,7 +1676,12 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
                 return purge_stale_delete_marker_targets(&bucket, &dobj).await;
             }
             Err(err) => {
-                source_state_verified = false;
+                // A transient source error (lock timeout, IO error) must not
+                // fall through to the marker-creation send below: that DELETE
+                // omits the versionId, so every such retry lets a generic S3
+                // target mint one more delete marker (rustfs#6823). Fail the
+                // entry without touching the target; the MRF replay / heal
+                // scanner retries once the source is readable again.
                 debug!(
                     event = EVENT_REPLICATION_DELETE_SKIPPED,
                     component = LOG_COMPONENT_ECSTORE,
@@ -1687,6 +1693,20 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
                     reason = "source_state_verification_failed",
                     "Failed to verify source delete-marker state before replication"
                 );
+                send_local_event(EventArgs {
+                    event_name: EventName::ObjectReplicationNotTracked.to_string(),
+                    bucket_name: bucket.clone(),
+                    object: ObjectInfo {
+                        bucket: bucket.clone(),
+                        name: dobj.delete_object.object_name.clone(),
+                        version_id,
+                        delete_marker: dobj.delete_object.delete_marker,
+                        ..Default::default()
+                    },
+                    user_agent: "Internal: [Replication]".to_string(),
+                    ..Default::default()
+                });
+                return false;
             }
         }
     }
@@ -1828,19 +1848,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
                 reason = "target_client_missing",
                 "Skipping replication delete because target client is unavailable"
             );
-            send_local_event(EventArgs {
-                event_name: EventName::ObjectReplicationNotTracked.to_string(),
-                bucket_name: bucket.clone(),
-                object: ObjectInfo {
-                    bucket: bucket.clone(),
-                    name: dobj.delete_object.object_name.clone(),
-                    version_id,
-                    delete_marker: dobj.delete_object.delete_marker,
-                    ..Default::default()
-                },
-                user_agent: "Internal: [Replication]".to_string(),
-                ..Default::default()
-            });
+            rinfos.targets.push(unavailable_delete_target_info(&dobj, &tgt_entry.arn));
             continue;
         };
 
@@ -2008,7 +2016,9 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
         expected_targets,
         rinfos.targets.len(),
         state_persisted,
-        source_state_verified,
+        // Source state is verified by construction here: a verification
+        // error returns early above instead of replicating unverified.
+        true,
         &replication_status,
     )
 }
@@ -2563,6 +2573,32 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
     all_succeeded
 }
 
+fn unavailable_delete_target_info(dobj: &DeletedObjectReplicationInfo, arn: &str) -> ReplicatedTargetInfo {
+    let mut rinfo = dobj
+        .delete_object
+        .replication_state
+        .as_ref()
+        .map(|state| state.target_state(arn))
+        .unwrap_or_else(|| ReplicatedTargetInfo {
+            arn: arn.to_string(),
+            ..Default::default()
+        });
+    rinfo.op_type = dobj.op_type;
+    if is_version_delete_replication(&dobj.delete_object) {
+        if rinfo.version_purge_status != VersionPurgeStatusType::Complete {
+            rinfo.version_purge_status = VersionPurgeStatusType::Failed;
+            rinfo.error = Some(TARGET_CLIENT_UNAVAILABLE_ERROR.to_string());
+        }
+    } else if rinfo.prev_replication_status == ReplicationStatusType::Completed && dobj.op_type != ReplicationType::ExistingObject
+    {
+        rinfo.replication_status = ReplicationStatusType::Completed;
+    } else {
+        rinfo.replication_status = ReplicationStatusType::Failed;
+        rinfo.error = Some(TARGET_CLIENT_UNAVAILABLE_ERROR.to_string());
+    }
+    rinfo
+}
+
 async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
     let version_id = if let Some(version_id) = &dobj.delete_object.delete_marker_version_id {
         version_id.to_owned()
@@ -2637,7 +2673,14 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
             &tgt_client.bucket,
             &dobj.delete_object.object_name,
             version_id.clone(),
-            replication_delete_remove_options(dobj.delete_object.delete_marker, dobj.delete_object.delete_marker_mtime),
+            // A version purge must keep the versionId on the DELETE even when
+            // the purged version is a delete marker: marker-creation semantics
+            // would drop it and a generic S3 target would mint a fresh marker
+            // on every retry (rustfs#6823).
+            replication_delete_remove_options(
+                delete_replication_creates_marker(&dobj.delete_object),
+                dobj.delete_object.delete_marker_mtime,
+            ),
         )
         .await
     {
@@ -2768,6 +2811,10 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
     };
 
     let mut join_set = JoinSet::new();
+    let mut rinfos = ReplicatedInfos {
+        replication_timestamp: Some(OffsetDateTime::now_utc()),
+        targets: Vec::with_capacity(tgt_arns.len()),
+    };
 
     for arn in tgt_arns {
         let Some(tgt_client) = ReplicationTargetStore::remote_target_client(&bucket, &arn).await else {
@@ -2775,7 +2822,8 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
             // stays unreachable would flood the log from the replication hot path. The
             // condition is reported once per pass by the site-replication reconciler and
             // once per rebuild by `update_all_targets`, which is where an operator can act
-            // on it; the per-object event below still records each dropped object.
+            // on it; the FAILED state below preserves retry visibility and the
+            // aggregate result emits the user-visible failure event once.
             debug!(
                 event = EVENT_RESYNC_RUNTIME_SKIPPED,
                 component = LOG_COMPONENT_ECSTORE,
@@ -2784,15 +2832,9 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
                 object = %object,
                 arn = %arn,
                 reason = "target_client_missing",
-                "Replication rule has no bucket target for its destination ARN; object not replicated"
+                "Replication target client unavailable"
             );
-            send_local_event(EventArgs {
-                event_name: EventName::ObjectReplicationNotTracked.to_string(),
-                bucket_name: bucket.clone(),
-                object: roi.to_object_info(),
-                user_agent: "Internal: [Replication]".to_string(),
-                ..Default::default()
-            });
+            rinfos.targets.push(unavailable_object_target_info(&roi, &arn));
             continue;
         };
 
@@ -2807,11 +2849,6 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
         });
     }
 
-    let mut rinfos = ReplicatedInfos {
-        replication_timestamp: Some(OffsetDateTime::now_utc()),
-        targets: Vec::with_capacity(join_set.len()),
-    };
-
     while let Some(result) = join_set.join_next().await {
         match result {
             Ok(tgt_info) => {
@@ -2917,6 +2954,23 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
     (merged_state, state_persisted)
 }
 
+fn unavailable_object_target_info(roi: &ReplicateObjectInfo, arn: &str) -> ReplicatedTargetInfo {
+    ReplicatedTargetInfo {
+        arn: arn.to_string(),
+        size: roi.actual_size,
+        replication_action: if roi.op_type == ReplicationType::Object {
+            ReplicationAction::All
+        } else {
+            ReplicationAction::Metadata
+        },
+        op_type: roi.op_type,
+        replication_status: ReplicationStatusType::Failed,
+        prev_replication_status: roi.target_replication_status(arn),
+        error: Some(TARGET_CLIENT_UNAVAILABLE_ERROR.to_string()),
+        ..Default::default()
+    }
+}
+
 trait ReplicateObjectInfoExt {
     async fn replicate_object<S: ReplicationObjectIO>(
         &self,
@@ -2951,11 +3005,14 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
         };
 
         if ReplicationTargetStore::target_is_offline(&tgt_client).await {
-            debug!(
+            // The object is reported FAILED here, so this must be as loud as a
+            // per-object put_object failure or the key never reaches the logs.
+            warn!(
                 event = EVENT_RESYNC_RUNTIME_SKIPPED,
                 component = LOG_COMPONENT_ECSTORE,
                 subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
                 bucket = %bucket,
+                object = %object,
                 arn = %tgt_client.arn,
                 reason = "target_offline",
                 endpoint = %tgt_client.to_url(),
@@ -3012,6 +3069,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                         component = LOG_COMPONENT_ECSTORE,
                         subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
                         bucket = %bucket,
+                        object = %object,
                         arn = %tgt_client.arn,
                         error = %e,
                         reason = "object_reader_unavailable",
@@ -3043,6 +3101,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                     component = LOG_COMPONENT_ECSTORE,
                     subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
                     bucket = %bucket,
+                    object = %object,
                     arn = %tgt_client.arn,
                     error = %e,
                     reason = "actual_size_unavailable",
@@ -3068,6 +3127,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                 component = LOG_COMPONENT_ECSTORE,
                 subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
                 bucket = %bucket,
+                object = %object,
                 arn = %tgt_client.arn,
                 reason = "target_bucket_empty",
                 "Skipping replication object target"
@@ -3130,6 +3190,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
                                 bucket = %bucket,
+                                object = %object,
                                 arn = %tgt_client.arn,
                                 operation = "head_object_fallback",
                                 error = %e2,
@@ -3145,6 +3206,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                         component = LOG_COMPONENT_ECSTORE,
                         subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
                         bucket = %bucket,
+                        object = %object,
                         arn = %tgt_client.arn,
                         operation = "head_object",
                         error = %e,
@@ -3174,6 +3236,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                     component = LOG_COMPONENT_ECSTORE,
                     subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
                     bucket = %bucket,
+                    object = %object,
                     arn = %tgt_client.arn,
                     operation = "build_put_options",
                     error = %e,
@@ -3443,11 +3506,14 @@ fn replicate_all_target_info(roi: &ReplicateObjectInfo, tgt_client: &TargetClien
 
 /// Log and notify that replication was skipped because the target is offline.
 fn note_replicate_all_target_offline(roi: &ReplicateObjectInfo, bucket: &str, tgt_client: &TargetClient) {
-    debug!(
+    // The object is reported FAILED here, so this must be as loud as a
+    // per-object put_object failure or the key never reaches the logs.
+    warn!(
         event = EVENT_RESYNC_RUNTIME_SKIPPED,
         component = LOG_COMPONENT_ECSTORE,
         subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
         bucket = %bucket,
+        object = %roi.name,
         arn = %tgt_client.arn,
         target = %tgt_client.to_url(),
         reason = "target_offline",
@@ -3485,6 +3551,7 @@ fn note_replicate_all_reader_unavailable(roi: &ReplicateObjectInfo, bucket: &str
             component = LOG_COMPONENT_ECSTORE,
             subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
             bucket = %bucket,
+            object = %roi.name,
             arn = %tgt_client.arn,
             error = %e,
             reason = "object_reader_unavailable",
@@ -3508,6 +3575,7 @@ fn note_replicate_all_size_unavailable(bucket: &str, tgt_client: &TargetClient,
         component = LOG_COMPONENT_ECSTORE,
         subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
         bucket = %bucket,
+        object = %object_info.name,
         arn = %tgt_client.arn,
         error = %e,
         reason = "actual_size_unavailable",
@@ -3530,6 +3598,7 @@ fn note_replicate_all_target_bucket_empty(bucket: &str, tgt_client: &TargetClien
         component = LOG_COMPONENT_ECSTORE,
         subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
         bucket = %bucket,
+        object = %object_info.name,
         arn = %tgt_client.arn,
         reason = "target_bucket_empty",
         "Skipped replication because target bucket is empty"
@@ -3738,6 +3807,7 @@ async fn resolve_replicate_all_action(
                             component = LOG_COMPONENT_ECSTORE,
                             subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
                             bucket = %bucket,
+                            object = %object,
                             arn = %tgt_client.arn,
                             error = %e2,
                             reason = "head_object_fallback_failed",
@@ -3768,6 +3838,7 @@ async fn resolve_replicate_all_action(
                     component = LOG_COMPONENT_ECSTORE,
                     subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
                     bucket = %bucket,
+                    object = %object,
                     arn = %tgt_client.arn,
                     error = %e,
                     reason = "head_object_failed",
@@ -3811,6 +3882,7 @@ fn fail_replicate_all_put_options(
         component = LOG_COMPONENT_ECSTORE,
         subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
         bucket = %bucket,
+        object = %object_info.name,
         arn = %tgt_client.arn,
         operation = "build_put_options",
         error = %e,
@@ -3935,20 +4007,13 @@ struct MultipartReplicationContext<'a, S: ReplicationObjectIO> {
 }
 
 async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartReplicationContext<'_, S>) -> std::io::Result<()> {
-    let MultipartReplicationContext {
-        storage,
-        cli,
-        src_bucket,
-        dst_bucket,
-        object,
-        object_info,
-        obj_opts,
-        arn,
-        put_opts,
-    } = ctx;
     let mut attempts = 1;
     let upload_id = loop {
-        match cli.create_multipart_upload(dst_bucket, object, &put_opts).await {
+        match ctx
+            .cli
+            .create_multipart_upload(ctx.dst_bucket, ctx.object, &ctx.put_opts)
+            .await
+        {
             Ok(id) => {
                 break id;
             }
@@ -3965,6 +4030,71 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
         }
     };
 
+    let cli = ctx.cli.clone();
+    let dst_bucket = ctx.dst_bucket;
+    let object = ctx.object;
+    let arn = ctx.arn;
+
+    let result = replicate_multipart_parts_and_complete(ctx, &upload_id).await;
+    abort_multipart_on_failure(result, dst_bucket, object, &upload_id, arn, || async {
+        cli.abort_multipart_upload(dst_bucket, object, &upload_id).await
+    })
+    .await
+}
+
+/// Best-effort abort of the target-side multipart upload once the transfer has
+/// failed past CreateMultipartUpload; without it every failed attempt leaves an
+/// invisible incomplete upload on the target that keeps billing for its parts.
+/// The abort outcome never replaces the transfer error: an abort failure is
+/// only logged and `result` is returned as-is.
+async fn abort_multipart_on_failure<F, Fut>(
+    result: std::io::Result<()>,
+    dst_bucket: &str,
+    object: &str,
+    upload_id: &str,
+    arn: &str,
+    abort: F,
+) -> std::io::Result<()>
+where
+    F: FnOnce() -> Fut,
+    Fut: std::future::Future<Output = std::result::Result<(), S3ClientError>>,
+{
+    if result.is_ok() {
+        return result;
+    }
+    if let Err(abort_err) = abort().await {
+        warn!(
+            event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+            target_bucket = %dst_bucket,
+            object = %object,
+            arn = %arn,
+            upload_id = %upload_id,
+            operation = "abort_multipart_upload",
+            error = %abort_err,
+            "Replication target operation failed"
+        );
+    }
+    result
+}
+
+async fn replicate_multipart_parts_and_complete<S: ReplicationObjectIO>(
+    ctx: MultipartReplicationContext<'_, S>,
+    upload_id: &str,
+) -> std::io::Result<()> {
+    let MultipartReplicationContext {
+        storage,
+        cli,
+        src_bucket,
+        dst_bucket,
+        object,
+        object_info,
+        obj_opts,
+        arn,
+        put_opts,
+    } = ctx;
+
     let mut uploaded_parts: Vec<CompletedPart> = Vec::new();
 
     let mut header_size = replication_put_object_header_size(&put_opts);
@@ -4003,7 +4133,7 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
             .put_object_part(
                 dst_bucket,
                 object,
-                &upload_id,
+                upload_id,
                 part_plan.part_number,
                 part_plan.part_size,
                 byte_stream,
@@ -4028,7 +4158,7 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
         .complete_multipart_upload(
             dst_bucket,
             object,
-            &upload_id,
+            upload_id,
             uploaded_parts,
             &replication_complete_multipart_options(
                 actual_size,
@@ -4053,6 +4183,88 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
 #[cfg(test)]
 mod tests {
     use super::super::replication_filemeta_boundary::ReplicateTargetDecision;
+
+    #[test]
+    fn unavailable_object_target_is_persisted_as_failed() {
+        let arn = "arn:object-target";
+        let roi = ReplicateObjectInfo {
+            actual_size: 42,
+            op_type: ReplicationType::Object,
+            replication_status_internal: Some(format!("{arn}=PENDING;")),
+            ..Default::default()
+        };
+
+        let target_info = unavailable_object_target_info(&roi, arn);
+        let merged = get_replication_state(
+            &ReplicatedInfos {
+                replication_timestamp: Some(OffsetDateTime::now_utc()),
+                targets: vec![target_info.clone()],
+            },
+            &ReplicationState::default(),
+            None,
+        );
+
+        assert_eq!(target_info.replication_status, ReplicationStatusType::Failed);
+        assert_eq!(target_info.prev_replication_status, ReplicationStatusType::Pending);
+        assert_eq!(target_info.replication_action, ReplicationAction::All);
+        assert_eq!(target_info.error.as_deref(), Some(TARGET_CLIENT_UNAVAILABLE_ERROR));
+        assert_eq!(merged.targets.get(arn), Some(&ReplicationStatusType::Failed));
+    }
+
+    #[test]
+    fn unavailable_delete_target_is_failed_without_overwriting_completed_state() {
+        let arn = "arn:delete-target";
+        let mut previous_state = ReplicationState::default();
+        previous_state.targets.insert(arn.to_string(), ReplicationStatusType::Pending);
+        let mut dobj = DeletedObjectReplicationInfo {
+            delete_object: ReplicationDeletedObject {
+                delete_marker: true,
+                replication_state: Some(previous_state),
+                ..Default::default()
+            },
+            op_type: ReplicationType::Delete,
+            ..Default::default()
+        };
+
+        let failed = unavailable_delete_target_info(&dobj, arn);
+        assert_eq!(failed.replication_status, ReplicationStatusType::Failed);
+        assert_eq!(failed.prev_replication_status, ReplicationStatusType::Pending);
+        assert_eq!(failed.error.as_deref(), Some(TARGET_CLIENT_UNAVAILABLE_ERROR));
+
+        dobj.delete_object
+            .replication_state
+            .as_mut()
+            .expect("previous state should exist")
+            .targets
+            .insert(arn.to_string(), ReplicationStatusType::Completed);
+        let completed = unavailable_delete_target_info(&dobj, arn);
+        assert_eq!(completed.replication_status, ReplicationStatusType::Completed);
+        assert!(completed.error.is_none());
+    }
+
+    #[test]
+    fn unavailable_version_purge_target_is_persisted_as_failed() {
+        let arn = "arn:purge-target";
+        let mut previous_state = ReplicationState::default();
+        previous_state
+            .purge_targets
+            .insert(arn.to_string(), VersionPurgeStatusType::Pending);
+        let dobj = DeletedObjectReplicationInfo {
+            delete_object: ReplicationDeletedObject {
+                version_id: Some(Uuid::new_v4()),
+                replication_state: Some(previous_state),
+                ..Default::default()
+            },
+            op_type: ReplicationType::Delete,
+            ..Default::default()
+        };
+
+        let target_info = unavailable_delete_target_info(&dobj, arn);
+
+        assert_eq!(target_info.version_purge_status, VersionPurgeStatusType::Failed);
+        assert_eq!(target_info.error.as_deref(), Some(TARGET_CLIENT_UNAVAILABLE_ERROR));
+    }
+
     fn resync_target_state(resync_id: &str, status: ResyncStatusType, replicated_count: i64) -> TargetReplicationResyncStatus {
         TargetReplicationResyncStatus {
             resync_id: resync_id.to_string(),
@@ -5141,4 +5353,42 @@ mod tests {
         assert!(resync_state_accepts_update(&current, &matching));
         assert!(!resync_state_accepts_update(&current, &stale));
     }
+
+    #[tokio::test]
+    async fn abort_multipart_on_failure_skips_abort_when_transfer_succeeded() {
+        let aborted = Arc::new(AtomicBool::new(false));
+        let flag = aborted.clone();
+
+        let result = abort_multipart_on_failure(Ok(()), "dst-bucket", "obj", "upload-1", "arn:dest", move || async move {
+            flag.store(true, Ordering::SeqCst);
+            Ok(())
+        })
+        .await;
+
+        assert!(result.is_ok());
+        assert!(!aborted.load(Ordering::SeqCst));
+    }
+
+    #[tokio::test]
+    async fn abort_multipart_on_failure_aborts_and_keeps_transfer_error() {
+        let aborted = Arc::new(AtomicBool::new(false));
+        let flag = aborted.clone();
+
+        // The abort itself failing must not mask the transfer error.
+        let result = abort_multipart_on_failure(
+            Err(std::io::Error::other("transfer failed")),
+            "dst-bucket",
+            "obj",
+            "upload-1",
+            "arn:dest",
+            move || async move {
+                flag.store(true, Ordering::SeqCst);
+                Err(S3ClientError::new("abort failed"))
+            },
+        )
+        .await;
+
+        assert!(aborted.load(Ordering::SeqCst));
+        assert_eq!(result.unwrap_err().to_string(), "transfer failed");
+    }
 }

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -36,8 +36,8 @@ use time::OffsetDateTime;
 use time::format_description::well_known::Rfc3339;
 
 pub(crate) use crate::bucket::bucket_target_sys::{
-    AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
-    resolve_read_api_version_id,
+    AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, S3ClientError,
+    TargetClient, resolve_read_api_version_id,
 };
 #[cfg(test)]
 pub(crate) use crate::bucket::target::BucketTarget;

crates/ecstore/src/bucket/target/bucket_target.rs

@@ -25,6 +25,8 @@ use time::OffsetDateTime;
 use url::Url;
 
 const REDACTED_CREDENTIAL: &str = "<redacted>";
+const GO_YEAR_ONE_START_UNIX_SECONDS: i64 = -62_135_596_800;
+const GO_YEAR_TWO_START_UNIX_SECONDS: i64 = -62_104_060_800;
 
 #[derive(Deserialize, Serialize, Default, Clone)]
 pub struct Credentials {
@@ -41,6 +43,26 @@ pub struct Credentials {
 }
 
 impl Credentials {
+    /// Returns the session token used for request signing.
+    ///
+    /// MinIO-compatible payloads may carry an empty token. Treat whitespace-only
+    /// values as absent without rewriting a real token, whose bytes are opaque.
+    pub fn effective_session_token(&self) -> Option<&str> {
+        self.session_token.as_deref().filter(|token| !token.trim().is_empty())
+    }
+
+    /// Returns the credential expiry after normalizing Go's zero `time.Time`.
+    ///
+    /// Go JSON encoders emit year 1 for an unset `time.Time`; persisted MinIO
+    /// target metadata can therefore contain that sentinel even for static
+    /// credentials.
+    pub fn effective_expiration(&self) -> Option<Timestamp> {
+        self.expiration.filter(|expiration| {
+            let unix_seconds = expiration.as_second();
+            !(GO_YEAR_ONE_START_UNIX_SECONDS..GO_YEAR_TWO_START_UNIX_SECONDS).contains(&unix_seconds)
+        })
+    }
+
     pub fn redacted(&self) -> Self {
         Self {
             access_key: self.access_key.clone(),
@@ -355,6 +377,24 @@ mod tests {
     use std::time::Duration;
     use time::OffsetDateTime;
 
+    #[test]
+    fn credential_effective_values_normalize_only_compatibility_sentinels() {
+        let mut credentials = Credentials {
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            session_token: Some("  ".to_string()),
+            expiration: Some("0001-01-01T08:00:00+08:00".parse().expect("Go zero time should parse")),
+        };
+
+        assert!(credentials.effective_session_token().is_none());
+        assert!(credentials.effective_expiration().is_none());
+
+        credentials.session_token = Some(" opaque token ".to_string());
+        credentials.expiration = Some("2099-01-01T00:00:00Z".parse().expect("future timestamp should parse"));
+        assert_eq!(credentials.effective_session_token(), Some(" opaque token "));
+        assert_eq!(credentials.effective_expiration(), credentials.expiration);
+    }
+
     #[test]
     fn test_bucket_target_json_deserialize() {
         let json = r#"

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -122,6 +122,14 @@ fn control_plane_failure(op: &str, bucket: Option<&str>, error_code: Option<i32>
     if error_code == Some(rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32) {
         return Error::RemoteNotInitialized;
     }
+    if error_code == Some(rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode::ControlPlaneErrorInvalidArgument as i32)
+    {
+        return Error::InvalidArgument(
+            "control-plane".to_string(),
+            op.to_string(),
+            error_info.unwrap_or_else(|| format!("{op}: peer rejected invalid argument without details")),
+        );
+    }
     match error_info {
         Some(msg) => Error::other(msg),
         None => peer_failure_without_details(op, bucket),
@@ -725,7 +733,7 @@ impl PeerRestClient {
     /// never take it offline no matter what its message says. The substring
     /// fallback only covers failures that exist purely as text, such as the
     /// dial errors `get_client` wraps.
-    fn is_network_like_error(err: &Error) -> bool {
+    pub(crate) fn is_network_like_error(err: &Error) -> bool {
         if let Error::Io(io_err) = err
             && let Some(status) = embedded_tonic_status(io_err)
         {
@@ -2335,6 +2343,29 @@ mod tests {
         use rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode;
         assert_eq!(ControlPlaneErrorCode::ControlPlaneErrorUnspecified as i32, 0);
         assert_eq!(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32, 1);
+        assert_eq!(ControlPlaneErrorCode::ControlPlaneErrorInvalidArgument as i32, 2);
+    }
+
+    #[test]
+    fn control_plane_failure_preserves_typed_invalid_argument_reason() {
+        use rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode;
+
+        let reason = "durable unresolved-entry recovery requires pool metadata V2 or V3";
+        let err = control_plane_failure(
+            "start_decommission",
+            None,
+            Some(ControlPlaneErrorCode::ControlPlaneErrorInvalidArgument as i32),
+            Some(reason.to_string()),
+        );
+
+        assert!(
+            matches!(
+                err,
+                Error::InvalidArgument(ref scope, ref operation, ref actual_reason)
+                    if scope == "control-plane" && operation == "start_decommission" && actual_reason == reason
+            ),
+            "forwarded validation failures must remain typed and actionable"
+        );
     }
 
     #[test]

crates/ecstore/src/config/com.rs

@@ -800,11 +800,22 @@ where
             if log_error {
                 error!("save_config_with_opts: err: {:?}, file: {}", err, file);
             }
-            Err(err)
+            Err(map_system_metadata_write_error(err, file))
         }
     }
 }
 
+/// A system metadata volume outage must remain retryable instead of being
+/// exposed as the user-facing bucket-not-found response.
+pub(crate) fn map_system_metadata_write_error(err: Error, file: &str) -> Error {
+    match err {
+        Error::BucketNotFound(_) | Error::VolumeNotFound => {
+            Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), file.to_string())
+        }
+        other => other,
+    }
+}
+
 fn new_server_config() -> Config {
     Config::new()
 }
@@ -2361,6 +2372,7 @@ where
         scan_mode: HealScanMode::Deep,
         update_parity: false,
         no_lock: false,
+        read_repair: false,
         pool: None,
         set: None,
     };
@@ -2795,14 +2807,14 @@ mod tests {
     use super::{
         SERVER_CONFIG_LOCK, ServerConfigSnapshot, apply_dynamic_config_for_sub_sys_with, build_scalar_config_object,
         config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
-        heal_config_descriptor, is_standard_object_server_config, lookup_configs, new_and_save_server_config, read_config,
-        read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty, read_config_with_metadata,
-        read_config_without_migrate, read_server_config_snapshot, save_server_config, save_server_config_snapshot,
-        save_server_config_snapshot_with_generation, server_config_transaction_lock_path, should_warn_ignored_scalar_section,
-        storage_class_kvs_mut,
+        heal_config_descriptor, is_standard_object_server_config, lookup_configs, map_system_metadata_write_error,
+        new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty,
+        read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot, save_config_with_opts_inner,
+        save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation,
+        server_config_transaction_lock_path, should_warn_ignored_scalar_section, storage_class_kvs_mut,
     };
     use crate::config::{audit, heal, notify, oidc, scanner};
-    use crate::disk::endpoint::Endpoint;
+    use crate::disk::{RUSTFS_META_BUCKET, endpoint::Endpoint};
     use crate::error::{Error, Result};
     use crate::layout::endpoints::SetupType;
     use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
@@ -2834,6 +2846,72 @@ mod tests {
         assert!(rendered.contains("panicked"));
         assert!(!rendered.contains("do-not-expose-payload"));
     }
+
+    #[test]
+    fn system_metadata_volume_failures_map_to_retryable_write_errors() {
+        for error in [Error::VolumeNotFound, Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())] {
+            assert_eq!(
+                map_system_metadata_write_error(error, "buckets/example/.metadata.bin"),
+                Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), "buckets/example/.metadata.bin".to_string())
+            );
+        }
+
+        let other = Error::other("metadata encoding failed");
+        assert_eq!(map_system_metadata_write_error(other.clone(), "buckets/example/.metadata.bin"), other);
+    }
+
+    #[derive(Debug, Default)]
+    struct MetadataWriteStore {
+        error: Option<Error>,
+    }
+
+    #[async_trait::async_trait]
+    impl crate::storage_api_contracts::object::ObjectIO for MetadataWriteStore {
+        type Error = Error;
+        type RangeSpec = HTTPRangeSpec;
+        type HeaderMap = HeaderMap;
+        type ObjectOptions = ObjectOptions;
+        type ObjectInfo = ObjectInfo;
+        type GetObjectReader = GetObjectReader;
+        type PutObjectReader = PutObjReader;
+
+        async fn get_object_reader(
+            &self,
+            _bucket: &str,
+            _object: &str,
+            _range: Option<Self::RangeSpec>,
+            _headers: Self::HeaderMap,
+            _opts: &Self::ObjectOptions,
+        ) -> core::result::Result<Self::GetObjectReader, Self::Error> {
+            Err(Error::FileNotFound)
+        }
+
+        async fn put_object(
+            &self,
+            _bucket: &str,
+            _object: &str,
+            _data: &mut Self::PutObjectReader,
+            _opts: &Self::ObjectOptions,
+        ) -> core::result::Result<Self::ObjectInfo, Self::Error> {
+            Err(self.error.clone().expect("test store error should be configured"))
+        }
+    }
+
+    #[tokio::test]
+    async fn save_config_preserves_retryable_system_volume_errors() {
+        let store = Arc::new(MetadataWriteStore {
+            error: Some(Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())),
+        });
+        let error =
+            save_config_with_opts_inner(store, "buckets/example/.metadata.bin", Vec::new(), &ObjectOptions::default(), false)
+                .await
+                .expect_err("missing metadata volume must fail");
+
+        assert_eq!(
+            error,
+            Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), "buckets/example/.metadata.bin".to_string())
+        );
+    }
     use rustfs_lock::client::LockClient;
     use rustfs_lock::client::local::LocalClient;
     use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};

crates/ecstore/src/core/pools.rs

@@ -176,6 +176,34 @@ fn pool_meta_v3_writer_enabled() -> bool {
     )
 }
 
+fn ensure_decommission_ledger_persistence_supported_for(
+    version: u16,
+    v2_writer_enabled: bool,
+    v3_writer_enabled: bool,
+) -> Result<()> {
+    if matches!(version, POOL_META_VERSION | POOL_META_GENERATION_VERSION) || v2_writer_enabled || v3_writer_enabled {
+        return Ok(());
+    }
+
+    Err(Error::InvalidArgument(
+        "decommission".to_string(),
+        "pool-metadata-version".to_string(),
+        format!(
+            "durable unresolved-entry recovery requires pool metadata V2 or V3; enable both {} and {} only after every reader and writer supports V2",
+            rustfs_config::ENV_POOL_META_V2_WRITE,
+            rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
+        ),
+    ))
+}
+
+fn ensure_decommission_ledger_persistence_supported(pool_meta: &PoolMeta) -> Result<()> {
+    ensure_decommission_ledger_persistence_supported_for(
+        pool_meta.version,
+        pool_meta_v2_writer_enabled(),
+        pool_meta_v3_writer_enabled(),
+    )
+}
+
 #[derive(Clone, Debug)]
 pub struct DecommissionCanceler {
     operation: Arc<DecommissionOperation>,
@@ -2027,6 +2055,7 @@ pub(crate) async fn pause_pool_activation_after_durable_save<S>(pool: &Arc<S>, f
 #[cfg(test)]
 struct PoolActivationStartProbeState {
     kind: PoolActivationStartKind,
+    preflight_side_effect_attempted: std::sync::atomic::AtomicBool,
     attempted: std::sync::atomic::AtomicBool,
     notify: tokio::sync::Notify,
 }
@@ -2045,6 +2074,7 @@ impl PoolActivationStartProbe {
     pub(crate) fn install(kind: PoolActivationStartKind) -> Self {
         let state = Arc::new(PoolActivationStartProbeState {
             kind,
+            preflight_side_effect_attempted: std::sync::atomic::AtomicBool::new(false),
             attempted: std::sync::atomic::AtomicBool::new(false),
             notify: tokio::sync::Notify::new(),
         });
@@ -2061,6 +2091,14 @@ impl PoolActivationStartProbe {
             self.state.notify.notified().await;
         }
     }
+
+    pub(crate) fn preflight_side_effect_was_attempted(&self) -> bool {
+        self.state.preflight_side_effect_attempted.load(Ordering::Acquire)
+    }
+
+    pub(crate) fn activation_was_attempted(&self) -> bool {
+        self.state.attempted.load(Ordering::Acquire)
+    }
 }
 
 #[cfg(test)]
@@ -2090,6 +2128,21 @@ pub(crate) fn observe_pool_activation_start_attempt(kind: PoolActivationStartKin
     }
 }
 
+#[cfg(test)]
+fn observe_pool_activation_preflight_side_effect_attempt(kind: PoolActivationStartKind) {
+    let probes = POOL_ACTIVATION_START_PROBES
+        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
+        .lock()
+        .expect("pool activation start probe should not be poisoned")
+        .iter()
+        .filter(|state| state.kind == kind)
+        .cloned()
+        .collect::<Vec<_>>();
+    for state in probes {
+        state.preflight_side_effect_attempted.store(true, Ordering::Release);
+    }
+}
+
 fn rollback_decommission_pool_meta(pool_meta: &mut PoolMeta, previous_pool_meta: &PoolMeta, indices: &[usize]) {
     publish_pool_meta_updates(pool_meta, previous_pool_meta, indices);
 }
@@ -6189,6 +6242,7 @@ impl ECStore {
     ) -> Result<()> {
         {
             let mut pool_meta = self.pool_meta.write().await;
+            ensure_decommission_ledger_persistence_supported(&pool_meta)?;
             record_decommission_unresolved_entry(&mut pool_meta, idx, generation, entry)?;
         }
         self.save_current_pool_meta(&[idx])
@@ -6339,6 +6393,7 @@ impl ECStore {
         }
 
         ensure_decommission_start_pool_states(&latest_pool_meta, indices)?;
+        ensure_decommission_ledger_persistence_supported(&latest_pool_meta)?;
 
         let previous_pool_meta = latest_pool_meta.clone();
         let first_idx = indices.first().copied();
@@ -7040,10 +7095,14 @@ impl ECStore {
         save_guard.ensure_write_safe("decommission cannot be scheduled while pool metadata requires recovery")?;
         let indices = {
             let pool_meta = self.pool_meta.read().await;
-            resumable_decommission_queue_indices(&pool_meta)
+            let indices = resumable_decommission_queue_indices(&pool_meta)
                 .into_iter()
                 .filter(|idx| indices.contains(idx))
-                .collect::<Vec<_>>()
+                .collect::<Vec<_>>();
+            if !indices.is_empty() {
+                ensure_decommission_ledger_persistence_supported(&pool_meta)?;
+            }
+            indices
         };
         if indices.is_empty() {
             return Ok(Vec::new());
@@ -9299,8 +9358,11 @@ impl ECStore {
         {
             let pool_meta = self.pool_meta.read().await;
             ensure_decommission_start_pool_states(&pool_meta, &indices)?;
+            ensure_decommission_ledger_persistence_supported(&pool_meta)?;
         }
 
+        #[cfg(test)]
+        observe_pool_activation_preflight_side_effect_attempt(PoolActivationStartKind::Decommission);
         let decom_buckets = self.get_buckets_to_decommission().await?;
 
         let mut healed_buckets = HashSet::with_capacity(decom_buckets.len());
@@ -10856,10 +10918,98 @@ mod tests {
         assert!(!is_pool_activation_fleet_proof_error(&Error::ConfigNotFound));
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn decommission_v1_start_preflights_reject_before_metadata_writes() {
+        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
+        let baseline = load_pool_meta_replicas(store.pools.clone(), true)
+            .await
+            .expect("baseline pool metadata should be readable");
+        assert_eq!(baseline.meta.version, POOL_META_V1_VERSION);
+        *store.pool_meta.write().await = baseline.meta.clone();
+        let start_probe = PoolActivationStartProbe::install(PoolActivationStartKind::Decommission);
+        let err = store
+            .start_decommission(vec![0])
+            .await
+            .expect_err("the initial V1 start preflight must reject before side effects");
+        assert!(matches!(err, Error::InvalidArgument(..)));
+        assert!(
+            !start_probe.preflight_side_effect_was_attempted(),
+            "V1 rejection must precede bucket listing, healing, and metadata-bucket creation"
+        );
+        assert!(
+            !start_probe.activation_was_attempted(),
+            "V1 rejection must not enter the authoritative activation save"
+        );
+        let after_early_rejection = load_pool_meta_replicas(store.pools.clone(), true)
+            .await
+            .expect("early rejection must leave durable pool metadata readable");
+        assert_eq!(after_early_rejection.canonical, baseline.canonical);
+        assert!(
+            store
+                .pool_meta
+                .read()
+                .await
+                .pools
+                .iter()
+                .all(|pool| pool.decommission.is_none())
+        );
+        assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
+        store
+            .ensure_pool_meta_side_effects_safe("V1 start preflight")
+            .await
+            .expect("a deterministic start rejection must not latch recovery");
+        drop(start_probe);
+
+        let err = store
+            .save_current_pool_meta_for_decommission_start(
+                &[0],
+                vec![(
+                    0,
+                    PoolSpaceInfo {
+                        free: 50,
+                        total: 100,
+                        used: 50,
+                    },
+                )],
+                Vec::new(),
+            )
+            .await
+            .expect_err("the authoritative V1 start preflight must reject before saving");
+        assert!(matches!(err, Error::InvalidArgument(..)));
+
+        let after_authoritative_rejection = load_pool_meta_replicas(store.pools.clone(), true)
+            .await
+            .expect("authoritative rejection must leave durable pool metadata readable");
+        assert_eq!(after_authoritative_rejection.canonical, baseline.canonical);
+        assert!(
+            after_authoritative_rejection
+                .meta
+                .pools
+                .iter()
+                .all(|pool| pool.decommission.is_none())
+        );
+        assert!(
+            store
+                .pool_meta
+                .read()
+                .await
+                .pools
+                .iter()
+                .all(|pool| pool.decommission.is_none())
+        );
+        assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
+        store
+            .ensure_pool_meta_side_effects_safe("authoritative V1 start preflight")
+            .await
+            .expect("an authoritative capability rejection must not latch recovery");
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn decommission_activation_fence_loss_after_durable_save_blocks_publication() {
         let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
+        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
         let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
         let start_store = Arc::clone(&store);
         let start_task = tokio::spawn(async move {
@@ -10914,6 +11064,7 @@ mod tests {
     #[serial_test::serial]
     async fn decommission_activation_adopts_canonical_commit_after_replica_failure() {
         let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
+        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
         let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
         let start_store = Arc::clone(&store);
         let start_task = tokio::spawn(async move {
@@ -11226,6 +11377,35 @@ mod tests {
         assert!(pool_meta_v3_writer_enabled_for(true, true));
     }
 
+    #[test]
+    fn decommission_ledger_persistence_requires_an_observed_or_confirmed_format() {
+        for (version, v2_enabled, v3_enabled, expected) in [
+            (POOL_META_V1_VERSION, false, false, false),
+            (POOL_META_V1_VERSION, true, false, true),
+            (POOL_META_V1_VERSION, false, true, true),
+            (POOL_META_VERSION, false, false, true),
+            (super::POOL_META_GENERATION_VERSION, false, false, true),
+        ] {
+            let result = super::ensure_decommission_ledger_persistence_supported_for(version, v2_enabled, v3_enabled);
+            assert_eq!(
+                result.is_ok(),
+                expected,
+                "unexpected capability result for pool metadata version {version}"
+            );
+        }
+
+        let half_confirmed_v2 = pool_meta_v2_writer_enabled_for(true, false);
+        let half_confirmed_v3 = pool_meta_v3_writer_enabled_for(false, true);
+        let err = super::ensure_decommission_ledger_persistence_supported_for(
+            POOL_META_V1_VERSION,
+            half_confirmed_v2,
+            half_confirmed_v3,
+        )
+        .expect_err("half-enabled rollout gates must not admit decommission");
+        assert!(matches!(err, Error::InvalidArgument(..)));
+        assert!(err.to_string().contains("durable unresolved-entry recovery"));
+    }
+
     #[test]
     fn pool_meta_stale_write_rejection_metric_is_countable() {
         let recorder = metrics_util::debugging::DebuggingRecorder::new();
@@ -14494,6 +14674,113 @@ mod pools_tests {
         assert!(store.decommission_cancelers.read().await[0].is_none());
     }
 
+    #[tokio::test]
+    async fn test_v1_unresolved_ledger_rejection_keeps_live_state_and_write_gate_safe() {
+        let generation = OffsetDateTime::UNIX_EPOCH;
+        let status = decommission_test_pool_status(
+            0,
+            Some(PoolDecommissionInfo {
+                start_time: Some(generation),
+                ..Default::default()
+            }),
+        );
+        let last_update = status.last_update;
+        let store = decommission_worker_test_store(
+            PoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: vec![status],
+                ..Default::default()
+            },
+            vec![None],
+        );
+        let entry = DecommissionUnresolvedEntry {
+            bucket: "bucket-a".to_string(),
+            object: "directory/".to_string(),
+            pool_index: 0,
+            set_index: 0,
+            source_generation: generation,
+            candidate_count: 1,
+            disk_error_count: 0,
+            observed_at: generation,
+            reason: "metadata_resolution_failed".to_string(),
+        };
+
+        let err = store
+            .persist_decommission_unresolved_entry(0, generation, entry)
+            .await
+            .expect_err("V1 must reject the ledger before changing live state");
+
+        assert!(matches!(err, Error::InvalidArgument(..)));
+        let pool_meta = store.pool_meta.read().await;
+        let status = &pool_meta.pools[0];
+        assert_eq!(status.last_update, last_update);
+        assert!(
+            status
+                .decommission
+                .as_ref()
+                .expect("active decommission metadata should remain present")
+                .unresolved_entries
+                .is_empty()
+        );
+        drop(pool_meta);
+        store
+            .pool_meta_save_gate
+            .lock()
+            .await
+            .ensure_write_safe("V1 unresolved-entry preflight")
+            .expect("a deterministic capability rejection must not latch recovery");
+    }
+
+    #[tokio::test]
+    async fn test_v1_runtime_recovery_rejects_worker_but_keeps_cancel_persistable() {
+        let generation = OffsetDateTime::UNIX_EPOCH;
+        let store = decommission_worker_test_store(
+            PoolMeta {
+                version: POOL_META_V1_VERSION,
+                pools: vec![decommission_test_pool_status(
+                    0,
+                    Some(PoolDecommissionInfo {
+                        start_time: Some(generation),
+                        ..Default::default()
+                    }),
+                )],
+                ..Default::default()
+            },
+            vec![None],
+        );
+
+        let err = store
+            .reserve_decommission_routines(&CancellationToken::new(), &[0])
+            .await
+            .err()
+            .expect("V1 recovery must not install a worker that cannot persist an unresolved ledger");
+        assert!(matches!(err, Error::InvalidArgument(..)));
+        assert!(store.decommission_cancelers.read().await[0].is_none());
+
+        let save_called = Arc::new(AtomicBool::new(false));
+        store
+            .decommission_cancel_with_owner_and_save(0, None, {
+                let save_called = save_called.clone();
+                move |snapshot, _| async move {
+                    snapshot.encode_config_data_for_v2_gate(false)?;
+                    save_called.store(true, Ordering::SeqCst);
+                    Ok(())
+                }
+            })
+            .await
+            .expect("a rejected V1 recovery must remain cancelable without restart");
+
+        assert!(save_called.load(Ordering::SeqCst));
+        let pool_meta = store.pool_meta.read().await;
+        let info = pool_meta.pools[0]
+            .decommission
+            .as_ref()
+            .expect("cancel metadata should remain present");
+        assert!(info.canceled);
+        assert!(!info.failed);
+        assert!(!info.complete);
+    }
+
     #[tokio::test]
     async fn test_decommission_transition_waits_without_registered_canceler() {
         let store = decommission_worker_test_store(PoolMeta::default(), vec![None]);
@@ -17487,6 +17774,7 @@ mod pools_tests {
     #[tokio::test]
     async fn test_runtime_recovery_reserves_the_startup_resumable_queue() {
         let meta = PoolMeta {
+            version: super::POOL_META_VERSION,
             pools: vec![
                 decommission_test_pool_status(
                     0,
@@ -17544,6 +17832,7 @@ mod pools_tests {
     #[tokio::test]
     async fn test_runtime_recovery_does_not_reserve_behind_active_predecessor() {
         let meta = PoolMeta {
+            version: super::POOL_META_VERSION,
             pools: vec![
                 decommission_test_pool_status(
                     0,

crates/ecstore/src/core/sets.rs

@@ -781,7 +781,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
             .await
     }
 
-    #[tracing::instrument(skip(self))]
+    #[tracing::instrument(skip(self, opts))]
     async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
         if opts.delete_prefix && !opts.delete_prefix_object {
             self.delete_prefix(bucket, object, &opts).await?;

crates/ecstore/src/data_usage/mod.rs

@@ -422,9 +422,7 @@ async fn save_data_usage_in_backend(
     if publication_epoch != expected_publication_epoch {
         return Err(Error::other("data usage publication epoch changed before save"));
     }
-    crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
-        .await
-        .map_err(Error::other)?;
+    crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data).await?;
     drop(publication_guard);
 
     cleanup_observed_data_usage_after_authoritative_save_with_publication(store.as_ref(), &data_usage_info, Some(store.as_ref()))
@@ -578,7 +576,7 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
     guard: Option<&rustfs_lock::NamespaceLockGuard>,
 ) -> Result<(), Error> {
     ensure_bucket_namespace_guard(guard, bucket, "data usage cache cleanup")?;
-    let _ = USAGE_MEMORY_GENERATION.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(1)));
+    let _ = USAGE_MEMORY_GENERATION.try_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(1)));
     live_bucket_usage_cache().invalidate(bucket).await;
     clear_bucket_usage_memory(bucket, guard).await?;
 
@@ -641,7 +639,7 @@ where
     {
         Ok(reader) => reader,
         Err(Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::ConfigNotFound) => return Ok(None),
-        Err(err) => return Err(err),
+        Err(err) => return Err(map_data_usage_metadata_read_error(err, object)),
     };
     let revision = reader
         .object_info
@@ -656,6 +654,18 @@ where
     Ok(Some((data_usage_info, revision)))
 }
 
+/// A missing usage object is harmless during bucket creation, but a missing
+/// system metadata volume is a storage outage. Keep the latter retryable and
+/// distinguishable from the user bucket not existing.
+fn map_data_usage_metadata_read_error(err: Error, object: &str) -> Error {
+    match err {
+        Error::BucketNotFound(_) | Error::VolumeNotFound => {
+            Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), object.to_string())
+        }
+        other => other,
+    }
+}
+
 fn data_usage_contains_bucket(data_usage_info: &DataUsageInfo, bucket: &str) -> bool {
     data_usage_info.buckets_usage.contains_key(bucket) || data_usage_info.bucket_sizes.contains_key(bucket)
 }
@@ -912,7 +922,7 @@ where
             )
             .await;
         drop(publication_guard);
-        match save_result {
+        match save_result.map_err(|err| crate::config::com::map_system_metadata_write_error(err, object)) {
             Ok(_) => return Ok(()),
             Err(err) => {
                 if let Some((observed, observed_revision)) = load_data_usage_for_bucket_removal(store, object).await? {
@@ -2759,6 +2769,7 @@ mod tests {
     struct UsageCacheReadStore {
         transient_failures: Mutex<usize>,
         reads: Mutex<Vec<String>>,
+        terminal_error: Mutex<Option<Error>>,
     }
 
     impl UsageCacheReadStore {
@@ -2766,6 +2777,15 @@ mod tests {
             Self {
                 transient_failures: Mutex::new(n),
                 reads: Mutex::new(Vec::new()),
+                terminal_error: Mutex::new(None),
+            }
+        }
+
+        fn with_terminal_error(error: Error) -> Self {
+            Self {
+                transient_failures: Mutex::new(0),
+                reads: Mutex::new(Vec::new()),
+                terminal_error: Mutex::new(Some(error)),
             }
         }
 
@@ -2793,6 +2813,9 @@ mod tests {
             _opts: &Self::ObjectOptions,
         ) -> Result<Self::GetObjectReader, Self::Error> {
             self.reads.lock().await.push(object.to_string());
+            if let Some(error) = self.terminal_error.lock().await.clone() {
+                return Err(error);
+            }
             let mut remaining = self.transient_failures.lock().await;
             if *remaining > 0 {
                 *remaining -= 1;
@@ -2857,6 +2880,22 @@ mod tests {
         assert!(!is_data_usage_cache_absent(&Error::DiskNotFound));
     }
 
+    #[test]
+    fn data_usage_removal_maps_missing_system_volume_to_read_quorum() {
+        for error in [Error::VolumeNotFound, Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())] {
+            assert_eq!(
+                map_data_usage_metadata_read_error(error, "bucket-metadata/.usage.json"),
+                Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string())
+            );
+        }
+
+        let missing_object = Error::ObjectNotFound(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string());
+        assert_eq!(
+            map_data_usage_metadata_read_error(missing_object.clone(), "bucket-metadata/.usage.json"),
+            missing_object
+        );
+    }
+
     #[tokio::test]
     async fn load_data_usage_cache_treats_absence_as_an_empty_cache_without_retrying() {
         let name = "usage-cache";
@@ -2872,6 +2911,22 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn data_usage_removal_surfaces_missing_system_volume_as_read_quorum() {
+        for cause in [Error::BucketNotFound(RUSTFS_META_BUCKET.to_string()), Error::VolumeNotFound] {
+            let store = UsageCacheReadStore::with_terminal_error(cause);
+
+            let error = load_data_usage_for_bucket_removal(&store, "bucket-metadata/.usage.json")
+                .await
+                .expect_err("missing system metadata volume must not be treated as an absent usage object");
+
+            assert_eq!(
+                error,
+                Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string())
+            );
+        }
+    }
+
     #[tokio::test]
     async fn load_data_usage_cache_retries_a_transient_failure() {
         let name = "usage-cache";

crates/ecstore/src/disk/disk_store.rs

@@ -250,6 +250,40 @@ pub(crate) trait DiskStoreRenameDataExt {
         dst_volume: &str,
         dst_path: &str,
     ) -> Result<RenameDataResp>;
+
+    async fn rename_data_borrowed_with_guard(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: &FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+        external_guard: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<RenameDataResp> {
+        let _ = external_guard;
+        self.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
+            .await
+    }
+}
+
+/// Run a mutation in an owned task when a caller supplied publication guard.
+/// RPC cancellation drops only the waiter; the mutation owner keeps the guard
+/// until its operation has returned, including any detached blocking syscall.
+async fn run_owned_mutation<T, F, Fut>(external_guard: Option<Arc<dyn Send + Sync>>, operation: F) -> Result<T>
+where
+    T: Send + 'static,
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = Result<T>> + Send + 'static,
+{
+    if external_guard.is_none() {
+        return operation().await;
+    }
+    tokio::spawn(async move {
+        let _external_guard = external_guard;
+        operation().await
+    })
+    .await
+    .map_err(|_| Error::other("owned mutation task failed"))?
 }
 
 impl DiskStoreRenameDataExt for LocalDiskWrapper {
@@ -273,6 +307,49 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
         )
         .await
     }
+
+    async fn rename_data_borrowed_with_guard(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: &FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+        external_guard: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<RenameDataResp> {
+        let operation = self.clone();
+        let src_volume = src_volume.to_owned();
+        let src_path = src_path.to_owned();
+        let fi = fi.clone();
+        let dst_volume = dst_volume.to_owned();
+        let dst_path = dst_path.to_owned();
+        let timeout_duration = if external_guard.is_some() {
+            // A fenced mutation owns the publication guard until the storage
+            // operation returns. Timing out this waiter would cancel the
+            // LocalDisk future while a spawn_blocking namespace syscall could
+            // still be committing, reopening the movement window. The caller
+            // may drop its waiter; the owned task drains the mutation.
+            Duration::ZERO
+        } else {
+            get_max_timeout_duration()
+        };
+        run_owned_mutation(external_guard, move || async move {
+            operation
+                .track_disk_health_mutation(
+                    "rename_data",
+                    DiskMetricMutation::Write,
+                    || async {
+                        operation
+                            .disk
+                            .rename_data_borrowed(&src_volume, &src_path, &fi, &dst_volume, &dst_path)
+                            .await
+                    },
+                    timeout_duration,
+                )
+                .await
+        })
+        .await
+    }
 }
 
 pub fn get_drive_walkdir_timeout() -> Duration {
@@ -678,17 +755,20 @@ impl DiskOperationMetrics {
         let elapsed_nanos = u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX);
         let slot = &self.last_minute[(now_sec % 60) as usize];
         loop {
-            let version = slot.version.load(Ordering::Acquire);
+            // The successful CAS below is AcqRel, so it is the publication
+            // fence for the writer that owns this slot. The initial parity
+            // check does not need to acquire the slot payload.
+            let version = slot.version.load(Ordering::Relaxed);
             if !version.is_multiple_of(2) {
                 std::hint::spin_loop();
                 continue;
             }
             if slot
                 .version
-                .compare_exchange(version, version.wrapping_add(1), Ordering::AcqRel, Ordering::Acquire)
+                .compare_exchange(version, version.wrapping_add(1), Ordering::AcqRel, Ordering::Relaxed)
                 .is_ok()
             {
-                if slot.unix_sec.load(Ordering::Acquire) != now_sec {
+                if slot.unix_sec.load(Ordering::Relaxed) != now_sec {
                     slot.count.store(0, Ordering::Relaxed);
                     slot.acc_time.store(0, Ordering::Relaxed);
                     slot.unix_sec.store(now_sec, Ordering::Release);
@@ -704,14 +784,10 @@ impl DiskOperationMetrics {
     fn last_minute_snapshot(&self, now_sec: u64) -> TimedAction {
         let mut snapshot = TimedAction::default();
         for slot in &self.last_minute {
-            let version = slot.version.load(Ordering::Acquire);
-            if !version.is_multiple_of(2) {
+            let Some((slot_sec, count, acc_time)) = slot.snapshot() else {
                 continue;
-            }
-            let slot_sec = slot.unix_sec.load(Ordering::Acquire);
-            let count = slot.count.load(Ordering::Acquire);
-            let acc_time = slot.acc_time.load(Ordering::Acquire);
-            if slot.version.load(Ordering::Acquire) == version && slot_sec <= now_sec && now_sec.saturating_sub(slot_sec) < 60 {
+            };
+            if slot_sec <= now_sec && now_sec.saturating_sub(slot_sec) < 60 {
                 snapshot.count = snapshot.count.saturating_add(count);
                 snapshot.acc_time = snapshot.acc_time.saturating_add(acc_time);
             }
@@ -720,6 +796,23 @@ impl DiskOperationMetrics {
     }
 }
 
+impl TimedActionSlot {
+    fn snapshot(&self) -> Option<(u64, u64, u64)> {
+        let version = self.version.load(Ordering::Acquire);
+        if !version.is_multiple_of(2) {
+            return None;
+        }
+
+        // The first Acquire load publishes the payload written before the
+        // matching Release store. Relaxed payload loads are sufficient while
+        // the final Acquire version load validates that no writer intervened.
+        let slot_sec = self.unix_sec.load(Ordering::Relaxed);
+        let count = self.count.load(Ordering::Relaxed);
+        let acc_time = self.acc_time.load(Ordering::Relaxed);
+        (self.version.load(Ordering::Acquire) == version).then_some((slot_sec, count, acc_time))
+    }
+}
+
 pub(crate) struct DiskHealthWaitingGuard<'a> {
     health: &'a DiskHealthTracker,
 }
@@ -1097,6 +1190,37 @@ impl LocalDiskWrapper {
         )
     }
 
+    /// Run a delete under an owned coordinator task when a publication guard
+    /// is present. This keeps the guard alive if the RPC waiter is cancelled
+    /// while the local namespace mutation is still in progress.
+    pub(crate) async fn delete_with_publication_guard(
+        &self,
+        volume: &str,
+        path: &str,
+        options: DeleteOptions,
+        external_guard: Option<Arc<dyn Send + Sync>>,
+    ) -> Result<()> {
+        let operation = self.clone();
+        let volume = volume.to_owned();
+        let path = path.to_owned();
+        let timeout_duration = if external_guard.is_some() {
+            Duration::ZERO
+        } else {
+            get_max_timeout_duration()
+        };
+        run_owned_mutation(external_guard, move || async move {
+            operation
+                .track_disk_health_mutation(
+                    "delete",
+                    DiskMetricMutation::Delete,
+                    || async { operation.disk.delete(&volume, &path, options).await },
+                    timeout_duration,
+                )
+                .await
+        })
+        .await
+    }
+
     pub(crate) fn new_with_reconnect_state(
         disk: Arc<LocalDisk>,
         health_check: bool,
@@ -2247,6 +2371,44 @@ mod tests {
     };
     use tokio::io::AsyncWrite;
 
+    struct DropProbe(Arc<std::sync::atomic::AtomicUsize>);
+
+    impl Drop for DropProbe {
+        fn drop(&mut self) {
+            self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+        }
+    }
+
+    #[tokio::test]
+    async fn owned_mutation_keeps_publication_guard_after_waiter_cancellation() {
+        let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let guard: Arc<dyn Send + Sync> = Arc::new(DropProbe(Arc::clone(&drops)));
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
+        let (finished_tx, finished_rx) = tokio::sync::oneshot::channel();
+
+        let waiter = tokio::spawn(run_owned_mutation(Some(guard), move || async move {
+            started_tx.send(()).expect("mutation should signal start");
+            release_rx.await.expect("mutation should be released");
+            finished_tx.send(()).expect("mutation should signal completion");
+            Ok::<_, Error>(())
+        }));
+
+        started_rx.await.expect("mutation owner should start");
+        waiter.abort();
+        assert_eq!(drops.load(std::sync::atomic::Ordering::SeqCst), 0);
+
+        release_tx.send(()).expect("mutation owner should still be alive");
+        finished_rx.await.expect("mutation owner should finish");
+        tokio::time::timeout(Duration::from_secs(1), async {
+            while drops.load(std::sync::atomic::Ordering::SeqCst) == 0 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("publication guard should be released after mutation completion");
+    }
+
     struct PendingWriter;
 
     #[test]

crates/ecstore/src/disk/local.rs

@@ -3148,9 +3148,10 @@ impl LocalIoBackend for StdBackend {
                 direct_read_copy_fault_delta: MmapPageFaultDelta,
                 blocking_task_duration: StdDuration,
                 used_direct_io: bool,
-                /// The descriptor opened by THIS call (None on a cache hit), handed
-                /// back so the async caller can index it in the fd cache.
-                opened_fd: Option<Arc<std::fs::File>>,
+                /// The descriptor and size snapshot opened by THIS call (None on a
+                /// cache hit), handed back so the async caller can index it in the
+                /// fd cache.
+                opened_fd: Option<Arc<FdCacheEntry>>,
             }
 
             enum MmapCopyReadError {
@@ -3197,12 +3198,12 @@ impl LocalIoBackend for StdBackend {
                 (cache, key, gen_at_open)
             });
             #[cfg(target_os = "linux")]
-            let cached_fd: Option<Arc<std::fs::File>> = match &fd_lookup {
+            let cached_fd: Option<Arc<FdCacheEntry>> = match &fd_lookup {
                 Some((cache, key, _)) => cache.get(key).await,
                 None => None,
             };
             #[cfg(not(target_os = "linux"))]
-            let cached_fd: Option<Arc<std::fs::File>> = None;
+            let cached_fd: Option<Arc<FdCacheEntry>> = None;
 
             let blocking_wait_start = metrics_enabled.then(std::time::Instant::now);
             let read_result = tokio::task::spawn_blocking(move || {
@@ -3224,8 +3225,15 @@ impl LocalIoBackend for StdBackend {
                 // the read below is positioned (mmap offset argument / `read_exact_at`)
                 // and never depends on the descriptor's current offset. `cached_fd` being
                 // None also marks this call as a miss for the cache-insert side-channel.
-                let (file, access_check_duration) = if let Some(cached) = cached_fd.as_ref() {
-                    (cached.as_ref().try_clone().map_err(DiskError::from)?, StdDuration::ZERO)
+                // The cached length is the metadata snapshot captured at open time;
+                // all in-place/replacement writers invalidate this entry before
+                // publishing a mutation, so cache hits avoid a redundant fstat.
+                let (file, cached_len, access_check_duration) = if let Some(cached) = cached_fd.as_ref() {
+                    (
+                        cached.file.as_ref().try_clone().map_err(DiskError::from)?,
+                        Some(cached.len),
+                        StdDuration::ZERO,
+                    )
                 } else {
                     // Measure the volume access probe only — the part-path resolution
                     // above is accounted in `path_resolve_duration` (rustfs/backlog#1801).
@@ -3236,20 +3244,27 @@ impl LocalIoBackend for StdBackend {
                             .map_err(|e| DiskError::from(to_access_error(e, DiskError::VolumeAccessDenied)))?;
                     }
                     let access_check_duration = access_check_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
-                    (std::fs::File::open(&file_path).map_err(DiskError::from)?, access_check_duration)
+                    (std::fs::File::open(&file_path).map_err(DiskError::from)?, None, access_check_duration)
                 };
                 let file_open_duration = file_open_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
 
-                let metadata_lookup_start = metrics_enabled.then(StdInstant::now);
-                // On a cache hit this fstats the cached descriptor — the inode it was
-                // opened against, which invalidation keeps current for live entries. EC
-                // shards are fixed-length, so a still-cached pre-heal length is benign.
-                let meta = file.metadata().map_err(DiskError::from)?;
-                let metadata_lookup_duration = metadata_lookup_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
+                let (metadata_len, metadata_lookup_duration) = if let Some(len) = cached_len {
+                    // Reuse the open-time metadata snapshot on a cache hit. The
+                    // generation fence and mutation invalidation keep this value
+                    // tied to the inode held by `file`.
+                    (len, StdDuration::ZERO)
+                } else {
+                    let metadata_lookup_start = metrics_enabled.then(StdInstant::now);
+                    let meta = file.metadata().map_err(DiskError::from)?;
+                    let duration = metadata_lookup_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
+                    (meta.len(), duration)
+                };
 
                 let metadata_validate_start = metrics_enabled.then(StdInstant::now);
-                if meta.len() < end_offset_u64 {
-                    return Err(MmapCopyReadError::OutOfBounds { actual_size: meta.len() });
+                if metadata_len < end_offset_u64 {
+                    return Err(MmapCopyReadError::OutOfBounds {
+                        actual_size: metadata_len,
+                    });
                 }
                 let metadata_validate_duration =
                     metadata_validate_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
@@ -3395,9 +3410,14 @@ impl LocalIoBackend for StdBackend {
                 // Arc; `cached_fd.is_none()` is true exactly when this call did the open.
                 // Non-Linux has no fd cache, so skip the Arc allocation there.
                 #[cfg(target_os = "linux")]
-                let opened_fd: Option<Arc<std::fs::File>> = cached_fd.is_none().then(|| Arc::new(file));
+                let opened_fd: Option<Arc<FdCacheEntry>> = cached_fd.is_none().then(|| {
+                    Arc::new(FdCacheEntry {
+                        file: Arc::new(file),
+                        len: metadata_len,
+                    })
+                });
                 #[cfg(not(target_os = "linux"))]
-                let opened_fd: Option<Arc<std::fs::File>> = None;
+                let opened_fd: Option<Arc<FdCacheEntry>> = None;
 
                 Ok::<MmapCopyReadResult, MmapCopyReadError>(MmapCopyReadResult {
                     bytes,
@@ -3520,7 +3540,7 @@ impl LocalIoBackend for StdBackend {
                     }
                 }
             }
-            // Index the freshly opened descriptor for future cache hits
+            // Index the freshly opened descriptor and metadata snapshot for future cache hits
             // (rustfs/backlog#1801). `insert_if_fresh` refuses to cache if an
             // invalidation (heal/delete/rename) bumped the generation between the
             // open snapshot and now, so a stale pre-mutation inode is never served
@@ -3872,6 +3892,18 @@ struct FdKey {
     direct: bool,
 }
 
+/// Descriptor and immutable size snapshot retained for one cached shard inode.
+///
+/// The generation fence and explicit mutation invalidation keep the snapshot
+/// tied to the inode held by `file`, allowing cache hits to avoid a repeated
+/// metadata syscall without weakening replacement/heal semantics.
+struct FdCacheEntry {
+    /// An independently cloneable descriptor for the immutable shard inode.
+    file: Arc<std::fs::File>,
+    /// File length captured together with the descriptor.
+    len: u64,
+}
+
 /// Per-disk cache of open descriptors for io_uring reads (backlog#1145).
 ///
 /// Why this exists: `pread_uring` opened the file on the blocking pool for every
@@ -3901,7 +3933,7 @@ struct FdKey {
 /// the descriptor once no in-flight read still holds it.
 #[cfg(target_os = "linux")]
 struct FdCache {
-    cache: moka::future::Cache<FdKey, Arc<std::fs::File>>,
+    cache: moka::future::Cache<FdKey, Arc<FdCacheEntry>>,
     /// Bumped by every invalidation. A miss-path open snapshots this before it
     /// opens and refuses to insert if it moved, so an fd opened before a
     /// heal/delete commit can never be resurrected into the cache after the
@@ -3931,7 +3963,7 @@ impl FdCache {
         }
     }
 
-    async fn get(&self, key: &FdKey) -> Option<Arc<std::fs::File>> {
+    async fn get(&self, key: &FdKey) -> Option<Arc<FdCacheEntry>> {
         self.cache.get(key).await
     }
 
@@ -3946,11 +3978,11 @@ impl FdCache {
     /// open bumped the generation, so a stale pre-heal/pre-delete inode is never
     /// cached. The post-insert re-check closes the tiny window where an
     /// invalidate races the insert itself, by removing the entry we just added.
-    async fn insert_if_fresh(&self, key: FdKey, file: Arc<std::fs::File>, gen_at_open: u64) {
+    async fn insert_if_fresh(&self, key: FdKey, entry: Arc<FdCacheEntry>, gen_at_open: u64) {
         if self.generation.load(Ordering::Acquire) != gen_at_open {
             return;
         }
-        self.cache.insert(key.clone(), file).await;
+        self.cache.insert(key.clone(), entry).await;
         if self.generation.load(Ordering::Acquire) != gen_at_open {
             self.cache.invalidate(&key).await;
         }
@@ -3986,7 +4018,7 @@ impl FdCache {
         self.generation.fetch_add(1, Ordering::AcqRel);
         let volume = volume.to_owned();
         let prefix = prefix.trim_end_matches('/').to_owned();
-        let matches = move |k: &FdKey, _: &Arc<std::fs::File>| {
+        let matches = move |k: &FdKey, _: &Arc<FdCacheEntry>| {
             k.volume == volume && (k.path == prefix || k.path.strip_prefix(&prefix).is_some_and(|r| r.starts_with('/')))
         };
         if self.cache.invalidate_entries_if(matches).is_err() {
@@ -4002,7 +4034,7 @@ impl FdCache {
     fn invalidate_volume(&self, volume: &str) {
         self.generation.fetch_add(1, Ordering::AcqRel);
         let volume = volume.to_owned();
-        let matches = move |k: &FdKey, _: &Arc<std::fs::File>| k.volume == volume;
+        let matches = move |k: &FdKey, _: &Arc<FdCacheEntry>| k.volume == volume;
         if self.cache.invalidate_entries_if(matches).is_err() {
             self.cache.invalidate_all();
         }
@@ -4020,7 +4052,8 @@ impl FdCache {
     /// tests that drive the cache directly.
     #[cfg(test)]
     async fn insert(&self, key: FdKey, file: Arc<std::fs::File>) {
-        self.cache.insert(key, file).await;
+        let len = file.metadata().map(|metadata| metadata.len()).unwrap_or_default();
+        self.cache.insert(key, Arc::new(FdCacheEntry { file, len })).await;
     }
 
     #[cfg(test)]
@@ -4399,7 +4432,12 @@ impl UringBackend {
         };
 
         let file = match cached {
-            Some(file) => file,
+            Some(entry) => {
+                if entry.len < u64::try_from(end_offset).map_err(|_| DiskError::FileCorrupt)? {
+                    return Err(DiskError::FileCorrupt);
+                }
+                Arc::clone(&entry.file)
+            }
             None => {
                 // Snapshot the cache generation BEFORE opening (rustfs/backlog#1176):
                 // if a heal/delete invalidation runs while this open is in flight,
@@ -4409,7 +4447,7 @@ impl UringBackend {
                 let root = self.root.clone();
                 let volume_owned = volume.to_owned();
                 let path_owned = path.to_owned();
-                let file = tokio::task::spawn_blocking(move || -> Result<std::fs::File> {
+                let (file, len) = tokio::task::spawn_blocking(move || -> Result<(std::fs::File, u64)> {
                     let file_path = resolve_uring_object_path(&root, &volume_owned, &path_owned)?;
                     let file = std::fs::File::open(&file_path).map_err(DiskError::from)?;
                     let meta = file.metadata().map_err(DiskError::from)?;
@@ -4417,30 +4455,22 @@ impl UringBackend {
                     if meta.len() < end_offset_u64 {
                         return Err(DiskError::FileCorrupt);
                     }
-                    Ok(file)
+                    Ok((file, meta.len()))
                 })
                 .await
                 .map_err(|e| DiskError::other(format!("uring pread join error: {e}")))??;
-                let file = Arc::new(file);
+                let file = Arc::new(FdCacheEntry {
+                    file: Arc::new(file),
+                    len,
+                });
                 if let (Some((cache, key)), Some(gen_at_open)) = (cache_entry, gen_at_open) {
                     cache.insert_if_fresh(key, Arc::clone(&file), gen_at_open).await;
                 }
-                file
+                file.file.clone()
             }
         };
 
         if length == 0 {
-            // Parity with StdBackend and the miss path (rustfs/backlog#1173): a
-            // zero-length read still rejects an offset past EOF. The miss path
-            // validated `meta.len() < end_offset` (end_offset == offset here), but
-            // a cache hit skipped it — so fstat the descriptor and match. This is
-            // a rare path (callers do not issue zero-length reads), so the one
-            // extra fstat is negligible.
-            match file.metadata() {
-                Ok(meta) if offset_u64 > meta.len() => return Err(DiskError::FileCorrupt),
-                Ok(_) => {}
-                Err(e) => return Err(DiskError::from(e)),
-            }
             return Ok(Bytes::new());
         }
 
@@ -21407,11 +21437,10 @@ mod test {
 
     /// Zero-length read bounds parity on the cache-HIT path (backlog#1173/#1180).
     /// A `length == 0` read past EOF must be rejected identically whether the
-    /// descriptor is freshly opened (miss path) or served from the cache: the
-    /// cache-hit branch fstats the descriptor to reproduce the miss path's
-    /// `offset > len` check instead of returning empty unconditionally. Seeds
-    /// the cache with a normal read so the zero-length reads are hits, then pins
-    /// that UringBackend and StdBackend agree on every case.
+    /// descriptor is freshly opened (miss path) or served from the cache. Seeds
+    /// the cache with a normal read so the zero-length reads reuse the same
+    /// open-time size snapshot, then pins that UringBackend and StdBackend agree
+    /// on every case.
     #[cfg(target_os = "linux")]
     #[tokio::test(flavor = "multi_thread")]
     async fn uring_zero_length_read_bounds_match_std_on_cache_hit() {

crates/ecstore/src/disk/mod.rs

@@ -677,15 +677,20 @@ impl DiskAPI for Disk {
 }
 
 impl Disk {
-    pub(crate) async fn delete_with_scanner_publication_lease(
+    pub async fn delete_with_scanner_publication_lease_and_guard(
         &self,
         volume: &str,
         path: &str,
         opts: DeleteOptions,
         scanner_publication_lease_token: Option<Uuid>,
+        external_guard: Option<Arc<dyn Send + Sync>>,
     ) -> Result<()> {
         match self {
-            Disk::Local(local_disk) => local_disk.delete(volume, path, opts).await,
+            Disk::Local(local_disk) => {
+                local_disk
+                    .delete_with_publication_guard(volume, path, opts, external_guard)
+                    .await
+            }
             Disk::Remote(remote_disk) => {
                 remote_disk
                     .delete_with_scanner_publication_lease(volume, path, opts, scanner_publication_lease_token)
@@ -714,11 +719,34 @@ impl Disk {
         dst_volume: &str,
         dst_path: &str,
         scanner_publication_lease_token: Option<Uuid>,
+    ) -> Result<RenameDataResp> {
+        self.rename_data_borrowed_with_fence_and_guard(
+            src_volume,
+            src_path,
+            fi,
+            dst_volume,
+            dst_path,
+            scanner_publication_lease_token,
+            None,
+        )
+        .await
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub async fn rename_data_borrowed_with_fence_and_guard(
+        &self,
+        src_volume: &str,
+        src_path: &str,
+        fi: &FileInfo,
+        dst_volume: &str,
+        dst_path: &str,
+        scanner_publication_lease_token: Option<Uuid>,
+        external_guard: Option<Arc<dyn Send + Sync>>,
     ) -> Result<RenameDataResp> {
         match self {
             Disk::Local(local_disk) => {
                 local_disk
-                    .rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
+                    .rename_data_borrowed_with_guard(src_volume, src_path, fi, dst_volume, dst_path, external_guard)
                     .await
             }
             Disk::Remote(remote_disk) => {

crates/ecstore/src/object_api/types.rs

@@ -19,6 +19,9 @@ use crate::storage_api_contracts::{
         HTTPPreconditions, ObjectLockRetentionOptions, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState,
     },
 };
+use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
+use tokio::sync::{Mutex, Notify, OwnedRwLockReadGuard};
+use tokio_util::sync::CancellationToken;
 
 #[derive(Clone)]
 pub struct NamespaceLockFence {
@@ -347,7 +350,338 @@ impl QuotaAdmission {
     }
 }
 
-#[derive(Debug, Default, Clone)]
+const SCANNER_PUBLICATION_SCOPE_ADMITTED: u8 = 0;
+const SCANNER_PUBLICATION_SCOPE_IN_FLIGHT: u8 = 1;
+const SCANNER_PUBLICATION_SCOPE_COMMITTED: u8 = 2;
+const SCANNER_PUBLICATION_SCOPE_ABORTED_BEFORE_COMMIT: u8 = 3;
+const SCANNER_PUBLICATION_SCOPE_INDETERMINATE: u8 = 4;
+
+/// The terminal result of a storage-owned scanner publication mutation.
+///
+/// This state is deliberately not serialized. It is the ownership hand-off
+/// between the scanner coordinator and the storage mutation task, so a
+/// detached rename/cleanup task can retain the movement permit until it has
+/// reported a definitive result.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum ScannerPublicationCommitState {
+    Admitted,
+    InFlight,
+    Committed,
+    AbortedBeforeCommit,
+    Indeterminate,
+}
+
+impl ScannerPublicationCommitState {
+    fn as_u8(self) -> u8 {
+        match self {
+            Self::Admitted => SCANNER_PUBLICATION_SCOPE_ADMITTED,
+            Self::InFlight => SCANNER_PUBLICATION_SCOPE_IN_FLIGHT,
+            Self::Committed => SCANNER_PUBLICATION_SCOPE_COMMITTED,
+            Self::AbortedBeforeCommit => SCANNER_PUBLICATION_SCOPE_ABORTED_BEFORE_COMMIT,
+            Self::Indeterminate => SCANNER_PUBLICATION_SCOPE_INDETERMINATE,
+        }
+    }
+
+    fn from_u8(value: u8) -> Self {
+        match value {
+            SCANNER_PUBLICATION_SCOPE_IN_FLIGHT => Self::InFlight,
+            SCANNER_PUBLICATION_SCOPE_COMMITTED => Self::Committed,
+            SCANNER_PUBLICATION_SCOPE_ABORTED_BEFORE_COMMIT => Self::AbortedBeforeCommit,
+            SCANNER_PUBLICATION_SCOPE_INDETERMINATE => Self::Indeterminate,
+            _ => Self::Admitted,
+        }
+    }
+
+    /// A caller may release its remote lease only after one of these states.
+    /// `Indeterminate` is intentionally excluded: the mutation may have
+    /// committed after cancellation or a transport failure.
+    pub fn permits_lease_release(self) -> bool {
+        matches!(self, Self::Committed | Self::AbortedBeforeCommit)
+    }
+}
+
+/// Why a storage-owned publication scope could not start its mutation.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum ScannerPublicationCommitStartError {
+    Cancelled,
+    DeadlineExceeded,
+    AlreadyStarted,
+    Terminal,
+}
+
+struct ScannerPublicationCommitScopeInner {
+    expected_movement_epoch: u64,
+    safe_deadline: tokio::time::Instant,
+    remote_lease_tokens: Arc<[Uuid]>,
+    cancellation: CancellationToken,
+    state: AtomicU8,
+    completed: Notify,
+    /// Set once a storage mutation task has taken ownership of the scope.
+    /// The caller-side RAII guard must not classify cancellation as
+    /// indeterminate while that owner can still report a definitive result.
+    owner_attached: AtomicBool,
+    /// The permit is storage-owned rather than borrowed from the scanner
+    /// future. A detached mutation task keeps the scope alive and therefore
+    /// keeps this guard alive until it reports a terminal state.
+    movement_permit: Mutex<Option<OwnedRwLockReadGuard<()>>>,
+    lease_release_safe: Arc<AtomicBool>,
+}
+
+/// Storage-owned ownership scope for one fenced scanner metadata mutation.
+///
+/// The scope is an in-memory capability. It is intentionally carried through
+/// [`ObjectOptions`] as a hidden field and never participates in serde, object
+/// metadata, RPC wire structures, or on-disk formats.
+#[derive(Clone)]
+pub struct ScannerPublicationCommitScope {
+    inner: Arc<ScannerPublicationCommitScopeInner>,
+}
+
+/// RAII fallback for storage paths that return before their commit closure
+/// takes ownership. An in-flight scope is never guessed to be aborted: it is
+/// marked indeterminate so remote lease release remains blocked.
+pub(crate) struct ScannerPublicationCommitScopeGuard {
+    scope: Option<ScannerPublicationCommitScope>,
+}
+
+impl ScannerPublicationCommitScopeGuard {
+    pub(crate) fn new(scope: ScannerPublicationCommitScope) -> Self {
+        Self { scope: Some(scope) }
+    }
+
+    pub(crate) fn disarm(&mut self) {
+        self.scope = None;
+    }
+}
+
+impl Drop for ScannerPublicationCommitScopeGuard {
+    fn drop(&mut self) {
+        let Some(scope) = self.scope.as_ref() else {
+            return;
+        };
+        if scope.owner_attached() {
+            return;
+        }
+        match scope.state() {
+            ScannerPublicationCommitState::Admitted => {
+                let _ = scope.mark_aborted_before_commit();
+            }
+            ScannerPublicationCommitState::InFlight => {
+                let _ = scope.mark_indeterminate();
+            }
+            ScannerPublicationCommitState::Committed
+            | ScannerPublicationCommitState::AbortedBeforeCommit
+            | ScannerPublicationCommitState::Indeterminate => {}
+        }
+    }
+}
+
+impl Debug for ScannerPublicationCommitScope {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ScannerPublicationCommitScope")
+            .field("expected_movement_epoch", &self.expected_movement_epoch())
+            .field("safe_deadline", &self.safe_deadline())
+            .field("remote_lease_token_count", &self.remote_lease_tokens().len())
+            .field("state", &self.state())
+            .finish()
+    }
+}
+
+impl ScannerPublicationCommitScope {
+    /// Construct a scope after the storage layer has acquired its movement
+    /// read permit. Callers must keep the scope attached to the actual
+    /// mutation owner until [`Self::wait_for_completion`] has resolved.
+    pub(crate) fn new_storage_owned(
+        expected_movement_epoch: u64,
+        safe_deadline: tokio::time::Instant,
+        remote_lease_tokens: Vec<Uuid>,
+        movement_permit: OwnedRwLockReadGuard<()>,
+    ) -> Self {
+        Self::new_storage_owned_with_release_flag(
+            expected_movement_epoch,
+            safe_deadline,
+            remote_lease_tokens,
+            movement_permit,
+            Arc::new(AtomicBool::new(true)),
+        )
+    }
+
+    pub(crate) fn new_storage_owned_with_release_flag(
+        expected_movement_epoch: u64,
+        safe_deadline: tokio::time::Instant,
+        remote_lease_tokens: Vec<Uuid>,
+        movement_permit: OwnedRwLockReadGuard<()>,
+        lease_release_safe: Arc<AtomicBool>,
+    ) -> Self {
+        lease_release_safe.store(false, Ordering::Release);
+        Self {
+            inner: Arc::new(ScannerPublicationCommitScopeInner {
+                expected_movement_epoch,
+                safe_deadline,
+                remote_lease_tokens: remote_lease_tokens.into(),
+                cancellation: CancellationToken::new(),
+                state: AtomicU8::new(SCANNER_PUBLICATION_SCOPE_ADMITTED),
+                completed: Notify::new(),
+                owner_attached: AtomicBool::new(false),
+                movement_permit: Mutex::new(Some(movement_permit)),
+                lease_release_safe,
+            }),
+        }
+    }
+
+    pub fn expected_movement_epoch(&self) -> u64 {
+        self.inner.expected_movement_epoch
+    }
+
+    pub fn safe_deadline(&self) -> tokio::time::Instant {
+        self.inner.safe_deadline
+    }
+
+    pub fn is_expired(&self) -> bool {
+        tokio::time::Instant::now() >= self.safe_deadline()
+    }
+
+    pub fn remote_lease_tokens(&self) -> &[Uuid] {
+        &self.inner.remote_lease_tokens
+    }
+
+    pub fn cancellation_token(&self) -> CancellationToken {
+        self.inner.cancellation.clone()
+    }
+
+    pub fn is_cancelled(&self) -> bool {
+        self.inner.cancellation.is_cancelled()
+    }
+
+    /// Whether a mutation that has already begun may still enter its durable
+    /// commit boundary. The storage owner must check this immediately before
+    /// starting each irreversible fan-out/rename operation.
+    pub fn can_commit(&self) -> bool {
+        self.state() == ScannerPublicationCommitState::InFlight && !self.is_cancelled() && !self.is_expired()
+    }
+
+    /// Transfer terminal-state responsibility from the caller to a detached
+    /// storage mutation owner. Once set, dropping a scanner waiter leaves the
+    /// scope in-flight until that owner reports committed or indeterminate.
+    pub fn attach_mutation_owner(&self) {
+        self.inner.owner_attached.store(true, Ordering::Release);
+    }
+
+    fn owner_attached(&self) -> bool {
+        self.inner.owner_attached.load(Ordering::Acquire)
+    }
+
+    pub fn state(&self) -> ScannerPublicationCommitState {
+        ScannerPublicationCommitState::from_u8(self.inner.state.load(Ordering::Acquire))
+    }
+
+    /// Request cancellation without claiming that a mutation has stopped.
+    /// The owner must still report `AbortedBeforeCommit` or `Indeterminate`.
+    pub fn cancel(&self) {
+        self.inner.cancellation.cancel();
+    }
+
+    pub fn try_begin(&self) -> std::result::Result<(), ScannerPublicationCommitStartError> {
+        if self.inner.cancellation.is_cancelled() {
+            return Err(ScannerPublicationCommitStartError::Cancelled);
+        }
+        if self.is_expired() {
+            return Err(ScannerPublicationCommitStartError::DeadlineExceeded);
+        }
+        self.inner
+            .state
+            .compare_exchange(
+                SCANNER_PUBLICATION_SCOPE_ADMITTED,
+                SCANNER_PUBLICATION_SCOPE_IN_FLIGHT,
+                Ordering::AcqRel,
+                Ordering::Acquire,
+            )
+            .map(|_| ())
+            .map_err(|state| {
+                if ScannerPublicationCommitState::from_u8(state).permits_lease_release() {
+                    ScannerPublicationCommitStartError::Terminal
+                } else {
+                    ScannerPublicationCommitStartError::AlreadyStarted
+                }
+            })
+    }
+
+    pub fn mark_committed(&self) -> bool {
+        self.mark_terminal(ScannerPublicationCommitState::Committed)
+    }
+
+    pub fn mark_aborted_before_commit(&self) -> bool {
+        if self
+            .inner
+            .state
+            .compare_exchange(
+                SCANNER_PUBLICATION_SCOPE_ADMITTED,
+                SCANNER_PUBLICATION_SCOPE_ABORTED_BEFORE_COMMIT,
+                Ordering::AcqRel,
+                Ordering::Acquire,
+            )
+            .is_ok()
+        {
+            self.inner.lease_release_safe.store(true, Ordering::Release);
+            self.inner.completed.notify_waiters();
+            return true;
+        }
+        false
+    }
+
+    pub fn mark_indeterminate(&self) -> bool {
+        self.mark_terminal(ScannerPublicationCommitState::Indeterminate)
+    }
+
+    fn mark_terminal(&self, terminal: ScannerPublicationCommitState) -> bool {
+        self.inner
+            .state
+            .compare_exchange(SCANNER_PUBLICATION_SCOPE_IN_FLIGHT, terminal.as_u8(), Ordering::AcqRel, Ordering::Acquire)
+            .is_ok()
+            .then(|| {
+                if terminal.permits_lease_release() {
+                    self.inner.lease_release_safe.store(true, Ordering::Release);
+                }
+                self.inner.completed.notify_waiters()
+            })
+            .is_some()
+    }
+
+    /// Wait until the mutation owner has reported a definitive terminal
+    /// state. The permit remains owned by this scope until all scope clones are
+    /// dropped or [`Self::release_movement_permit`] is called safely.
+    pub async fn wait_for_completion(&self) -> ScannerPublicationCommitState {
+        loop {
+            let notified = self.inner.completed.notified();
+            tokio::pin!(notified);
+            notified.as_mut().enable();
+            let state = self.state();
+            if state != ScannerPublicationCommitState::Admitted && state != ScannerPublicationCommitState::InFlight {
+                return state;
+            }
+            notified.await;
+        }
+    }
+
+    /// Release the storage-owned movement permit only after a known-safe
+    /// terminal result. Returns `false` for in-flight or indeterminate work.
+    pub async fn release_movement_permit(&self) -> bool {
+        if !self.state().permits_lease_release() {
+            return false;
+        }
+        self.inner.movement_permit.lock().await.take().is_some()
+    }
+}
+
+impl Drop for ScannerPublicationCommitScopeInner {
+    fn drop(&mut self) {
+        if !ScannerPublicationCommitState::from_u8(self.state.load(Ordering::Acquire)).permits_lease_release() {
+            self.lease_release_safe.store(false, Ordering::Release);
+        }
+    }
+}
+
+#[derive(Default, Clone)]
 pub struct ObjectOptions {
     // Use the maximum parity (N/2), used when saving server configuration files
     pub max_parity: bool,
@@ -384,6 +718,11 @@ pub struct ObjectOptions {
     #[doc(hidden)]
     pub put_object_cancellation: Option<tokio_util::sync::CancellationToken>,
 
+    /// Storage-owned scanner publication capability. This field is an
+    /// in-memory hand-off only; it is never copied into object metadata.
+    #[doc(hidden)]
+    pub scanner_publication_commit_scope: Option<ScannerPublicationCommitScope>,
+
     pub data_movement: bool,
     pub raw_data_movement_read: bool,
     /// Materialize the data-movement per-part checksum sidecar for APIs that
@@ -451,6 +790,76 @@ pub struct ObjectOptions {
     pub tier_delete_journal_api: Option<Arc<crate::store::ECStore>>,
 }
 
+impl std::fmt::Debug for ObjectOptions {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ObjectOptions")
+            .field("max_parity", &self.max_parity)
+            .field("mod_time", &self.mod_time)
+            .field("part_number", &self.part_number)
+            .field("delete_prefix", &self.delete_prefix)
+            .field("delete_prefix_object", &self.delete_prefix_object)
+            .field("version_id", &self.version_id.is_some())
+            .field("lifecycle_delete_all", &self.lifecycle_delete_all.is_some())
+            .field("lifecycle_delete_all_journal", &self.lifecycle_delete_all_journal.is_some())
+            .field("expected_current_version_id", &self.expected_current_version_id.is_some())
+            .field("expected_bucket_incarnation_id", &self.expected_bucket_incarnation_id)
+            .field("no_lock", &self.no_lock)
+            .field("metadata_cache_safe", &self.metadata_cache_safe)
+            .field("versioned", &self.versioned)
+            .field("version_suspended", &self.version_suspended)
+            .field("incl_free_versions", &self.incl_free_versions)
+            .field("skip_decommissioned", &self.skip_decommissioned)
+            .field("skip_rebalancing", &self.skip_rebalancing)
+            .field("skip_free_version", &self.skip_free_version)
+            .field("put_object_cancellation", &self.put_object_cancellation.is_some())
+            .field("scanner_publication_commit_scope", &self.scanner_publication_commit_scope)
+            .field("data_movement", &self.data_movement)
+            .field("raw_data_movement_read", &self.raw_data_movement_read)
+            .field("include_part_checksums", &self.include_part_checksums)
+            .field("src_pool_idx", &self.src_pool_idx)
+            .field("user_defined_count", &self.user_defined.len())
+            .field("preserve_etag", &self.preserve_etag.is_some())
+            .field("metadata_chg", &self.metadata_chg)
+            .field("http_preconditions", &self.http_preconditions.is_some())
+            .field("delete_replication", &self.delete_replication.is_some())
+            .field("delete_replication_config_snapshot", &self.delete_replication_config_snapshot)
+            .field("namespace_lock_fence", &self.namespace_lock_fence.is_some())
+            .field("bucket_lifecycle_lock_fence", &self.bucket_lifecycle_lock_fence.is_some())
+            .field("replication_request", &self.replication_request)
+            .field("proxy_request", &self.proxy_request)
+            .field("proxy_header_set", &self.proxy_header_set)
+            .field("replication_tagging_timestamp", &self.replication_tagging_timestamp)
+            .field("replication_retention_timestamp", &self.replication_retention_timestamp)
+            .field("replication_legalhold_timestamp", &self.replication_legalhold_timestamp)
+            .field("preserve_ciphertext", &self.preserve_ciphertext)
+            .field("delete_marker", &self.delete_marker)
+            .field("synthetic_version_id", &self.synthetic_version_id)
+            .field(
+                "transition",
+                &(self.data_movement
+                    || !self.transition.status.is_empty()
+                    || !self.transition.tier.is_empty()
+                    || self.transition.expected_data_dir.is_some()),
+            )
+            .field("expiration", &self.expiration)
+            .field(
+                "lifecycle_audit_event",
+                &(!self.lifecycle_audit_event.event.rule_id.is_empty()
+                    || !self.lifecycle_audit_event.event.storage_class.is_empty()),
+            )
+            .field("eval_metadata_count", &self.eval_metadata.as_ref().map(HashMap::len))
+            .field("object_lock_retention", &self.object_lock_retention.is_some())
+            .field("object_lock_delete", &self.object_lock_delete)
+            .field("object_lock_config_snapshot", &self.object_lock_config_snapshot.is_some())
+            .field("want_checksum", &self.want_checksum)
+            .field("skip_verify_bitrot", &self.skip_verify_bitrot)
+            .field("capacity_scope_token", &self.capacity_scope_token)
+            .field("quota_admission", &self.quota_admission)
+            .field("tier_delete_journal_api", &self.tier_delete_journal_api.is_some())
+            .finish()
+    }
+}
+
 /// Transient scanner-only carrier for target-side publication lease tokens.
 /// SetDisks consumes and removes this key before constructing durable
 /// FileInfo metadata; it must never appear in an S3-visible object.

crates/ecstore/src/runtime/instance.rs

@@ -391,7 +391,7 @@ impl InstanceContext {
         let previous = self.data_movement_operation_epoch.load(Ordering::Acquire);
         let _ = self
             .data_movement_operation_epoch
-            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
+            .try_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
         let result = self.data_movement_operation_epoch.load(Ordering::Acquire);
         if result == u64::MAX {
             self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
@@ -412,7 +412,7 @@ impl InstanceContext {
         }
         let updated = self
             .data_movement_generation
-            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |generation| generation.checked_add(1));
+            .try_update(Ordering::AcqRel, Ordering::Acquire, |generation| generation.checked_add(1));
         match updated {
             Ok(previous) => {
                 let Some(generation) = previous.checked_add(1) else {

crates/ecstore/src/services/notification_sys.rs

@@ -1483,7 +1483,7 @@ impl NotificationSys {
             futures.push(async move {
                 let client = client.ok_or_else(|| Error::other(format!("scanner activity peer[{idx}] is unreachable")))?;
                 let host = client.grid_host.clone();
-                scanner_activity_with_timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, &host, client.scanner_activity())
+                scanner_activity_with_retry(&client, &host)
                     .await
                     .map(|activity| (host, activity))
             });
@@ -1962,6 +1962,50 @@ where
         .map_err(|_| Error::other(format!("scanner activity peer {host} timed out after {timeout_duration:?}")))?
 }
 
+/// Classify transport-only activity failures without treating an answered
+/// peer's application error as an outage.
+pub fn scanner_peer_transport_error_message_is_retryable(error: &str) -> bool {
+    crate::cluster::rpc::client::message_has_network_needle(error)
+}
+
+fn scanner_activity_should_retry(first_error: Option<&Error>, timed_out: bool) -> bool {
+    timed_out || first_error.is_some_and(PeerRestClient::is_network_like_error)
+}
+
+/// Retry one activity probe after a bounded reconnect when the first attempt
+/// failed at the transport boundary.  A peer that answered with an invalid or
+/// incompatible activity response is not retried here: it must remain a hard
+/// fail-closed result for the all-peer publication proof.
+async fn scanner_activity_with_retry(client: &PeerRestClient, host: &str) -> Result<ScannerPeerActivity> {
+    let first = timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, client.scanner_activity()).await;
+    let should_retry = match &first {
+        Ok(Ok(_)) => false,
+        Ok(Err(err)) => scanner_activity_should_retry(Some(err), false),
+        Err(_) => scanner_activity_should_retry(None, true),
+    };
+
+    match first {
+        Ok(Ok(activity)) => return Ok(activity),
+        Ok(Err(err)) if !should_retry => return Err(err),
+        Ok(Err(err)) => {
+            debug!(peer = host, error = %err, "scanner activity probe failed on first transport attempt; reconnecting");
+            client.prepare_retry().await;
+        }
+        Err(_) => {
+            debug!(peer = host, timeout = ?SCANNER_ACTIVITY_PROBE_TIMEOUT, "scanner activity probe timed out on first attempt; reconnecting");
+            client.prepare_retry().await;
+        }
+    }
+
+    match timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, client.scanner_activity()).await {
+        Ok(result) => result,
+        Err(_) => {
+            client.evict_connection().await;
+            Err(Error::Timeout)
+        }
+    }
+}
+
 #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
 async fn call_peer_with_timeout<F, Fut>(
     timeout_dur: Duration,
@@ -2882,6 +2926,20 @@ mod tests {
         assert!(err.to_string().contains("peer-1"));
     }
 
+    #[test]
+    fn scanner_activity_retry_only_reconnects_transport_failures() {
+        assert!(scanner_activity_should_retry(None, true));
+        assert!(scanner_activity_should_retry(Some(&Error::other("connection refused")), false));
+        assert!(!scanner_activity_should_retry(
+            Some(&Error::other("peer returned an invalid scanner activity response proof")),
+            false
+        ));
+        assert!(!scanner_activity_should_retry(
+            Some(&Error::from(tonic::Status::internal("peer rejected activity"))),
+            false
+        ));
+    }
+
     #[tokio::test]
     async fn scanner_dirty_usage_acknowledgement_rejects_missing_and_duplicate_targets() {
         let sys = NotificationSys {

crates/ecstore/src/services/rebalance/control.rs

@@ -1667,6 +1667,8 @@ mod tests {
     async fn assert_real_activation_start_race(paused_kind: PoolActivationStartKind) {
         let (_temp_dirs, rebalance_store, decommission_store) =
             crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
+        crate::services::rebalance::promote_test_pool_meta_to_v2(&rebalance_store).await;
+        crate::services::rebalance::promote_test_pool_meta_to_v2(&decommission_store).await;
         let disk_stats = vec![
             DiskStat {
                 total_space: 100,

crates/ecstore/src/services/rebalance/mod.rs

@@ -111,6 +111,17 @@ pub(crate) async fn test_two_pool_stores_with_isolated_node_contexts(
     test_two_pool_stores_with_contexts(rebalance_meta, true).await
 }
 
+#[cfg(test)]
+pub(crate) async fn promote_test_pool_meta_to_v2(store: &std::sync::Arc<crate::store::ECStore>) {
+    let mut pool_meta = store.pool_meta.read().await.clone();
+    pool_meta.version = crate::core::pools::POOL_META_VERSION;
+    pool_meta
+        .save(store.pools.clone())
+        .await
+        .expect("test pool metadata should be promoted to V2");
+    *store.pool_meta.write().await = pool_meta;
+}
+
 #[cfg(test)]
 async fn test_two_pool_stores_with_contexts(
     rebalance_meta: Option<RebalanceMeta>,

crates/ecstore/src/services/tier/test_util.rs

@@ -625,7 +625,7 @@ impl WarmBackend for MockWarmBackend {
         let reject_once = self
             .inner
             .reject_non_empty_remote_version_validations
-            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| remaining.checked_sub(1))
+            .try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| remaining.checked_sub(1))
             .is_ok();
         if reject_once || self.inner.reject_non_empty_remote_versions.load(Ordering::Acquire) {
             return Err(std::io::Error::other("mock warm backend requires an unversioned remote object"));

crates/ecstore/src/services/tier/tier.rs

@@ -536,10 +536,6 @@ enum TierCandidateMutation {
 }
 
 impl TierCandidateMutation {
-    fn checks_lifecycle_references(&self) -> bool {
-        !matches!(self, Self::Add(_, true) | Self::Remove(_, true) | Self::Clear(true))
-    }
-
     fn intent_kind(&self) -> TierMutationIntentKind {
         match self {
             Self::Add(_, _) => TierMutationIntentKind::Add,
@@ -549,6 +545,16 @@ impl TierCandidateMutation {
         }
     }
 
+    /// `force` as supplied by the caller, or `false` for `Edit` (credential rebind never
+    /// accepts a force override). Used to gate the lifecycle-config reference check, which
+    /// unlike the persisted/physical-object reference checks is meant to be force-bypassable.
+    fn force(&self) -> bool {
+        match self {
+            Self::Add(_, force) | Self::Remove(_, force) | Self::Clear(force) => *force,
+            Self::Edit(_, _) => false,
+        }
+    }
+
     fn explicit_tier_name(&self) -> Option<&str> {
         match self {
             Self::Add(config, _) => Some(&config.name),
@@ -731,7 +737,7 @@ impl TierReferenceProofStore for ECStore {
 async fn ensure_no_authoritative_tier_object_references<S>(
     api: Arc<S>,
     affected_targets: &[TierMutationIntentTarget],
-    check_lifecycle_references: bool,
+    force: bool,
 ) -> std::result::Result<(), AdminError>
 where
     S: TierReferenceProofStore,
@@ -744,13 +750,13 @@ where
     if targets.is_empty() {
         return Ok(());
     }
-    ensure_no_authoritative_target_references(api, &targets, check_lifecycle_references).await
+    ensure_no_authoritative_target_references(api, &targets, force).await
 }
 
 async fn ensure_no_authoritative_target_references<S>(
     api: Arc<S>,
     targets: &[TierMutationIntentTarget],
-    check_lifecycle_references: bool,
+    force: bool,
 ) -> std::result::Result<(), AdminError>
 where
     S: TierReferenceProofStore,
@@ -760,7 +766,11 @@ where
         .await
         .map_err(tier_reference_proof_admin_error)?;
     for bucket in buckets {
-        if check_lifecycle_references {
+        // The lifecycle-config check only warns about a *future* transition attempt against
+        // this tier name, not an existing object/journal/transaction reference — unlike those,
+        // it is meant to be bypassable with `force`, mirroring `TierConfigMgr::remove()`'s
+        // `!force` gate on its own `driver.in_use()` probe (rustfs/backlog#2077).
+        if !force {
             ensure_no_authoritative_lifecycle_references(api.as_ref(), &bucket.name, targets).await?;
         }
         let mut marker = None;
@@ -1659,7 +1669,7 @@ impl TierOperationLease {
     ) -> std::result::Result<Self, AdminError> {
         inner
             .active_leases
-            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| active.checked_add(1))
+            .try_update(Ordering::AcqRel, Ordering::Acquire, |active| active.checked_add(1))
             .map_err(|_| {
                 let mut err = ERR_TIER_INVALID_CONFIG.clone();
                 err.message = "Remote tier operation lease capacity exhausted".to_string();
@@ -1678,7 +1688,7 @@ impl Drop for TierOperationLease {
         let result = self
             .inner
             .active_leases
-            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| active.checked_sub(1));
+            .try_update(Ordering::AcqRel, Ordering::Acquire, |active| active.checked_sub(1));
         match result {
             Ok(1) => self.inner.drained.notify_one(),
             Ok(_) => {}
@@ -3327,13 +3337,13 @@ impl TierConfigMgr {
                     let _config_lock = config_lock;
                     let _update = update;
                     let mutation_kind = mutation.intent_kind();
-                    let check_lifecycle_references = mutation.checks_lifecycle_references();
                     if version.is_none() && !candidate.tiers.is_empty() && mutation_kind != TierMutationIntentKind::Add {
                         return Err(TierConfigUpdateError::Load(io::Error::other(
                             "tier configuration mutation requires an existing config ETag",
                         )));
                     }
                     let explicit_tier_name = mutation.explicit_tier_name().map(str::to_string);
+                    let mutation_force = mutation.force();
                     let current_for_targets = TierConfigMgr {
                         driver_cache: HashMap::new(),
                         tiers: candidate
@@ -3366,7 +3376,7 @@ impl TierConfigMgr {
                     let affected_targets =
                         build_tier_mutation_affected_targets(mutation_kind, proof_targets, &current_for_targets, &candidate)
                             .map_err(TierConfigUpdateError::Publish)?;
-                    ensure_no_authoritative_tier_object_references(api.clone(), &affected_targets, check_lifecycle_references)
+                    ensure_no_authoritative_tier_object_references(api.clone(), &affected_targets, mutation_force)
                         .await
                         .map_err(TierConfigUpdateError::Publish)?;
                     let coordinator_intent =
@@ -6251,17 +6261,6 @@ mod tests {
         assert!(noop_targets.is_empty());
     }
 
-    #[test]
-    fn forced_tier_mutations_skip_only_lifecycle_reference_checks() {
-        assert!(!TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true).checks_lifecycle_references());
-        assert!(TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), false).checks_lifecycle_references());
-        assert!(!TierCandidateMutation::Remove("COLD-A".to_string(), true).checks_lifecycle_references());
-        assert!(TierCandidateMutation::Remove("COLD-A".to_string(), false).checks_lifecycle_references());
-        assert!(!TierCandidateMutation::Clear(true).checks_lifecycle_references());
-        assert!(TierCandidateMutation::Clear(false).checks_lifecycle_references());
-        assert!(TierCandidateMutation::Edit("COLD-A".to_string(), TierCreds::default()).checks_lifecycle_references());
-    }
-
     /// A fully offline `WarmBackend` used to exercise the driver-facing
     /// branches of `remove`/`verify` without touching a remote tier.
     struct MockWarmBackend {
@@ -11929,7 +11928,7 @@ mod tests {
                 let should_pause =
                     match barrier
                         .matches_before_pause
-                        .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| remaining.checked_sub(1))
+                        .try_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| remaining.checked_sub(1))
                     {
                         Ok(_) => false,
                         Err(_) => barrier.armed.swap(false, Ordering::SeqCst),
@@ -12110,32 +12109,7 @@ mod tests {
 
     #[tokio::test]
     async fn remove_and_clear_full_update_paths_preserve_force() {
-        let lifecycle_config = BucketLifecycleConfiguration {
-            expiry_updated_at: None,
-            rules: vec![LifecycleRule {
-                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
-                expiration: None,
-                abort_incomplete_multipart_upload: None,
-                del_marker_expiration: None,
-                filter: None,
-                id: Some("force-remove".to_string()),
-                noncurrent_version_expiration: None,
-                noncurrent_version_transitions: None,
-                prefix: None,
-                transitions: Some(vec![Transition {
-                    days: Some(1),
-                    date: None,
-                    storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
-                }]),
-            }],
-        };
         let remove_store = Arc::new(CasConfigStore::default());
-        remove_store.add_listed_version(ObjectInfo {
-            bucket: "photos".to_string(),
-            name: "safe.txt".to_string(),
-            ..Default::default()
-        });
-        remove_store.add_lifecycle_config("photos", lifecycle_config.clone());
         let mut persisted = empty_mgr();
         persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
         persisted
@@ -12162,12 +12136,6 @@ mod tests {
         );
 
         let clear_store = Arc::new(CasConfigStore::default());
-        clear_store.add_listed_version(ObjectInfo {
-            bucket: "photos".to_string(),
-            name: "safe.txt".to_string(),
-            ..Default::default()
-        });
-        clear_store.add_lifecycle_config("photos", lifecycle_config);
         persisted
             .save_tiering_config_if_current(clear_store.clone(), None)
             .await
@@ -12230,6 +12198,70 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn force_remove_and_save_bypasses_lifecycle_only_reference() {
+        // rustfs/rustfs#6832: reproduces the admin RemoveTier path (not just the lower-level
+        // reference-proof function) for a tier with zero transitioned objects but a lifecycle
+        // rule still pointing at it — the exact shape of
+        // `test_manual_transition_async_tier_failure_reports_terminal_partial` in e2e_test,
+        // which force-removes a tier a lifecycle rule still references to simulate a
+        // decommissioned backend.
+        let store = Arc::new(CasConfigStore::default());
+        let tier = build_rustfs_tier("COLD-A");
+        let mut persisted = empty_mgr();
+        persisted.tiers.insert("COLD-A".to_string(), tier.clone_with_credentials());
+        persisted
+            .save_tiering_config_if_current(store.clone(), None)
+            .await
+            .expect("reference proof fixture should persist");
+        let config = BucketLifecycleConfiguration {
+            expiry_updated_at: None,
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: None,
+                abort_incomplete_multipart_upload: None,
+                del_marker_expiration: None,
+                filter: None,
+                id: Some("move-current".to_string()),
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: Some(vec![Transition {
+                    days: Some(1),
+                    date: None,
+                    storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
+                }]),
+            }],
+        };
+        // CasConfigStore::list_bucket derives its fake bucket listing from listed_versions, so
+        // a bucket with a lifecycle rule but zero objects still needs a decoy entry to be
+        // visible to the reference-proof walk at all (mirrors production, where list_bucket
+        // enumerates real buckets independent of their contents).
+        store.add_listed_version(ObjectInfo {
+            bucket: "photos".to_string(),
+            name: "safe.txt".to_string(),
+            ..Default::default()
+        });
+        store.add_lifecycle_config("photos", config);
+
+        let manager = TierConfigMgr::new();
+        manager.write().await.tiers.insert("COLD-A".to_string(), tier);
+        TierConfigMgr::remove_and_save_with(&manager, store.clone(), "COLD-A", true)
+            .await
+            .expect("force remove must bypass a lifecycle-config-only reference");
+
+        assert!(!manager.read().await.tiers.contains_key("COLD-A"));
+        assert!(
+            !load_tier_config_for_update(store)
+                .await
+                .expect("config should still reload")
+                .0
+                .tiers
+                .contains_key("COLD-A"),
+            "force removal must persist the empty candidate"
+        );
+    }
+
     #[tokio::test]
     async fn zero_reference_proof_blocks_clear_before_config_save() {
         let store = Arc::new(CasConfigStore::default());
@@ -12295,7 +12327,7 @@ mod tests {
             "COLD-A",
             Some(replacement_identity),
         ));
-        ensure_no_authoritative_tier_object_references(store.clone(), std::slice::from_ref(&target), true)
+        ensure_no_authoritative_tier_object_references(store.clone(), std::slice::from_ref(&target), false)
             .await
             .expect("references already written with the new destination identity should not block rebind");
 
@@ -12305,7 +12337,7 @@ mod tests {
             "COLD-A",
             Some(current_identity),
         ));
-        let err = ensure_no_authoritative_tier_object_references(store, &[target], true)
+        let err = ensure_no_authoritative_tier_object_references(store, &[target], false)
             .await
             .expect_err("references to the old destination identity must block rebind");
         assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
@@ -12348,7 +12380,7 @@ mod tests {
         });
         store.add_lifecycle_config("photos", config);
 
-        let err = ensure_no_authoritative_tier_object_references(store, &[target], true)
+        let err = ensure_no_authoritative_tier_object_references(store, &[target], false)
             .await
             .expect_err("lifecycle rule should block deletion");
 
@@ -12393,7 +12425,7 @@ mod tests {
         });
         store.add_lifecycle_config("photos", config);
 
-        let err = ensure_no_authoritative_tier_object_references(store, &[target], true)
+        let err = ensure_no_authoritative_tier_object_references(store, &[target], false)
             .await
             .expect_err("lifecycle rule should block deletion");
 
@@ -12402,8 +12434,57 @@ mod tests {
         assert!(err.message.contains("photos"), "{}", err.message);
     }
 
+    #[tokio::test]
+    async fn zero_reference_proof_force_true_skips_lifecycle_reference_check() {
+        // rustfs/rustfs#6832: force=true must bypass a lifecycle-config-only reference,
+        // mirroring the existing `!force` gate on `TierConfigMgr::remove()`'s own
+        // `driver.in_use()` probe. It must NOT bypass real object/journal/transaction
+        // references — those stay force-immune and are covered separately below.
+        let current = build_rustfs_tier("COLD-A");
+        let current_identity = tier_backend_identity(&current).expect("current identity should encode");
+        let target = TierMutationIntentTarget {
+            tier_name: "COLD-A".to_string(),
+            old_backend_identity: Some(current_identity),
+            new_backend_identity: None,
+        };
+        let config = BucketLifecycleConfiguration {
+            expiry_updated_at: None,
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: None,
+                abort_incomplete_multipart_upload: None,
+                del_marker_expiration: None,
+                filter: None,
+                id: Some("move-current".to_string()),
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: Some(vec![Transition {
+                    days: Some(1),
+                    date: None,
+                    storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
+                }]),
+            }],
+        };
+        let store = Arc::new(CasConfigStore::default());
+        store.add_listed_version(ObjectInfo {
+            bucket: "photos".to_string(),
+            name: "safe.txt".to_string(),
+            ..Default::default()
+        });
+        store.add_lifecycle_config("photos", config);
+
+        ensure_no_authoritative_tier_object_references(store, &[target], true)
+            .await
+            .expect("force=true must bypass a lifecycle-config-only reference");
+    }
+
     #[tokio::test]
     async fn zero_reference_proof_blocks_persisted_journal_transaction_and_free_version_references() {
+        // All three sub-checks below pass `force: true` to pin that physical/persisted
+        // references stay force-immune post rustfs/rustfs#6832 (only the lifecycle-config
+        // check, covered by `zero_reference_proof_force_true_skips_lifecycle_reference_check`
+        // above, is meant to be force-bypassable).
         let current = build_rustfs_tier("COLD-A");
         let current_identity = tier_backend_identity(&current).expect("current identity should encode");
         let replacement = build_azure_tier("account-a");
@@ -12434,7 +12515,7 @@ mod tests {
             .await;
         let err = ensure_no_authoritative_tier_object_references(journal_store, std::slice::from_ref(&target), true)
             .await
-            .expect_err("unfinished delete journal for the old backend must block rebind");
+            .expect_err("unfinished delete journal for the old backend must block rebind even with force");
         assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
         assert!(err.message.contains(TIER_DELETE_JOURNAL_PREFIX), "{}", err.message);
 
@@ -12472,7 +12553,7 @@ mod tests {
             .await;
         let err = ensure_no_authoritative_tier_object_references(transaction_store, std::slice::from_ref(&target), true)
             .await
-            .expect_err("unfinished transition transaction for the old backend must block rebind");
+            .expect_err("unfinished transition transaction for the old backend must block rebind even with force");
         assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
         assert!(err.message.contains(TRANSITION_TRANSACTION_RECORD_PREFIX), "{}", err.message);
 
@@ -12486,7 +12567,7 @@ mod tests {
         free_version_store.add_listed_version(free_version);
         let err = ensure_no_authoritative_tier_object_references(free_version_store, &[target], true)
             .await
-            .expect_err("recoverable free version for the old backend must block rebind");
+            .expect_err("recoverable free version for the old backend must block rebind even with force");
         assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
         assert!(err.message.contains("photos/2026/free-version.jpg"), "{}", err.message);
     }
@@ -12510,7 +12591,7 @@ mod tests {
         }
         store.omit_truncated_reference_marker();
 
-        let err = ensure_no_authoritative_tier_object_references(store, &[target], true)
+        let err = ensure_no_authoritative_tier_object_references(store, &[target], false)
             .await
             .expect_err("truncated authoritative reference scan without a marker must fail closed");
         assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -3657,6 +3657,7 @@ pub(in crate::set_disk) struct RenameTailOutcome {
 pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
     write_quorum: usize,
     scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
+    scanner_publication_commit_scope: Option<crate::object_api::ScannerPublicationCommitScope>,
 }
 
 impl<'a> RenameDataFenceOptions<'a> {
@@ -3667,8 +3668,17 @@ impl<'a> RenameDataFenceOptions<'a> {
         Self {
             write_quorum,
             scanner_publication_lease_tokens,
+            scanner_publication_commit_scope: None,
         }
     }
+
+    pub(in crate::set_disk) fn with_publication_scope(
+        mut self,
+        scanner_publication_commit_scope: Option<crate::object_api::ScannerPublicationCommitScope>,
+    ) -> Self {
+        self.scanner_publication_commit_scope = scanner_publication_commit_scope;
+        self
+    }
 }
 
 #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
@@ -3778,6 +3788,37 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal<
     }
 }
 
+async fn run_scanner_publication_delete_owner<F, Fut>(
+    scope: Option<crate::object_api::ScannerPublicationCommitScope>,
+    operation: F,
+) -> disk::error::Result<()>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: Future<Output = disk::error::Result<()>> + Send + 'static,
+{
+    if scope.is_none() {
+        return operation().await;
+    }
+    if let Some(scope) = scope.as_ref() {
+        scope.attach_mutation_owner();
+    }
+    tokio::spawn(async move {
+        let result = operation().await;
+        if let Some(scope) = scope.as_ref() {
+            if result.is_ok() {
+                let _ = scope.mark_committed();
+            } else {
+                // A failed quorum does not prove that no replica committed;
+                // keep the permit indeterminate for supervisor reconciliation.
+                let _ = scope.mark_indeterminate();
+            }
+        }
+        result
+    })
+    .await
+    .map_err(|_| DiskError::other("scanner publication delete owner failed"))?
+}
+
 impl SetDisks {
     pub(in crate::set_disk) fn default_read_quorum(&self) -> usize {
         self.set_drive_count - self.default_parity_count
@@ -3995,6 +4036,7 @@ impl SetDisks {
         let RenameDataFenceOptions {
             write_quorum,
             scanner_publication_lease_tokens,
+            scanner_publication_commit_scope: _scanner_publication_commit_scope,
         } = fence_options;
         if let Some(file_info) = disks
             .iter()
@@ -4352,6 +4394,7 @@ impl SetDisks {
         let RenameDataFenceOptions {
             write_quorum,
             scanner_publication_lease_tokens,
+            scanner_publication_commit_scope,
         } = fence_options;
         if let Some(file_info) = disks
             .iter()
@@ -4383,11 +4426,15 @@ impl SetDisks {
         let fanout_src_object = src_object.clone();
         let fanout_dst_bucket = dst_bucket.clone();
         let fanout_dst_object = dst_object.clone();
+        let fanout_publication_scope = scanner_publication_commit_scope.clone();
         // Keep one coordinator task so a cancelled caller cannot drop partially
         // completed disk mutations. Per-disk futures stay ordered in `join_all`,
         // preserving slot-indexed quorum and convergence accounting without a
         // scheduler task for every disk.
         let fanout = tokio::spawn(async move {
+            // Keep the storage-owned movement permit attached to the actual
+            // fan-out owner, even if the caller future is cancelled.
+            let _fanout_publication_scope = fanout_publication_scope;
             let successful_rename_completion_rank =
                 rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
             let futures = fanout_disks
@@ -4401,6 +4448,7 @@ impl SetDisks {
                     let dst_object = fanout_dst_object.clone();
                     let dst_bucket = fanout_dst_bucket.clone();
                     let successful_rename_completion_rank = successful_rename_completion_rank.clone();
+                    let publication_scope = scanner_publication_commit_scope.clone();
 
                     std::panic::AssertUnwindSafe(async move {
                         // Test-only introspection guard: counts this operation as
@@ -4433,6 +4481,13 @@ impl SetDisks {
                             return Err(err);
                         }
 
+                        if let Some(scope) = publication_scope.as_ref()
+                            && !scope.can_commit()
+                        {
+                            let _ = scope.mark_indeterminate();
+                            return Err(DiskError::other("scanner publication commit scope deadline or cancellation reached"));
+                        }
+
                         let disk_wait_started = rustfs_io_metrics::put_stage_timer();
                         let result = disk
                             .rename_data_borrowed_with_fence(
@@ -5841,7 +5896,8 @@ impl SetDisks {
 
     #[cfg(test)]
     pub(in crate::set_disk) async fn delete_prefix(&self, bucket: &str, prefix: &str) -> disk::error::Result<()> {
-        self.delete_prefix_with_scanner_publication_lease(bucket, prefix, None).await
+        self.delete_prefix_with_scanner_publication_lease(bucket, prefix, None, None)
+            .await
     }
 
     /// Delete a prefix with an optional per-remote-disk scanner publication
@@ -5852,6 +5908,7 @@ impl SetDisks {
         bucket: &str,
         prefix: &str,
         scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
+        scanner_publication_commit_scope: Option<crate::object_api::ScannerPublicationCommitScope>,
     ) -> disk::error::Result<()> {
         let disks = self.get_disks_internal().await;
         let write_quorum = disks.len() / 2 + 1;
@@ -5860,11 +5917,21 @@ impl SetDisks {
         let mut futures = Vec::with_capacity(disks.len());
 
         for (disk_op, scanner_publication_lease_token) in disks.iter().zip(fanout_fence_tokens) {
+            let disk_op = disk_op.clone();
             let bucket = bucket.to_string();
             let prefix = prefix.to_string();
+            let scanner_publication_commit_scope = scanner_publication_commit_scope.clone();
             futures.push(async move {
                 if let Some(disk) = disk_op {
-                    disk.delete_with_scanner_publication_lease(
+                    if let Some(scope) = scanner_publication_commit_scope.as_ref()
+                        && !scope.can_commit()
+                    {
+                        return Err(DiskError::other("scanner publication delete scope cannot commit"));
+                    }
+                    let external_guard = scanner_publication_commit_scope
+                        .as_ref()
+                        .map(|scope| Arc::new(scope.clone()) as Arc<dyn Send + Sync>);
+                    disk.delete_with_scanner_publication_lease_and_guard(
                         &bucket,
                         &prefix,
                         DeleteOptions {
@@ -5873,6 +5940,7 @@ impl SetDisks {
                             ..Default::default()
                         },
                         scanner_publication_lease_token,
+                        external_guard,
                     )
                     .await
                 } else {
@@ -5881,7 +5949,10 @@ impl SetDisks {
             });
         }
 
-        Self::reduce_delete_prefix_results(join_all(futures).await, write_quorum)
+        run_scanner_publication_delete_owner(scanner_publication_commit_scope, move || async move {
+            Self::reduce_delete_prefix_results(join_all(futures).await, write_quorum)
+        })
+        .await
     }
 
     /// Scan a single disk's copy of `prefix` and decide whether it is an orphan
@@ -6809,6 +6880,63 @@ mod tests {
     use tempfile::TempDir;
     use tokio::io::AsyncReadExt;
 
+    #[tokio::test]
+    async fn scanner_delete_owner_survives_waiter_cancellation() {
+        let movement_gate = Arc::new(tokio::sync::RwLock::new(()));
+        let movement_permit = movement_gate.clone().read_owned().await;
+        let scope = crate::object_api::ScannerPublicationCommitScope::new_storage_owned(
+            7,
+            tokio::time::Instant::now() + std::time::Duration::from_secs(30),
+            Vec::new(),
+            movement_permit,
+        );
+        scope.try_begin().expect("delete scope should enter flight");
+        let scope_guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
+        let (finished_tx, finished_rx) = tokio::sync::oneshot::channel();
+
+        let waiter = tokio::spawn(run_scanner_publication_delete_owner(Some(scope.clone()), move || async move {
+            started_tx.send(()).expect("delete owner should start");
+            release_rx.await.expect("delete owner should be released");
+            finished_tx.send(()).expect("delete owner should finish");
+            Ok(())
+        }));
+        started_rx.await.expect("delete owner should run");
+        drop(scope_guard);
+        waiter.abort();
+        assert_eq!(
+            scope.state(),
+            crate::object_api::ScannerPublicationCommitState::InFlight,
+            "caller cancellation must not classify an owned delete as indeterminate"
+        );
+
+        let mut movement_writer = Box::pin(movement_gate.write_owned());
+        assert!(
+            tokio::time::timeout(std::time::Duration::from_millis(20), &mut movement_writer)
+                .await
+                .is_err(),
+            "movement transition must remain fenced while delete owner drains"
+        );
+        release_tx.send(()).expect("delete owner should remain alive");
+        finished_rx.await.expect("delete owner should drain");
+        tokio::time::timeout(std::time::Duration::from_secs(1), async {
+            loop {
+                if scope.state() == crate::object_api::ScannerPublicationCommitState::Committed {
+                    break;
+                }
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("delete owner should report a terminal result");
+        assert!(
+            scope.release_movement_permit().await,
+            "terminal delete should release its movement permit"
+        );
+        movement_writer.await;
+    }
+
     #[test]
     fn write_precondition_lookup_errors_fail_closed_unless_absence_is_known() {
         let create_only = HTTPPreconditions {

crates/ecstore/src/set_disk/mod.rs

@@ -2525,6 +2525,7 @@ fn record_get_object_reader_path_observation(
             GET_OBJECT_PATH_CODEC_STREAMING => 5,
             GET_OBJECT_PATH_REMOTE_TRANSITION => 6,
             GET_OBJECT_PATH_EMPTY => 7,
+            GET_OBJECT_PATH_LEGACY_DUPLEX => 8,
             _ => 255,
         },
         Ordering::Relaxed,
@@ -3649,7 +3650,7 @@ impl SetDisks {
         let hash_bytes = hash.to_le_bytes();
         let index = usize::from(u16::from_le_bytes([hash_bytes[0], hash_bytes[1]]) % GET_OBJECT_METADATA_CACHE_FENCE_SHARDS);
         let generation = &self.get_object_metadata_cache_generations[index];
-        let previous = match generation.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| current.checked_add(1)) {
+        let previous = match generation.try_update(Ordering::AcqRel, Ordering::Acquire, |current| current.checked_add(1)) {
             Ok(previous) | Err(previous) => previous,
         };
         let previous = GetObjectMetadataCacheGeneration {
@@ -3666,7 +3667,7 @@ impl SetDisks {
 
     fn invalidate_all_get_object_metadata_cache(&self) {
         for generation in self.get_object_metadata_cache_generations.iter() {
-            let _ = generation.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| current.checked_add(1));
+            let _ = generation.try_update(Ordering::AcqRel, Ordering::Acquire, |current| current.checked_add(1));
         }
         self.get_object_metadata_cache.invalidate_all();
     }
@@ -3937,6 +3938,44 @@ impl SetDisks {
         owner.scanner_data_usage_publication_admission_guard().await
     }
 
+    pub async fn scanner_data_usage_publication_commit_scope(
+        &self,
+        expected_movement_epoch: u64,
+        safe_deadline: tokio::time::Instant,
+        remote_lease_tokens: Vec<Uuid>,
+    ) -> Option<crate::object_api::ScannerPublicationCommitScope> {
+        let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
+        if epoch != expected_movement_epoch {
+            return None;
+        }
+        Some(crate::object_api::ScannerPublicationCommitScope::new_storage_owned(
+            epoch,
+            safe_deadline,
+            remote_lease_tokens,
+            movement_permit,
+        ))
+    }
+
+    pub async fn scanner_data_usage_publication_commit_scope_with_release_flag(
+        &self,
+        expected_movement_epoch: u64,
+        safe_deadline: tokio::time::Instant,
+        remote_lease_tokens: Vec<Uuid>,
+        lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
+    ) -> Option<crate::object_api::ScannerPublicationCommitScope> {
+        let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
+        if epoch != expected_movement_epoch {
+            return None;
+        }
+        Some(crate::object_api::ScannerPublicationCommitScope::new_storage_owned_with_release_flag(
+            epoch,
+            safe_deadline,
+            remote_lease_tokens,
+            movement_permit,
+            lease_release_safe,
+        ))
+    }
+
     /// Whether both sets' namespace-lock implementations cover the same object key.
     pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
         match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {

crates/ecstore/src/set_disk/ops/heal.rs

@@ -34,10 +34,77 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore";
 const LOG_SUBSYSTEM_HEAL: &str = "heal";
 const EVENT_HEAL_OBJECT_RENAME: &str = "heal_object_rename";
 const HEAL_RENAME_INCOMPLETE: &str = "heal rename incomplete";
+const READ_REPAIR_DATA_PHASE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60 * 60);
 
 #[cfg(test)]
 static HEAL_RENAME_FAILURES: std::sync::Mutex<Vec<(String, String, usize)>> = std::sync::Mutex::new(Vec::new());
 
+#[cfg(test)]
+struct ReadRepairCommitPause {
+    bucket: String,
+    object: String,
+    arrived: std::sync::Arc<tokio::sync::Notify>,
+    release: std::sync::Arc<tokio::sync::Notify>,
+}
+
+#[cfg(test)]
+static READ_REPAIR_COMMIT_PAUSES: std::sync::Mutex<Vec<ReadRepairCommitPause>> = std::sync::Mutex::new(Vec::new());
+
+#[cfg(test)]
+struct ReadRepairCommitPauseScope {
+    bucket: String,
+    object: String,
+}
+
+#[cfg(test)]
+impl ReadRepairCommitPauseScope {
+    fn install(bucket: &str, object: &str) -> (Self, std::sync::Arc<tokio::sync::Notify>, std::sync::Arc<tokio::sync::Notify>) {
+        let arrived = std::sync::Arc::new(tokio::sync::Notify::new());
+        let release = std::sync::Arc::new(tokio::sync::Notify::new());
+        READ_REPAIR_COMMIT_PAUSES
+            .lock()
+            .expect("read-repair commit pause registry should not poison")
+            .push(ReadRepairCommitPause {
+                bucket: bucket.to_string(),
+                object: object.to_string(),
+                arrived: arrived.clone(),
+                release: release.clone(),
+            });
+        (
+            Self {
+                bucket: bucket.to_string(),
+                object: object.to_string(),
+            },
+            arrived,
+            release,
+        )
+    }
+}
+
+#[cfg(test)]
+impl Drop for ReadRepairCommitPauseScope {
+    fn drop(&mut self) {
+        READ_REPAIR_COMMIT_PAUSES
+            .lock()
+            .expect("read-repair commit pause registry should not poison")
+            .retain(|pause| pause.bucket != self.bucket || pause.object != self.object);
+    }
+}
+
+#[cfg(test)]
+async fn pause_read_repair_before_commit(bucket: &str, object: &str) {
+    let pause = READ_REPAIR_COMMIT_PAUSES
+        .lock()
+        .expect("read-repair commit pause registry should not poison")
+        .iter()
+        .find(|pause| pause.bucket == bucket && pause.object == object)
+        .map(|pause| (pause.arrived.clone(), pause.release.clone()));
+    if let Some((arrived, release)) = pause {
+        arrived.notify_one();
+        release.notified().await;
+    }
+}
+
 #[cfg(test)]
 struct HealRenameFailureScope {
     bucket: String,
@@ -177,6 +244,34 @@ struct RecoverableMetaCandidate {
     local_payload: bool,
 }
 
+#[derive(Clone, Copy, Eq, PartialEq)]
+struct ReadRepairCommitFingerprint {
+    identity: [u8; 32],
+}
+
+impl ReadRepairCommitFingerprint {
+    fn from_file_info(fi: &FileInfo) -> Self {
+        Self {
+            identity: SetDisks::file_info_quorum_hash(fi),
+        }
+    }
+}
+
+#[derive(Clone, Copy)]
+enum HealObjectLockKind {
+    Read,
+    Write,
+}
+
+impl HealObjectLockKind {
+    fn as_str(self) -> &'static str {
+        match self {
+            Self::Read => "read",
+            Self::Write => "write",
+        }
+    }
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 enum DanglingDeleteSafety {
     UnsafeToDelete,
@@ -449,6 +544,107 @@ impl SetDisks {
         Box::pin(self.heal_object_with_explicit_version_regen(bucket, object, version_id, opts, true)).await
     }
 
+    async fn read_repair_commit_fingerprint(
+        &self,
+        disks: &[Option<DiskStore>],
+        bucket: &str,
+        object: &str,
+        version_id: &str,
+    ) -> disk::error::Result<ReadRepairCommitFingerprint> {
+        let (parts_metadata, errs, _) = Self::read_all_fileinfo_observed(
+            disks,
+            "",
+            bucket,
+            object,
+            version_id,
+            true,
+            false,
+            false,
+            true,
+            self.default_parity_count,
+        )
+        .await?;
+        let (read_quorum, _) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)?;
+        let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
+        let (_, quorum_mod_time, quorum_etag) = Self::list_online_disks(disks, &parts_metadata, &errs, read_quorum);
+        let mut latest_meta = Self::pick_valid_fileinfo(&parts_metadata, quorum_mod_time, quorum_etag, read_quorum)?;
+        Self::hydrate_selected_fileinfo_part_checksums(&mut latest_meta)?;
+        Ok(ReadRepairCommitFingerprint::from_file_info(&latest_meta))
+    }
+
+    async fn acquire_heal_object_lock(
+        &self,
+        bucket: &str,
+        object: &str,
+        kind: HealObjectLockKind,
+    ) -> disk::error::Result<rustfs_lock::NamespaceLockGuard> {
+        let ns_lock = self
+            .new_ns_lock(bucket, object)
+            .await
+            .map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?;
+        let lock_result = match kind {
+            HealObjectLockKind::Read => ns_lock.get_read_lock(get_lock_acquire_timeout()).await,
+            HealObjectLockKind::Write => ns_lock.get_write_lock(get_lock_acquire_timeout()).await,
+        };
+        lock_result.map_err(|e| {
+            self.map_namespace_lock_error(bucket, object, kind.as_str(), e)
+                .narrow_to_disk()
+                .unwrap_or_else(DiskError::other)
+        })
+    }
+
+    async fn acquire_revalidated_read_repair_commit_lock(
+        &self,
+        disks: &[Option<DiskStore>],
+        bucket: &str,
+        object: &str,
+        version_id: &str,
+        expected_fingerprint: ReadRepairCommitFingerprint,
+    ) -> disk::error::Result<Option<rustfs_lock::NamespaceLockGuard>> {
+        #[cfg(test)]
+        pause_read_repair_before_commit(bucket, object).await;
+
+        let write_lock_wait_start = std::time::Instant::now();
+        let guard = self
+            .acquire_heal_object_lock(bucket, object, HealObjectLockKind::Write)
+            .await?;
+        let write_lock_wait_ms = u64::try_from(write_lock_wait_start.elapsed().as_millis()).unwrap_or(u64::MAX);
+
+        let revalidation_start = std::time::Instant::now();
+        let current_fingerprint = self.read_repair_commit_fingerprint(disks, bucket, object, version_id).await?;
+        let revalidation_duration_ms = u64::try_from(revalidation_start.elapsed().as_millis()).unwrap_or(u64::MAX);
+
+        if current_fingerprint == expected_fingerprint {
+            debug!(
+                event = EVENT_SET_DISK_HEAL,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_SET_DISK,
+                state = "read_repair_commit_revalidated",
+                bucket,
+                object,
+                version_id,
+                write_lock_wait_ms,
+                revalidation_duration_ms,
+                "Read-repair heal revalidated metadata before rename"
+            );
+            Ok(Some(guard))
+        } else {
+            warn!(
+                event = EVENT_SET_DISK_HEAL,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_SET_DISK,
+                state = "read_repair_commit_stale",
+                bucket,
+                object,
+                version_id,
+                write_lock_wait_ms,
+                revalidation_duration_ms,
+                "Read-repair heal abandoned stale metadata before rename"
+            );
+            Ok(None)
+        }
+    }
+
     #[allow(clippy::too_many_lines)]
     async fn heal_object_with_explicit_version_regen(
         &self,
@@ -480,18 +676,36 @@ impl SetDisks {
             ..Default::default()
         };
 
+        let read_repair_uses_shared_lock = opts.read_repair && !opts.no_lock;
+        // Read-repair heals are triggered by successful reads of still-readable
+        // objects. Keep readers flowing during the long reconstruction phase;
+        // take the write lock only for the final verify-and-rename fence.
+        let mut _read_repair_read_lock_guard = if read_repair_uses_shared_lock {
+            let guard = self
+                .acquire_heal_object_lock(bucket, object, HealObjectLockKind::Read)
+                .await?;
+            debug!(
+                event = EVENT_SET_DISK_HEAL,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_SET_DISK,
+                state = "read_repair_shared_data_lock",
+                bucket,
+                object,
+                version_id,
+                "Read-repair heal using shared namespace lock for data phase"
+            );
+            Some(guard)
+        } else {
+            None
+        };
+
         // Bound, not `_`: this guard must live to the end of the scope. A bare
         // `_` would drop it here and release the namespace write lock.
-        let _write_lock_guard = if !opts.no_lock {
-            let ns_lock = self
-                .new_ns_lock(bucket, object)
-                .await
-                .map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?;
-            Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
-                self.map_namespace_lock_error(bucket, object, "write", e)
-                    .narrow_to_disk()
-                    .unwrap_or_else(DiskError::other)
-            })?)
+        let _write_lock_guard = if !opts.no_lock && !read_repair_uses_shared_lock {
+            Some(
+                self.acquire_heal_object_lock(bucket, object, HealObjectLockKind::Write)
+                    .await?,
+            )
         } else {
             None
         };
@@ -791,6 +1005,10 @@ impl SetDisks {
                                 );
                             }
 
+                            if read_repair_uses_shared_lock {
+                                return Ok((result, Some(cannot_heal_err)));
+                            }
+
                             // `disks_with_all_parts` normalizes conflicting entries
                             // in `parts_metadata` to defaults. Re-read only before
                             // destructive cleanup so the guard sees every original
@@ -952,6 +1170,9 @@ impl SetDisks {
                         // For a regular object a missing data_dir means the latest metadata is
                         // corrupt; fail this object's heal with a clear error instead of building
                         // part paths under a nil UUID directory.
+                        let read_repair_commit_fingerprint =
+                            read_repair_uses_shared_lock.then(|| ReadRepairCommitFingerprint::from_file_info(&latest_meta));
+
                         let data_dir = match latest_meta.data_dir {
                             Some(data_dir) => data_dir,
                             None => {
@@ -1111,7 +1332,16 @@ impl SetDisks {
                                     );
                                     writer_failure_warned = true;
                                 }
-                                if let Err(e) = erasure.heal(&mut writers, readers, part.size, &prefer).await {
+                                let heal_part = erasure.heal(&mut writers, readers, part.size, &prefer);
+                                let heal_part_result = if read_repair_uses_shared_lock {
+                                    match tokio::time::timeout(READ_REPAIR_DATA_PHASE_TIMEOUT, heal_part).await {
+                                        Ok(result) => result,
+                                        Err(_) => Err(DiskError::Timeout),
+                                    }
+                                } else {
+                                    heal_part.await
+                                };
+                                if let Err(e) = heal_part_result {
                                     // Don't leak the partially-written healed shards in
                                     // .rustfs/tmp when heal fails midway (backlog#799 B20).
                                     let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
@@ -1187,6 +1417,32 @@ impl SetDisks {
                                 );
                             }
                         }
+                        let _read_repair_write_lock_guard = if let Some(expected_fingerprint) = read_repair_commit_fingerprint {
+                            drop(_read_repair_read_lock_guard.take());
+                            match self
+                                .acquire_revalidated_read_repair_commit_lock(
+                                    &disks,
+                                    bucket,
+                                    object,
+                                    version_id,
+                                    expected_fingerprint,
+                                )
+                                .await
+                            {
+                                Ok(Some(guard)) => Some(guard),
+                                Ok(None) => {
+                                    let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
+                                    return Ok((result, Some(DiskError::Timeout)));
+                                }
+                                Err(err) => {
+                                    let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
+                                    return Err(err);
+                                }
+                            }
+                        } else {
+                            None
+                        };
+
                         // Rename from tmp location to the actual location.
                         // MinIO stops on the first RenameData error. RustFS intentionally
                         // continues per target, but reports any residue after all attempts
@@ -2377,7 +2633,10 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
 
 #[cfg(test)]
 mod heal_result_report_tests {
-    use super::{DanglingCheckPartsFailure, DanglingDeleteFailure, DanglingDeleteSafety, SetDisks, heal_writer_error_summary};
+    use super::{
+        DanglingCheckPartsFailure, DanglingDeleteFailure, DanglingDeleteSafety, ReadRepairCommitFingerprint,
+        ReadRepairCommitPauseScope, SetDisks, heal_writer_error_summary,
+    };
     use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope, HealWriterFailureScope};
     use crate::disk::endpoint::Endpoint;
     use crate::disk::error::DiskError;
@@ -2414,6 +2673,40 @@ mod heal_result_report_tests {
         assert!(!super::metadata_less_part_file("xl.meta"));
     }
 
+    #[test]
+    fn read_repair_commit_fingerprint_tracks_commit_identity_only() {
+        let data_dir = Uuid::parse_str("11111111-1111-1111-1111-111111111111").expect("data dir should parse");
+        let mut base = meta_regen_test_fileinfo("object.bin", data_dir, 42, 0);
+        base.volume = "bucket-a".to_string();
+        base.metadata.insert("etag".to_string(), "etag-a".to_string());
+
+        let mut observed = base.clone();
+        observed.idx = 3;
+        observed.fresh = true;
+        observed.num_versions = 7;
+        observed.erasure.index = base.erasure.index.saturating_add(1);
+        observed.successor_mod_time = Some(OffsetDateTime::from_unix_timestamp(43).expect("timestamp should parse"));
+
+        assert!(
+            ReadRepairCommitFingerprint::from_file_info(&base) == ReadRepairCommitFingerprint::from_file_info(&observed),
+            "read-observation and disk-local fields must not make an unchanged object look stale"
+        );
+
+        let mut overwritten = base.clone();
+        overwritten.data_dir = Some(Uuid::parse_str("22222222-2222-2222-2222-222222222222").expect("data dir should parse"));
+        assert!(
+            ReadRepairCommitFingerprint::from_file_info(&base) != ReadRepairCommitFingerprint::from_file_info(&overwritten),
+            "a new committed data directory must abort the stale read-repair commit"
+        );
+
+        let mut metadata_changed = base.clone();
+        metadata_changed.metadata.insert("etag".to_string(), "etag-b".to_string());
+        assert!(
+            ReadRepairCommitFingerprint::from_file_info(&base) != ReadRepairCommitFingerprint::from_file_info(&metadata_changed),
+            "metadata changes must abort the stale read-repair commit"
+        );
+    }
+
     #[derive(Clone, Default)]
     struct CapturedLogs {
         buffer: Arc<Mutex<Vec<u8>>>,
@@ -3981,78 +4274,159 @@ mod heal_result_report_tests {
         assert_eq!(survivor.size, 1024 * 1024, "survivor size must be intact");
     }
 
-    // HS-12 (backlog#1874): unversioned overwrite commits race a Deep heal on
-    // the same object. The overwrite's post-commit tail deletes the replaced
-    // data dir without the ns lock (object.rs commit tail), which is exactly
-    // the intersection the audit flagged: the heal must tolerate the tail race
-    // (retryable outcome) and every committed overwrite must survive — the
-    // final current version is exactly the last payload written.
     #[tokio::test]
     #[serial_test::serial]
-    async fn heal_racing_unversioned_overwrites_preserves_the_last_commit() {
+    async fn read_repair_commit_aborts_when_object_changes_before_final_rename() {
         let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
-        let bucket = "heal-race-put-overwrite";
+        let bucket = "read-repair-stale-commit";
         let object = "object.bin";
         set.make_bucket(bucket, &MakeBucketOptions::default())
             .await
             .expect("bucket should be created");
 
-        const ROUNDS: usize = 8;
-        const PAYLOAD_SIZE: usize = 256 * 1024;
-        let mut last_etag = String::new();
-        for round in 0..ROUNDS {
-            // Give the heal something to rebuild on alternating rounds: remove a
-            // shard of the current data dir right before the race.
-            if round % 2 == 1 {
-                let current = disks[2]
-                    .read_version("", bucket, object, "", &ReadOptions::default())
-                    .await
-                    .expect("current metadata should be readable");
-                if let Some(data_dir) = current.data_dir {
-                    let shard = temp_dirs[3]
-                        .path()
-                        .join(bucket)
-                        .join(object)
-                        .join(data_dir.to_string())
-                        .join("part.1");
-                    if shard.exists() {
-                        tokio::fs::remove_file(&shard)
-                            .await
-                            .expect("shard damage should be injectable mid-race");
-                    }
-                }
-            }
+        const PAYLOAD_SIZE: usize = 1024 * 1024;
+        let mut initial_reader = PutObjReader::from_vec(vec![0x11; PAYLOAD_SIZE]);
+        set.put_object(bucket, object, &mut initial_reader, &ObjectOptions::default())
+            .await
+            .expect("initial object should be written");
 
-            let payload = vec![round as u8; PAYLOAD_SIZE];
-            let mut put_reader = PutObjReader::from_vec(payload);
-            let put_opts = ObjectOptions::default();
-            let heal_opts = HealOpts {
-                scan_mode: HealScanMode::Deep,
-                ..Default::default()
-            };
-            let (put_res, heal_res) = tokio::join!(
-                set.put_object(bucket, object, &mut put_reader, &put_opts),
-                set.heal_object(bucket, object, "", &heal_opts),
-            );
-            let put_info = put_res.expect("overwrite must succeed under lock serialization");
-            last_etag = put_info.etag.clone().unwrap_or_default();
-            // Heal outcome is unconstrained (may hit the tail race and report a
-            // retryable error); the invariant is checked on the end state.
-            drop(heal_res);
-        }
+        let current = disks[2]
+            .read_version("", bucket, object, "", &ReadOptions::default())
+            .await
+            .expect("current metadata should be readable");
+        let data_dir = current.data_dir.expect("non-inline object should have a data dir");
+        let missing_part = temp_dirs
+            .iter()
+            .map(|dir| dir.path().join(bucket).join(object).join(data_dir.to_string()).join("part.1"))
+            .find(|part| part.is_file())
+            .expect("part.1 should exist on at least one erasure disk");
+        tokio::fs::remove_file(missing_part)
+            .await
+            .expect("shard damage should force read-repair data work");
+
+        let (_pause, arrived, release) = ReadRepairCommitPauseScope::install(bucket, object);
+        let heal_set = set.clone();
+        let heal_task = tokio::spawn(async move {
+            heal_set
+                .heal_object(
+                    bucket,
+                    object,
+                    "",
+                    &HealOpts {
+                        scan_mode: HealScanMode::Deep,
+                        read_repair: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+
+        tokio::time::timeout(std::time::Duration::from_secs(5), arrived.notified())
+            .await
+            .expect("read-repair should reach the final commit fence");
+
+        let mut overwrite_reader = PutObjReader::from_vec(vec![0x22; PAYLOAD_SIZE]);
+        let overwrite = set
+            .put_object(bucket, object, &mut overwrite_reader, &ObjectOptions::default())
+            .await
+            .expect("overwrite should commit while read-repair waits before final fence");
+        release.notify_one();
+
+        let (_result, err) = tokio::time::timeout(std::time::Duration::from_secs(5), heal_task)
+            .await
+            .expect("read-repair should finish after release")
+            .expect("read-repair task should join")
+            .expect("stale read-repair should report a typed retryable error");
+        assert_eq!(err, Some(DiskError::Timeout));
 
         let final_info = set
             .get_object_info(bucket, object, &ObjectOptions::default())
             .await
-            .expect("object must remain readable after the race loop");
-        assert_eq!(
-            final_info.size, PAYLOAD_SIZE as i64,
-            "final current version must be the last committed overwrite"
-        );
-        assert_eq!(
-            final_info.etag.unwrap_or_default(),
-            last_etag,
-            "the racing heal loop must never leave a stale or resurrected current version"
-        );
+            .expect("object must remain readable after stale read-repair abort");
+        assert_eq!(final_info.etag, overwrite.etag);
+    }
+
+    // HS-12 (backlog#1874): unversioned overwrite commits race a Deep heal on
+    // the same object. The overwrite's post-commit tail deletes the replaced
+    // data dir without the ns lock (object.rs commit tail), which is exactly
+    // the intersection the audit flagged: the heal must tolerate the tail race
+    // (retryable outcome) and every committed overwrite must survive — the
+    // final current version is exactly the last payload written. Also cover the
+    // read-repair path, which reconstructs under a shared data-phase lock and
+    // revalidates before the final rename.
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn heal_racing_unversioned_overwrites_preserves_the_last_commit() {
+        for read_repair in [false, true] {
+            let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
+            let bucket = if read_repair {
+                "read-repair-race-put-overwrite"
+            } else {
+                "heal-race-put-overwrite"
+            };
+            let object = "object.bin";
+            set.make_bucket(bucket, &MakeBucketOptions::default())
+                .await
+                .expect("bucket should be created");
+
+            const ROUNDS: usize = 8;
+            const PAYLOAD_SIZE: usize = 256 * 1024;
+            let mut last_etag = String::new();
+            for round in 0..ROUNDS {
+                // Give the heal something to rebuild on alternating rounds: remove a
+                // shard of the current data dir right before the race.
+                if round % 2 == 1 {
+                    let current = disks[2]
+                        .read_version("", bucket, object, "", &ReadOptions::default())
+                        .await
+                        .expect("current metadata should be readable");
+                    if let Some(data_dir) = current.data_dir {
+                        let shard = temp_dirs[3]
+                            .path()
+                            .join(bucket)
+                            .join(object)
+                            .join(data_dir.to_string())
+                            .join("part.1");
+                        if shard.exists() {
+                            tokio::fs::remove_file(&shard)
+                                .await
+                                .expect("shard damage should be injectable mid-race");
+                        }
+                    }
+                }
+
+                let payload = vec![round as u8; PAYLOAD_SIZE];
+                let mut put_reader = PutObjReader::from_vec(payload);
+                let put_opts = ObjectOptions::default();
+                let heal_opts = HealOpts {
+                    scan_mode: HealScanMode::Deep,
+                    read_repair,
+                    ..Default::default()
+                };
+                let (put_res, heal_res) = tokio::join!(
+                    set.put_object(bucket, object, &mut put_reader, &put_opts),
+                    set.heal_object(bucket, object, "", &heal_opts),
+                );
+                let put_info = put_res.expect("overwrite must succeed under lock serialization");
+                last_etag = put_info.etag.clone().unwrap_or_default();
+                // Heal outcome is unconstrained (may hit the tail race and report a
+                // retryable error); the invariant is checked on the end state.
+                drop(heal_res);
+            }
+
+            let final_info = set
+                .get_object_info(bucket, object, &ObjectOptions::default())
+                .await
+                .expect("object must remain readable after the race loop");
+            assert_eq!(
+                final_info.size, PAYLOAD_SIZE as i64,
+                "final current version must be the last committed overwrite"
+            );
+            assert_eq!(
+                final_info.etag.unwrap_or_default(),
+                last_etag,
+                "the racing heal loop must never leave a stale or resurrected current version"
+            );
+        }
     }
 }

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -395,7 +395,7 @@ async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartComm
         }
     };
     if let Some(barrier) = barrier
-        && let Ok(previous) = barrier.arrivals.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
+        && let Ok(previous) = barrier.arrivals.try_update(Ordering::AcqRel, Ordering::Acquire, |current| {
             (current < barrier.expected_arrivals).then_some(current + 1)
         })
     {

crates/ecstore/src/set_disk/ops/object.rs

@@ -66,6 +66,7 @@ use crate::bucket::lifecycle::bucket_lifecycle_ops::LifecycleOps;
 use crate::bucket::utils::is_meta_bucketname;
 use crate::bucket::versioning::VersioningApi;
 use crate::disk::DiskAPI;
+use crate::object_api::ScannerPublicationCommitScopeGuard;
 use crate::set_disk::coding;
 use crate::set_disk::core::io_primitives::GetCodecStreamingReaderBuildOutcome;
 use crate::set_disk::mem;
@@ -89,7 +90,10 @@ use tokio::io::AsyncWriteExt;
 const ENV_RUSTFS_GET_MID_SIZE_STREAMING_ENABLE: &str = "RUSTFS_GET_MID_SIZE_STREAMING_ENABLE";
 const DEFAULT_RUSTFS_GET_MID_SIZE_STREAMING_ENABLE: bool = true;
 const GET_MID_SIZE_STREAMING_MIN_SIZE: usize = 128 * 1024 + 1;
-const GET_MID_SIZE_STREAMING_MAX_SIZE: usize = 1024 * 1024;
+// Exclude 1 MiB from the bounded mid-size reader until it has a demonstrated
+// high-concurrency performance envelope; existing codec/legacy gates decide
+// which established reader handles the object.
+const GET_MID_SIZE_STREAMING_MAX_SIZE: usize = 512 * 1024;
 
 fn is_get_mid_size_streaming_enabled() -> bool {
     #[cfg(test)]
@@ -269,6 +273,22 @@ const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.js
 const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024;
 const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256;
 
+fn begin_scanner_publication_delete_mutation(scope: Option<&crate::object_api::ScannerPublicationCommitScope>) -> Result<()> {
+    let Some(scope) = scope else {
+        return Ok(());
+    };
+    if scope.state() == crate::object_api::ScannerPublicationCommitState::Admitted {
+        scope
+            .try_begin()
+            .map_err(|_| Error::other("scanner publication delete scope cannot start"))?;
+    }
+    if !scope.can_commit() {
+        let _ = scope.mark_indeterminate();
+        return Err(StorageError::OperationCanceled);
+    }
+    Ok(())
+}
+
 fn take_scanner_publication_lease_tokens(user_defined: &mut HashMap<String, String>) -> Result<Option<HashMap<String, Uuid>>> {
     let Some(encoded) = user_defined.remove(SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY) else {
         return Ok(None);
@@ -2624,6 +2644,10 @@ impl SetDisks {
         opts: &ObjectOptions,
     ) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
         crate::hp_guard!("SetDisks::put_object");
+        let mut scope_outcome_guard = opts
+            .scanner_publication_commit_scope
+            .clone()
+            .map(ScannerPublicationCommitScopeGuard::new);
         let storage_class_config = self.storage_class_config_snapshot();
         self.invalidate_get_object_metadata_cache(bucket, object).await;
 
@@ -3394,8 +3418,16 @@ impl SetDisks {
             let commit_tmp_dir = tmp_dir.clone();
             let commit_object_lock_guard = object_lock_guard.take();
             let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
-            let commit_allows_early_ack = commit_object_lock_guard.is_some();
-            let detach_commit_owner = commit_allows_early_ack || commit_bucket_lifecycle_guard.is_some() || quota_mutation_fence;
+            let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone();
+            // A scanner publication scope owns the movement permit until the
+            // complete rename fan-out drains. Keep this path synchronous so
+            // its terminal state is known before the coordinator releases
+            // remote leases.
+            let commit_allows_early_ack = commit_object_lock_guard.is_some() && commit_scanner_publication_scope.is_none();
+            let detach_commit_owner = commit_scanner_publication_scope.is_some()
+                || commit_allows_early_ack
+                || commit_bucket_lifecycle_guard.is_some()
+                || quota_mutation_fence;
             let commit_write_path_label = write_path.metric_label();
             let commit_is_versioned = opts.versioned || opts.version_suspended;
             let commit_versioned = opts.versioned;
@@ -3491,7 +3523,7 @@ impl SetDisks {
                     }
                     Ok(())
                 };
-                let pre_rename_result = if cancellation.is_some() || request_cancellation.is_some() {
+                let mut pre_rename_result = if cancellation.is_some() || request_cancellation.is_some() {
                     tokio::select! {
                         biased;
                         _ = wait_for_put_object_commit_cancellation(cancellation.as_ref(), request_cancellation.as_ref()) => {
@@ -3502,6 +3534,20 @@ impl SetDisks {
                 } else {
                     pre_rename.await
                 };
+                if pre_rename_result.is_ok()
+                    && let Some(scope) = commit_scanner_publication_scope.as_ref()
+                    && let Err(err) = scope.try_begin()
+                {
+                    let _ = scope.mark_aborted_before_commit();
+                    pre_rename_result = Err(Error::other(format!("scanner publication commit scope cannot start: {err:?}")));
+                }
+                if pre_rename_result.is_ok()
+                    && let Some(scope) = commit_scanner_publication_scope.as_ref()
+                    && !scope.can_commit()
+                {
+                    let _ = scope.mark_indeterminate();
+                    pre_rename_result = Err(StorageError::OperationCanceled);
+                }
                 if let Err(err) = pre_rename_result {
                     SetDisks::abort_quota_reservation_after_fence(
                         quota_reservation,
@@ -3537,9 +3583,17 @@ impl SetDisks {
                     crate::set_disk::core::io_primitives::RenameDataFenceOptions::new(
                         write_quorum,
                         commit_scanner_publication_lease_tokens.as_ref(),
-                    ),
+                    )
+                    .with_publication_scope(commit_scanner_publication_scope.clone()),
                 )
                 .await;
+                if let Some(scope) = commit_scanner_publication_scope.as_ref() {
+                    if rename_result.is_ok() {
+                        let _ = scope.mark_committed();
+                    } else {
+                        let _ = scope.mark_indeterminate();
+                    }
+                }
                 #[cfg(any(test, feature = "test-util"))]
                 if rename_result.is_ok() {
                     pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameQuorum).await;
@@ -3854,6 +3908,11 @@ impl SetDisks {
                 let _ = handoff.send(());
             }
             if detach_commit_owner {
+                if let Some(scope_outcome_guard) = scope_outcome_guard.as_mut() {
+                    // The spawned commit closure owns the scope clone and is
+                    // now responsible for its terminal outcome.
+                    scope_outcome_guard.disarm();
+                }
                 let mut cancellation = PutObjectCommitCancellation::new();
                 let child_token = cancellation.child_token();
                 let result = tokio::spawn(async move { Box::pin(commit(Some(child_token))).await })
@@ -3976,7 +4035,7 @@ impl<R: AsyncRead + Unpin> AsyncRead for TransitionUploadReader<R> {
                 let read =
                     u64::try_from(read).map_err(|_| std::io::Error::other("transition upload read count exceeds u64::MAX"))?;
                 self.consumed
-                    .fetch_update(Ordering::Release, Ordering::Relaxed, |consumed| consumed.checked_add(read))
+                    .try_update(Ordering::Release, Ordering::Relaxed, |consumed| consumed.checked_add(read))
                     .map_err(|_| std::io::Error::other("transition upload read count overflow"))?;
                 Poll::Ready(Ok(()))
             }
@@ -7049,8 +7108,13 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         (del_objects, del_errs, accounting)
     }
 
-    #[tracing::instrument(skip(self))]
+    #[tracing::instrument(skip(self, opts))]
     async fn delete_object(&self, bucket: &str, object: &str, mut opts: ObjectOptions) -> Result<ObjectInfo> {
+        let _scope_outcome_guard = opts
+            .scanner_publication_commit_scope
+            .clone()
+            .map(ScannerPublicationCommitScopeGuard::new);
+        let scanner_publication_commit_scope = opts.scanner_publication_commit_scope.clone();
         // Scanner cleanup carries the per-peer lease fence as transient
         // request metadata. Consume it before any delete-prefix fanout so it
         // cannot be persisted or treated as user metadata.
@@ -7145,6 +7209,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 }
                                 delete_request.set_skip_tier_free_version();
                             }
+                            begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
                             self.delete_object_version(bucket, object, &delete_request, false).await?;
                             if let Some((_, deleted_object)) = replication_delete {
                                 ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_object).await;
@@ -7159,6 +7224,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 ..Default::default()
                             };
                             delete_request.set_tier_free_version_id(&Uuid::new_v4().to_string());
+                            begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
                             self.delete_object_version(bucket, object, &delete_request, false).await?;
                         }
                         for version in &versions.free_versions {
@@ -7170,10 +7236,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 ..Default::default()
                             };
                             delete_request.set_tier_free_version();
+                            begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
                             self.delete_object_version(bucket, object, &delete_request, false).await?;
                         }
                     }
                 }
+                if let Some(scope) = scanner_publication_commit_scope.as_ref() {
+                    let _ = scope.mark_committed();
+                }
                 self.invalidate_get_object_metadata_cache(bucket, object).await;
                 return Ok(ObjectInfo::default());
             }
@@ -7181,10 +7251,19 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
             }
             ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
-            self.delete_prefix_with_scanner_publication_lease(bucket, object, scanner_publication_lease_tokens.as_ref())
-                .await
-                .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
+            begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
+            self.delete_prefix_with_scanner_publication_lease(
+                bucket,
+                object,
+                scanner_publication_lease_tokens.as_ref(),
+                scanner_publication_commit_scope.clone(),
+            )
+            .await
+            .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
 
+            if let Some(scope) = scanner_publication_commit_scope.as_ref() {
+                let _ = scope.mark_committed();
+            }
             self.invalidate_all_get_object_metadata_cache();
             return Ok(ObjectInfo::default());
         }
@@ -7257,10 +7336,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 ..Default::default()
             };
             ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
+            begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
             self.delete_object_version(bucket, object, &dfi, false)
                 .await
                 .map_err(|e| to_object_err(e, vec![bucket, object]))?;
             self.invalidate_get_object_metadata_cache(bucket, object).await;
+            if let Some(scope) = scanner_publication_commit_scope.as_ref() {
+                let _ = scope.mark_committed();
+            }
             return Ok(ObjectInfo::from_file_info(&dfi, bucket, object, opts.versioned || opts.version_suspended));
         }
 
@@ -7334,6 +7417,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             };
 
             ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
+            begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
             self.delete_object_version(bucket, object, &fi, should_force_delete_marker_for_missing_version(&opts))
                 .await
                 .map_err(|e| to_object_err(e, vec![bucket, object]))?;
@@ -7345,6 +7429,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             oi.user_tags = Arc::clone(&goi.user_tags);
             oi.replication_decision = goi.replication_decision;
             self.invalidate_get_object_metadata_cache(bucket, object).await;
+            if let Some(scope) = scanner_publication_commit_scope.as_ref() {
+                let _ = scope.mark_committed();
+            }
             return Ok(oi);
         }
 
@@ -7370,6 +7457,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         }
 
         ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
+        begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
         self.delete_object_version(bucket, object, &dfi, opts.delete_marker)
             .await
             .map_err(|e| to_object_err(e, vec![bucket, object]))?;
@@ -7395,6 +7483,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             obj_info.delete_marker = true;
         }
         self.invalidate_get_object_metadata_cache(bucket, object).await;
+        if let Some(scope) = scanner_publication_commit_scope.as_ref() {
+            let _ = scope.mark_committed();
+        }
         Ok(obj_info)
     }
 
@@ -8408,14 +8499,14 @@ mod mid_size_streaming_gate_tests {
 
     #[test]
     #[serial]
-    fn mid_size_streaming_includes_one_mib_and_rejects_larger_objects() {
-        let (object_info, fi) = plain_metadata(1024 * 1024);
+    fn mid_size_streaming_stops_at_512kib_and_rejects_one_mib() {
+        let (object_info, fi) = plain_metadata(512 * 1024);
         assert_eq!(
             get_mid_size_streaming_object_size_with_flags(&None, &object_info, &fi, &ObjectOptions::default(), true, true, true),
-            Some(1024 * 1024)
+            Some(512 * 1024)
         );
 
-        let (large_info, large_fi) = plain_metadata(1024 * 1024 + 1);
+        let (large_info, large_fi) = plain_metadata(512 * 1024 + 1);
         assert_eq!(
             get_mid_size_streaming_object_size_with_flags(
                 &None,
@@ -8428,6 +8519,20 @@ mod mid_size_streaming_gate_tests {
             ),
             None
         );
+
+        let (one_mib_info, one_mib_fi) = plain_metadata(1024 * 1024);
+        assert_eq!(
+            get_mid_size_streaming_object_size_with_flags(
+                &None,
+                &one_mib_info,
+                &one_mib_fi,
+                &ObjectOptions::default(),
+                true,
+                true,
+                true
+            ),
+            None
+        );
     }
 
     #[test]
@@ -9801,6 +9906,57 @@ mod inline_put_commit_path_tests {
         .await;
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn get_object_reader_routes_one_mib_away_from_mid_size_reader() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "one-mib-legacy-reader";
+        let object = "object.bin";
+        let payload = vec![0x5a; 1024 * 1024];
+        make_bucket(&disk_stores, bucket).await;
+        let storage_class = temp_env::with_var(INLINE_BLOCK_ENV, Some("1KiB"), || lookup_config_for_pools(&KVS::new(), &[4]))
+            .expect("test storage class should resolve");
+        set_disks.set_test_storage_class_config(storage_class);
+
+        let mut writer = PutObjReader::from_vec(payload.clone());
+        temp_env::async_with_vars(
+            [
+                (ENV_RUSTFS_GET_MID_SIZE_STREAMING_ENABLE, Some("true")),
+                (crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")),
+                (crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
+                (crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
+                (crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("off")),
+                (rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true")),
+            ],
+            async {
+                set_disks
+                    .put_object(bucket, object, &mut writer, &ObjectOptions::default())
+                    .await
+                    .expect("1 MiB fixture should commit");
+
+                crate::set_disk::reset_test_get_object_reader_path();
+                let mut reader = set_disks
+                    .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+                    .await
+                    .expect("1 MiB legacy GET should succeed");
+                let mut restored = Vec::new();
+                reader
+                    .stream
+                    .read_to_end(&mut restored)
+                    .await
+                    .expect("1 MiB legacy reader should stream");
+
+                assert_eq!(restored, payload);
+                assert_eq!(
+                    crate::set_disk::test_get_object_reader_path_id(),
+                    8,
+                    "1 MiB must bypass mid-size and use legacy duplex when codec rollout is off"
+                );
+            },
+        )
+        .await;
+    }
+
     #[tokio::test]
     async fn repeated_gets_reuse_the_set_erasure_shell() {
         let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;

crates/ecstore/src/store/list_objects.rs

@@ -659,7 +659,7 @@ pub(crate) fn observe_scanner_namespace_mutations(bucket: &str, delta: u64) {
     }
 
     let _ = SCANNER_NAMESPACE_MUTATION_GENERATION
-        .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(delta)));
+        .try_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(delta)));
 }
 
 pub(crate) async fn observe_list_objects_mutation(store: &ECStore, bucket: &str) -> u64 {

crates/ecstore/src/store/mod.rs

@@ -58,7 +58,7 @@ use crate::{
     core::sets::Sets,
     disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET},
     layout::endpoints::EndpointServerPools,
-    object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
+    object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, ScannerPublicationCommitScope},
 };
 use futures::future::join_all;
 use http::HeaderMap;
@@ -275,11 +275,13 @@ pub struct ECStore {
 
 impl std::fmt::Debug for ECStore {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let disk_slot_count: usize = self.disk_map.values().map(Vec::len).sum();
+
         f.debug_struct("ECStore")
             .field("id", &self.id)
-            .field("disk_map", &self.disk_map)
-            .field("pools", &self.pools)
-            .field("pool_meta", &self.pool_meta)
+            .field("disk_map_pool_count", &self.disk_map.len())
+            .field("disk_slot_count", &disk_slot_count)
+            .field("pool_count", &self.pools.len())
             .finish_non_exhaustive()
     }
 }
@@ -520,6 +522,48 @@ impl ECStore {
         Some((operation_guard, self.ctx.data_movement_operation_epoch()))
     }
 
+    /// Acquire a storage-owned scanner publication scope. Unlike the legacy
+    /// admission helper, the movement permit is owned by the returned scope
+    /// and therefore survives cancellation of the scanner coordinator while
+    /// the actual metadata mutation drains.
+    pub async fn scanner_data_usage_publication_commit_scope(
+        &self,
+        expected_movement_epoch: u64,
+        safe_deadline: tokio::time::Instant,
+        remote_lease_tokens: Vec<Uuid>,
+    ) -> Option<ScannerPublicationCommitScope> {
+        let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
+        if epoch != expected_movement_epoch {
+            return None;
+        }
+        Some(ScannerPublicationCommitScope::new_storage_owned(
+            epoch,
+            safe_deadline,
+            remote_lease_tokens,
+            movement_permit,
+        ))
+    }
+
+    pub async fn scanner_data_usage_publication_commit_scope_with_release_flag(
+        &self,
+        expected_movement_epoch: u64,
+        safe_deadline: tokio::time::Instant,
+        remote_lease_tokens: Vec<Uuid>,
+        lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
+    ) -> Option<ScannerPublicationCommitScope> {
+        let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
+        if epoch != expected_movement_epoch {
+            return None;
+        }
+        Some(ScannerPublicationCommitScope::new_storage_owned_with_release_flag(
+            epoch,
+            safe_deadline,
+            remote_lease_tokens,
+            movement_permit,
+            lease_release_safe,
+        ))
+    }
+
     /// Capture the current publication epoch without holding the movement
     /// gate across backend I/O. Callers must re-admit the same epoch before a
     /// mutation commits.
@@ -1154,6 +1198,7 @@ mod tests {
     use super::*;
     use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
     use crate::layout::endpoints::{Endpoints, PoolEndpoints, SetupType};
+    use crate::object_api::ObjectOptions;
     use crate::runtime::global::reset_local_disk_test_state;
     use crate::runtime::sources::{clear_local_disk_id_map_for_test, local_disk_path_by_id};
     use crate::store::init_format::{connect_load_init_formats, init_disks};
@@ -1170,6 +1215,81 @@ mod tests {
         assert!(infos.iter().all(|info| info.is_none()));
     }
 
+    #[test]
+    fn ecstore_debug_is_bounded_summary() {
+        let endpoint_pools = EndpointServerPools::default();
+        let ctx = Arc::new(InstanceContext::new());
+        let store = ECStore {
+            id: uuid::Uuid::new_v4(),
+            disk_map: [(0, vec![None, None, None, None])].into_iter().collect(),
+            pools: Vec::new(),
+            peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
+            pool_meta: RwLock::new(PoolMeta::default()),
+            rebalance_meta: RwLock::new(None),
+            decommission_cancelers: RwLock::new(Vec::new()),
+            start_gate: Mutex::new(()),
+            pool_meta_save_gate: Mutex::default(),
+            ctx,
+            bucket_fence_registry: Arc::default(),
+        };
+
+        let rendered = format!("{store:?}");
+
+        assert!(rendered.len() < 256, "ECStore Debug should stay bounded: {rendered}");
+        assert!(rendered.contains("disk_map_pool_count"));
+        assert!(rendered.contains("disk_slot_count"));
+        assert!(!rendered.contains("disk_map:"));
+        assert!(!rendered.contains("pools:"));
+        assert!(!rendered.contains("pool_meta"));
+        assert!(!rendered.contains("format.json"));
+        assert!(!rendered.contains("TimedActionSlot"));
+        assert!(!rendered.contains("DiskHealthTracker"));
+    }
+
+    #[test]
+    fn object_options_debug_does_not_expand_tier_store_handle() {
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        let mut opts = ObjectOptions {
+            version_id: Some("large-version-id".repeat(1024)),
+            expected_current_version_id: Some("large-expected-version-id".repeat(1024)),
+            preserve_etag: Some("large-etag".repeat(1024)),
+            http_preconditions: Some(crate::storage_api_contracts::object::HTTPPreconditions {
+                if_match: Some("large-if-match".repeat(1024)),
+                if_none_match: Some("large-if-none-match".repeat(1024)),
+                ..Default::default()
+            }),
+            tier_delete_journal_api: Some(store),
+            ..Default::default()
+        };
+        opts.user_defined.insert("large-user-metadata".to_owned(), "x".repeat(8192));
+        opts.eval_metadata = Some([("large-eval-metadata".to_owned(), "y".repeat(8192))].into_iter().collect());
+        opts.transition.status = "large-transition-status".repeat(1024);
+        opts.transition.tier = "large-transition-tier".repeat(1024);
+        opts.lifecycle_audit_event.event.rule_id = "large-rule-id".repeat(1024);
+        opts.lifecycle_audit_event.event.storage_class = "large-storage-class".repeat(1024);
+
+        let rendered = format!("{opts:?}");
+
+        assert!(rendered.len() < 4096, "ObjectOptions Debug should stay bounded: {rendered}");
+        assert!(rendered.contains("tier_delete_journal_api: true"));
+        assert!(rendered.contains("user_defined_count: 1"));
+        assert!(rendered.contains("eval_metadata_count: Some(1)"));
+        assert!(!rendered.contains("ECStore {"));
+        assert!(!rendered.contains("disk_map"));
+        assert!(!rendered.contains("large-version-id"));
+        assert!(!rendered.contains("large-expected-version-id"));
+        assert!(!rendered.contains("large-etag"));
+        assert!(!rendered.contains("large-if-match"));
+        assert!(!rendered.contains("large-transition"));
+        assert!(!rendered.contains("large-rule-id"));
+        assert!(!rendered.contains("large-storage-class"));
+        assert!(!rendered.contains("large-user-metadata"));
+        assert!(!rendered.contains("large-eval-metadata"));
+        assert!(!rendered.contains("format.json"));
+        assert!(!rendered.contains("TimedActionSlot"));
+        assert!(!rendered.contains("DiskHealthTracker"));
+    }
+
     // Build a minimal ECStore carrying an explicit instance context. Empty
     // pools/disks are sufficient: the Phase 5 accessors read only `self.ctx`.
     fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
@@ -1331,9 +1451,29 @@ mod tests {
             .await
             .expect("movement writer should proceed after lease expiry")
             .expect("expiry writer task should not panic");
+        assert!(
+            store.validate_scanner_publication_lease(expiring_token, 0).await.is_err(),
+            "an expired lease must not validate after its read guard is released"
+        );
         assert!(!store.release_scanner_publication_lease(expiring_token).await);
     }
 
+    #[tokio::test]
+    async fn scanner_publication_lease_rejects_a_new_movement_generation() {
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        let (token, generation) = store
+            .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
+            .await
+            .expect("an idle store should grant a publication lease");
+
+        assert_eq!(store.ctx.advance_data_movement_generation(), Some(1));
+        assert!(
+            store.validate_scanner_publication_lease(token, generation).await.is_err(),
+            "a lease from the prior movement generation must fail closed"
+        );
+        assert!(store.release_scanner_publication_lease(token).await);
+    }
+
     #[tokio::test]
     async fn scanner_publication_lease_rejects_stale_generation_before_install() {
         let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
@@ -1344,6 +1484,102 @@ mod tests {
         assert!(error.to_string().contains("generation is stale"));
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn scanner_publication_commit_scope_owns_permit_until_terminal_drain() {
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        let scope = store
+            .scanner_data_usage_publication_commit_scope(
+                0,
+                tokio::time::Instant::now() + Duration::from_secs(30),
+                vec![Uuid::new_v4()],
+            )
+            .await
+            .expect("idle storage should grant a publication scope");
+        assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::Admitted);
+        assert_eq!(scope.remote_lease_tokens().len(), 1);
+
+        let gate = store.ctx.data_movement_operation_gate();
+        let writer = tokio::spawn(async move { gate.write_owned().await });
+        tokio::task::yield_now().await;
+        assert!(!writer.is_finished(), "the scope must own its movement permit after the caller returns");
+
+        scope.cancel();
+        assert!(scope.mark_aborted_before_commit());
+        assert_eq!(
+            scope.wait_for_completion().await,
+            crate::object_api::ScannerPublicationCommitState::AbortedBeforeCommit
+        );
+        assert!(scope.release_movement_permit().await);
+        tokio::time::timeout(Duration::from_secs(1), writer)
+            .await
+            .expect("movement writer should proceed after the scope drains")
+            .expect("movement writer task should not panic");
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn scanner_publication_commit_scope_rejects_late_start_and_keeps_indeterminate_permit() {
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        let scope = store
+            .scanner_data_usage_publication_commit_scope(0, tokio::time::Instant::now() + Duration::from_secs(1), Vec::new())
+            .await
+            .expect("idle storage should grant a publication scope");
+        tokio::time::advance(Duration::from_secs(1)).await;
+        assert_eq!(
+            scope.try_begin(),
+            Err(crate::object_api::ScannerPublicationCommitStartError::DeadlineExceeded)
+        );
+        assert!(
+            !scope.release_movement_permit().await,
+            "an admitted scope is not safe to release before owner resolution"
+        );
+        assert!(scope.mark_aborted_before_commit());
+        assert!(scope.release_movement_permit().await);
+
+        let scope = store
+            .scanner_data_usage_publication_commit_scope(0, tokio::time::Instant::now() + Duration::from_secs(30), Vec::new())
+            .await
+            .expect("a second idle publication scope should be granted");
+        scope.try_begin().expect("scope should enter the mutation state");
+        scope.cancel();
+        assert!(scope.mark_indeterminate());
+        assert_eq!(
+            scope.wait_for_completion().await,
+            crate::object_api::ScannerPublicationCommitState::Indeterminate
+        );
+        assert!(!scope.release_movement_permit().await, "indeterminate mutation must retain the permit");
+    }
+
+    #[tokio::test]
+    async fn scanner_publication_scope_guard_classifies_early_returns_conservatively() {
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        let permit = store.ctx.data_movement_operation_gate().read_owned().await;
+        let scope = ScannerPublicationCommitScope::new_storage_owned(
+            0,
+            tokio::time::Instant::now() + Duration::from_secs(30),
+            Vec::new(),
+            permit,
+        );
+        {
+            let _guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
+        }
+        assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::AbortedBeforeCommit);
+        assert!(scope.release_movement_permit().await);
+
+        let permit = store.ctx.data_movement_operation_gate().read_owned().await;
+        let scope = ScannerPublicationCommitScope::new_storage_owned(
+            0,
+            tokio::time::Instant::now() + Duration::from_secs(30),
+            Vec::new(),
+            permit,
+        );
+        scope.try_begin().expect("scope should enter the mutation state");
+        {
+            let _guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
+        }
+        assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::Indeterminate);
+        assert!(!scope.release_movement_permit().await);
+    }
+
     #[tokio::test]
     async fn scanner_target_guard_keeps_movement_writer_fenced_after_lease_release() {
         let store = build_store_with_ctx(Arc::new(InstanceContext::new()));

crates/heal-contracts/Cargo.toml

@@ -33,6 +33,7 @@ tokio = { workspace = true, features = ["sync"] }
 uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
 
 [dev-dependencies]
+serde_json = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
 
 [lib]

crates/heal-contracts/src/heal_channel.rs

@@ -213,6 +213,8 @@ pub struct HealOpts {
     pub update_parity: bool,
     #[serde(rename = "nolock")]
     pub no_lock: bool,
+    #[serde(rename = "readRepair", default)]
+    pub read_repair: bool,
     #[serde(rename = "pool", default)]
     pub pool: Option<usize>,
     #[serde(rename = "set", default)]
@@ -672,6 +674,24 @@ mod tests {
         assert_eq!(request.source, HealRequestSource::Internal);
     }
 
+    #[test]
+    fn heal_opts_deserializes_missing_read_repair_as_false() {
+        let opts: HealOpts = serde_json::from_str(
+            r#"{
+                "recursive": false,
+                "dryRun": false,
+                "remove": false,
+                "recreate": false,
+                "scanMode": "normal",
+                "updateParity": false,
+                "nolock": false
+            }"#,
+        )
+        .expect("old heal options without readRepair should decode");
+
+        assert!(!opts.read_repair);
+    }
+
     #[test]
     fn heal_admission_result_labels_are_stable() {
         assert_eq!(HealAdmissionResult::Accepted.result_label(), "accepted");

crates/heal/src/heal/storage.rs

@@ -689,6 +689,7 @@ impl HealStorageAPI for ECStoreHealStorage {
             scan_mode: HealScanMode::Deep,
             update_parity: true,
             no_lock: false,
+            read_repair: false,
             pool: None,
             set: None,
         };

crates/heal/src/heal/task/heal_bucket.rs

@@ -95,6 +95,7 @@ impl HealTask {
             scan_mode: self.options.scan_mode,
             update_parity: self.options.update_parity,
             no_lock: self.options.no_lock,
+            read_repair: false,
             pool: self.options.pool_index,
             set: self.options.set_index,
         };
@@ -262,6 +263,7 @@ impl HealTask {
             scan_mode: self.options.scan_mode,
             update_parity: self.options.update_parity,
             no_lock: self.options.no_lock,
+            read_repair: false,
             pool: self.options.pool_index,
             set: self.options.set_index,
         };

crates/heal/src/heal/task/heal_erasure_set.rs

@@ -356,6 +356,7 @@ impl HealTask {
             scan_mode: self.options.scan_mode,
             update_parity: self.options.update_parity,
             no_lock: self.options.no_lock,
+            read_repair: false,
             pool: self.options.pool_index,
             set: self.options.set_index,
         };
@@ -411,6 +412,7 @@ impl HealTask {
             scan_mode: self.options.scan_mode,
             update_parity: self.options.update_parity,
             no_lock: self.options.no_lock,
+            read_repair: false,
             pool: self.options.pool_index,
             set: self.options.set_index,
         };

crates/heal/src/heal/task/heal_metadata.rs

@@ -97,6 +97,7 @@ impl HealTask {
             scan_mode: HealScanMode::Deep,
             update_parity: false,
             no_lock: self.options.no_lock,
+            read_repair: false,
             pool: self.options.pool_index,
             set: self.options.set_index,
         };
@@ -259,6 +260,7 @@ impl HealTask {
             scan_mode: HealScanMode::Deep,
             update_parity: true,
             no_lock: self.options.no_lock,
+            read_repair: false,
             pool: None,
             set: None,
         };

crates/heal/src/heal/task/heal_object.rs

@@ -158,13 +158,22 @@ impl HealTask {
             scan_mode: self.options.scan_mode,
             update_parity: self.options.update_parity,
             no_lock: self.options.no_lock,
+            read_repair: self.source == HealRequestSource::ReadRepair,
             pool: self.options.pool_index,
             set: self.options.set_index,
         };
 
-        let heal_result = self
-            .await_with_control(self.storage.heal_object(bucket, object, version_id, &heal_opts))
-            .await;
+        let heal_fut = self.storage.heal_object(bucket, object, version_id, &heal_opts);
+        let heal_result = if self.source == HealRequestSource::ReadRepair {
+            let result = heal_fut.await;
+            if self.cancel_token.is_cancelled() {
+                Err(Error::TaskCancelled)
+            } else {
+                result
+            }
+        } else {
+            self.await_with_control(heal_fut).await
+        };
 
         match heal_result {
             Ok((result, error)) => {
@@ -375,6 +384,7 @@ impl HealTask {
             scan_mode: HealScanMode::Deep,
             update_parity: true,
             no_lock: self.options.no_lock,
+            read_repair: false,
             pool: None,
             set: None,
         };

crates/heal/src/heal/task/tests.rs

@@ -1061,6 +1061,72 @@ async fn scoped_object_heal_slowdown_is_not_treated_as_deleted() {
         .expect("heal options lock should be available");
     assert_eq!(opts[0].pool, Some(0));
     assert_eq!(opts[0].set, Some(1));
+    assert!(!opts[0].read_repair);
+}
+
+#[tokio::test]
+async fn read_repair_object_heal_sets_read_repair_option() {
+    let storage = Arc::new(MockStorage::default());
+    let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), Some("version-a".to_string()));
+    request.source = HealRequestSource::ReadRepair;
+    let task = HealTask::from_request(request, storage.clone());
+
+    task.execute().await.expect("read-repair object heal should complete");
+
+    let opts = storage.object_heal_opts.lock().unwrap();
+    assert_eq!(opts.len(), 1);
+    assert!(opts[0].read_repair);
+    assert!(!opts[0].no_lock);
+}
+
+#[tokio::test]
+async fn read_repair_object_heal_is_not_failed_by_flat_task_timeout() {
+    let storage = Arc::new(MockStorage {
+        block_heal_object: Mutex::new(true),
+        ..Default::default()
+    });
+    let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
+    request.source = HealRequestSource::ReadRepair;
+    request.options.timeout = Some(Duration::from_millis(1));
+    let task = Arc::new(HealTask::from_request(request, storage.clone()));
+    let execution = tokio::spawn({
+        let task = task.clone();
+        async move { task.execute().await }
+    });
+
+    tokio::time::timeout(Duration::from_secs(1), async {
+        loop {
+            if !storage.object_heal_opts.lock().unwrap().is_empty() {
+                break;
+            }
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("read-repair object heal should start");
+    tokio::time::sleep(Duration::from_millis(20)).await;
+    assert!(!execution.is_finished(), "read repair must not be failed by the flat task timeout");
+
+    execution.abort();
+    assert!(execution.await.is_err(), "aborted mock execution should not join successfully");
+    assert!(storage.object_heal_opts.lock().unwrap()[0].read_repair);
+}
+
+#[tokio::test]
+async fn non_read_repair_object_heal_still_uses_flat_timeout() {
+    let storage = Arc::new(MockStorage {
+        block_heal_object: Mutex::new(true),
+        ..Default::default()
+    });
+    let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
+    request.options.timeout = Some(Duration::from_millis(1));
+    let task = HealTask::from_request(request, storage);
+
+    let result = tokio::time::timeout(Duration::from_secs(1), task.execute())
+        .await
+        .expect("flat timeout should finish the task");
+
+    assert!(matches!(result, Err(Error::TaskTimeout)));
 }
 
 async fn make_resume_disk(temp: &TempDir) -> DiskStore {

crates/heal/tests/heal_integration_test.rs

@@ -497,6 +497,7 @@ mod serial_tests {
             scan_mode: HealScanMode::Normal,
             update_parity: false,
             no_lock: false,
+            read_repair: false,
             pool: None,
             set: None,
         };
@@ -518,6 +519,7 @@ mod serial_tests {
             scan_mode: HealScanMode::Normal,
             update_parity: false,
             no_lock: false,
+            read_repair: false,
             pool: None,
             set: None,
         };

crates/iam/src/lib.rs

@@ -131,7 +131,7 @@ pub(crate) async fn notify_iam_load_user(access_key: &str, temp: bool) -> Vec<Ia
         assert!(!probe.panic, "notification probe panic");
         let should_fail = probe
             .remaining_failures
-            .fetch_update(std::sync::atomic::Ordering::SeqCst, std::sync::atomic::Ordering::SeqCst, |remaining| {
+            .try_update(std::sync::atomic::Ordering::SeqCst, std::sync::atomic::Ordering::SeqCst, |remaining| {
                 remaining.checked_sub(1)
             })
             .is_ok();

crates/io-core/src/backpressure.rs

@@ -227,7 +227,7 @@ impl BackpressureMonitor {
         // usize::MAX, which would permanently reject all future acquisitions.
         let prev = match self
             .current
-            .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| current.checked_sub(1))
+            .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| current.checked_sub(1))
         {
             Ok(prev) => prev,
             Err(_) => {

crates/io-core/src/pool.rs

@@ -448,7 +448,7 @@ impl PoolTier {
         if let Some(buffer) = buffer {
             let released_bytes = buffer.capacity() as u64;
             self.tier_current_allocated_bytes
-                .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
+                .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
                     Some(current.saturating_sub(released_bytes))
                 })
                 .ok();
@@ -457,7 +457,7 @@ impl PoolTier {
             {
                 metrics
                     .current_allocated_bytes
-                    .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
+                    .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
                         Some(current.saturating_sub(released_bytes))
                     })
                     .ok();

crates/io-metrics/src/process_lock_metrics.rs

@@ -69,12 +69,12 @@ pub fn record_write_lock_held_acquire() {
 
 #[inline(always)]
 pub fn record_read_lock_held_release() {
-    let _ = READ_LOCKS_HELD.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| Some(value.saturating_sub(1)));
+    let _ = READ_LOCKS_HELD.try_update(Ordering::Relaxed, Ordering::Relaxed, |value| Some(value.saturating_sub(1)));
 }
 
 #[inline(always)]
 pub fn record_write_lock_held_release() {
-    let _ = WRITE_LOCKS_HELD.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| Some(value.saturating_sub(1)));
+    let _ = WRITE_LOCKS_HELD.try_update(Ordering::Relaxed, Ordering::Relaxed, |value| Some(value.saturating_sub(1)));
 }
 
 #[inline(always)]

crates/kms/src/service_manager.rs

@@ -259,6 +259,25 @@ impl KmsServiceManager {
         (state.status.clone(), config)
     }
 
+    /// Publish an initialization failure when no usable KMS state exists yet.
+    ///
+    /// Startup configuration discovery happens outside this crate. Recording
+    /// its failure here keeps status truthful without allowing a late failure
+    /// to replace an already configured or running service.
+    pub async fn record_initialization_error(&self, message: impl Into<String>) {
+        let _guard = self.lifecycle_mutex.lock().await;
+        let current = self.state.load_full();
+        if current.config.is_some() || current.current_service.is_some() {
+            return;
+        }
+
+        self.state.store(Arc::new(RuntimeState {
+            config: None,
+            status: KmsServiceStatus::Error(message.into()),
+            current_service: None,
+        }));
+    }
+
     fn redact_config(config: &mut KmsConfig) {
         if let BackendConfig::Static(static_config) = &mut config.backend_config {
             use zeroize::Zeroize;
@@ -849,6 +868,39 @@ mod tests {
         assert!(manager.get_encryption_service().await.is_none());
     }
 
+    #[tokio::test]
+    async fn initialization_error_is_visible_until_configuration_succeeds() {
+        let manager = KmsServiceManager::new();
+
+        manager
+            .record_initialization_error("persisted configuration could not be loaded")
+            .await;
+
+        assert_eq!(
+            manager.get_status().await,
+            KmsServiceStatus::Error("persisted configuration could not be loaded".to_string())
+        );
+        assert!(manager.get_config().await.is_none());
+
+        manager
+            .configure(static_config("key-a", 0x11))
+            .await
+            .expect("configure after startup failure");
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Configured);
+    }
+
+    #[tokio::test]
+    async fn initialization_error_never_replaces_a_running_service() {
+        let manager = KmsServiceManager::new();
+        manager.configure(static_config("key-a", 0x11)).await.expect("configure");
+        manager.start().await.expect("start");
+
+        manager.record_initialization_error("late startup failure").await;
+
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
+        assert!(manager.get_encryption_service().await.is_some());
+    }
+
     #[tokio::test]
     async fn configure_rejects_running_service_without_changing_snapshot() {
         let manager = KmsServiceManager::new();

crates/lock/src/namespace/tests.rs

@@ -181,7 +181,7 @@ impl crate::client::LockClient for FlakyAcquireClient {
         self.acquire_attempts.fetch_add(1, Ordering::SeqCst);
         if self
             .failed_acquires_remaining
-            .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| remaining.checked_sub(1))
+            .try_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| remaining.checked_sub(1))
             .is_ok()
         {
             return Ok(LockResponse::failure("Lock acquisition timeout", request.acquire_timeout));
@@ -254,7 +254,7 @@ impl crate::client::LockClient for FlakyReleaseClient {
         self.release_attempts.fetch_add(1, Ordering::SeqCst);
         if self
             .failed_releases_remaining
-            .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| remaining.checked_sub(1))
+            .try_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| remaining.checked_sub(1))
             .is_ok()
         {
             return Ok(false);

crates/object-data-cache/src/memory.rs

@@ -192,7 +192,7 @@ impl MemorySnapshotCell {
         }
         if self
             .pending_release
-            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |pending| pending.checked_add(bytes))
+            .try_update(Ordering::AcqRel, Ordering::Acquire, |pending| pending.checked_add(bytes))
             .is_err()
         {
             self.release_accounting_failed.store(true, Ordering::Release);

crates/object-data-cache/src/moka_backend.rs

@@ -590,7 +590,7 @@ impl MokaBackend {
         // the synchronization, so relaxed ordering is sufficient here.
         let generation = match self
             .next_generation
-            .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| current.checked_add(1))
+            .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| current.checked_add(1))
         {
             Ok(previous) => previous + 1,
             Err(_) => return leader.finish(ObjectDataCacheFillResult::SkippedIdentityOverflow),

crates/protos/src/generated/proto_gen/node_service.rs

@@ -1553,6 +1553,9 @@ pub enum ControlPlaneErrorCode {
     /// The peer answered but its storage/IAM layer is not initialized yet
     /// (legacy string form: "errServerNotInitialized").
     ControlPlaneErrorNotInitialized = 1,
+    /// The peer rejected a control-plane request before changing durable state.
+    /// error_info carries the actionable validation reason.
+    ControlPlaneErrorInvalidArgument = 2,
 }
 impl ControlPlaneErrorCode {
     /// String value of the enum field names used in the ProtoBuf definition.
@@ -1563,6 +1566,7 @@ impl ControlPlaneErrorCode {
         match self {
             Self::ControlPlaneErrorUnspecified => "CONTROL_PLANE_ERROR_UNSPECIFIED",
             Self::ControlPlaneErrorNotInitialized => "CONTROL_PLANE_ERROR_NOT_INITIALIZED",
+            Self::ControlPlaneErrorInvalidArgument => "CONTROL_PLANE_ERROR_INVALID_ARGUMENT",
         }
     }
     /// Creates an enum from field names used in the ProtoBuf definition.
@@ -1570,6 +1574,7 @@ impl ControlPlaneErrorCode {
         match value {
             "CONTROL_PLANE_ERROR_UNSPECIFIED" => Some(Self::ControlPlaneErrorUnspecified),
             "CONTROL_PLANE_ERROR_NOT_INITIALIZED" => Some(Self::ControlPlaneErrorNotInitialized),
+            "CONTROL_PLANE_ERROR_INVALID_ARGUMENT" => Some(Self::ControlPlaneErrorInvalidArgument),
             _ => None,
         }
     }

crates/protos/src/node.proto

@@ -30,6 +30,9 @@ enum ControlPlaneErrorCode {
   // The peer answered but its storage/IAM layer is not initialized yet
   // (legacy string form: "errServerNotInitialized").
   CONTROL_PLANE_ERROR_NOT_INITIALIZED = 1;
+  // The peer rejected a control-plane request before changing durable state.
+  // error_info carries the actionable validation reason.
+  CONTROL_PLANE_ERROR_INVALID_ARGUMENT = 2;
 }
 
 message PingRequest {

crates/replication/src/config.rs

@@ -60,15 +60,17 @@ pub const REPLICATION_READ_ONLY_HISTORICAL_FIELDS: &[&str] = &[
     "Destination.ReplicationTime",
 ];
 
-// v2: disableProxy moved from unsupported to writable (per-target read-proxy
-// opt-out is accepted by set-remote-target and the `proxy` update op).
-pub const REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION: u32 = 2;
+// v4: temporary-credential fields moved from read-only historical metadata to
+// writable fields because remote targets now use them for request signing.
+pub const REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION: u32 = 4;
 
 pub const REMOTE_TARGET_WRITABLE_FIELDS: &[&str] = &[
     "sourcebucket",
     "endpoint",
     "credentials.accessKey",
     "credentials.secretKey",
+    "credentials.sessionToken",
+    "credentials.expiration",
     "targetbucket",
     "secure",
     "path",
@@ -90,6 +92,13 @@ pub const REMOTE_TARGET_WRITABLE_FIELDS: &[&str] = &[
     "disableProxy",
 ];
 
+/// Remote target fields that are readable for persisted-data compatibility but
+/// cannot be written through the admin API.
+///
+/// The empty slice remains public for source compatibility with consumers of
+/// the v3 capability API.
+pub const REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS: &[&str] = &[];
+
 pub const REMOTE_TARGET_UNSUPPORTED_FIELDS: &[&str] = &["edge", "edgeSyncBeforeExpiry"];
 
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]

crates/replication/src/delete.rs

@@ -108,6 +108,20 @@ pub fn is_version_delete_replication(dobj: &DeletedObject) -> bool {
     dobj.version_id.is_some() || (dobj.delete_marker_version_id.is_some() && !dobj.delete_marker)
 }
 
+/// Whether the target DELETE for this delete replication may use
+/// marker-CREATION semantics (`replication_delete_marker`, which omits the
+/// `versionId` query so the target mints its own marker).
+///
+/// A version purge must never do so: the DELETE has to address the exact
+/// version, otherwise a generic S3 target — which ignores the internal
+/// source-version headers — mints a fresh delete marker on every retry
+/// instead of removing one (rustfs#6823). The `delete_marker` flag alone is
+/// not enough because heal/resync rebuilds carry `delete_marker: true`
+/// together with a purge-shaped entry.
+pub fn delete_replication_creates_marker(dobj: &DeletedObject) -> bool {
+    dobj.delete_marker && !is_version_delete_replication(dobj)
+}
+
 pub fn should_retry_delete_marker_purge(dobj: &DeletedObject) -> bool {
     dobj.delete_marker_version_id.is_some()
 }
@@ -223,13 +237,14 @@ mod tests {
 
     use super::{
         DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
-        is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome,
-        should_retry_delete_marker_purge, target_delete_version_id,
+        delete_replication_creates_marker, is_retryable_delete_replication_head_error, is_version_delete_replication,
+        replicate_delete_outcome, resync_existing_delete_replication_info, should_retry_delete_marker_purge,
+        target_delete_version_id,
     };
     use crate::storage_api::DeletedObject;
     use crate::{
-        MrfOpKind, NULL_VERSION_ID, ReplicationState, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation,
-        VersionPurgeStatusType,
+        MrfOpKind, NULL_VERSION_ID, ReplicateObjectInfo, ReplicationState, ReplicationStatusType, ReplicationType,
+        ReplicationWorkerOperation, VersionPurgeStatusType,
     };
     use uuid::Uuid;
 
@@ -400,6 +415,80 @@ mod tests {
         assert!(!is_version_delete_replication(&dobj));
     }
 
+    /// rustfs#6823 regression guard: a purge-shaped entry must never take
+    /// marker-creation semantics, which would drop the `versionId` from the
+    /// target DELETE and let a generic S3 target mint a fresh delete marker
+    /// on every heal/resync/MRF retry.
+    #[test]
+    fn purge_shapes_never_take_marker_creation_semantics() {
+        // Heal/resync rebuild of a delete-marker version purge: the entry
+        // carries `delete_marker: true` together with the purged version id.
+        let heal_purge = DeletedObject {
+            delete_marker: true,
+            version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+        assert!(
+            !delete_replication_creates_marker(&heal_purge),
+            "a version purge must address the version, not mint a marker"
+        );
+
+        // Live delete-marker version purge addressed via the marker id.
+        let marker_purge = DeletedObject {
+            delete_marker: false,
+            delete_marker_version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+        assert!(!delete_replication_creates_marker(&marker_purge));
+
+        // Only a plain delete-marker creation may let the target mint one.
+        let marker_creation = DeletedObject {
+            delete_marker: true,
+            delete_marker_version_id: Some(Uuid::new_v4()),
+            ..Default::default()
+        };
+        assert!(delete_replication_creates_marker(&marker_creation));
+
+        let versionless = DeletedObject {
+            delete_marker: false,
+            ..Default::default()
+        };
+        assert!(!delete_replication_creates_marker(&versionless));
+    }
+
+    /// The resync scan rebuilds purge work items with `delete_marker: true`
+    /// (crates/replication/src/delete.rs `resync_existing_delete_replication_info`);
+    /// pin that this shape flows into purge — not marker-creation — semantics.
+    #[test]
+    fn resync_rebuilt_purge_entry_keeps_versioned_delete_semantics() {
+        let roi = ReplicateObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "object".to_string(),
+            version_id: Some(Uuid::new_v4()),
+            delete_marker: true,
+            version_purge_status: VersionPurgeStatusType::Pending,
+            ..Default::default()
+        };
+
+        let info = resync_existing_delete_replication_info(&roi, "arn:target-a");
+
+        assert!(is_version_delete_replication(&info.delete_object));
+        assert!(
+            !delete_replication_creates_marker(&info.delete_object),
+            "a rebuilt purge must not re-mint delete markers on the target"
+        );
+
+        // Without a pending purge the rebuild is a marker creation again.
+        let roi = ReplicateObjectInfo {
+            delete_marker: true,
+            version_id: Some(Uuid::new_v4()),
+            version_purge_status: VersionPurgeStatusType::Empty,
+            ..roi
+        };
+        let info = resync_existing_delete_replication_info(&roi, "arn:target-a");
+        assert!(delete_replication_creates_marker(&info.delete_object));
+    }
+
     #[test]
     fn delete_marker_purge_retry_covers_version_purge_and_marker_creation() {
         let version_purge = DeletedObject {

crates/replication/src/lib.rs

@@ -29,19 +29,21 @@ mod storage_api;
 pub mod tagging;
 
 pub use config::{
-    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
-    REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
-    REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
-    active_replication_rule_destination_arns, assign_site_replication_rule_priorities, invalid_replication_config_status_field,
-    is_reconciler_owned_site_replication_rule, is_site_replication_role, is_site_replication_rule,
-    merge_incoming_replication_config, merge_user_replication_config, replication_target_arn_deployment_id,
-    replication_target_arns, should_remove_replication_target, site_replication_rule_deployment_id,
-    unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
+    ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS,
+    REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION,
+    REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError,
+    ReplicationConfigurationExt, ReplicationTargetValidationError, active_replication_rule_destination_arns,
+    assign_site_replication_rule_priorities, invalid_replication_config_status_field, is_reconciler_owned_site_replication_rule,
+    is_site_replication_role, is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
+    replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
+    site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
+    validate_replication_config_target_arns,
 };
 pub use delete::{
     DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
-    is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome,
-    resync_existing_delete_replication_info, should_retry_delete_marker_purge, target_delete_version_id,
+    delete_replication_creates_marker, is_retryable_delete_replication_head_error, is_version_delete_replication,
+    replicate_delete_outcome, resync_existing_delete_replication_info, should_retry_delete_marker_purge,
+    target_delete_version_id,
 };
 pub use filemeta::{
     NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING,

crates/replication/src/stats.rs

@@ -265,7 +265,7 @@ fn saturating_atomic_sub(value: &AtomicI64, delta: i64) {
         return;
     }
 
-    let _ = value.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(delta).max(0)));
+    let _ = value.try_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(delta).max(0)));
 }
 
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]

crates/rio/src/etag_reader.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use crate::compress_index::{Index, TryGetIndex};
-use crate::{EtagResolvable, HashReaderDetector, HashReaderMut};
+use crate::{BadDigest, EtagResolvable, HashReaderDetector, HashReaderMut};
 use md5::{Digest, Md5};
 use pin_project_lite::pin_project;
 use std::pin::Pin;
@@ -89,7 +89,13 @@ where
                     && *checksum != etag
                 {
                     error!("Checksum mismatch, expected={:?}, actual={:?}", checksum, etag);
-                    return Poll::Ready(Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Checksum mismatch")));
+                    return Poll::Ready(Err(std::io::Error::new(
+                        std::io::ErrorKind::InvalidData,
+                        BadDigest {
+                            expected_md5: checksum.clone(),
+                            calculated_md5: etag,
+                        },
+                    )));
                 }
             }
         }
@@ -255,6 +261,7 @@ mod tests {
     async fn test_etag_reader_checksum_mismatch() {
         let data = b"checksum test data";
         let wrong_checksum = "deadbeefdeadbeefdeadbeefdeadbeef".to_string();
+        let calculated_md5 = hex_simd::encode_to_string(Md5::digest(data), hex_simd::AsciiCase::Lower);
         let reader = BufReader::new(&data[..]);
         let mut etag_reader = EtagReader::new(reader, Some(wrong_checksum.clone()));
 
@@ -262,5 +269,11 @@ mod tests {
         // Verification failed, should return InvalidData error
         let err = etag_reader.read_to_end(&mut buf).await.unwrap_err();
         assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+        let digest = err
+            .get_ref()
+            .and_then(|source| source.downcast_ref::<BadDigest>())
+            .expect("checksum mismatch should preserve the BadDigest type");
+        assert_eq!(digest.expected_md5, wrong_checksum);
+        assert_eq!(digest.calculated_md5, calculated_md5);
     }
 }

crates/s3select-api/src/query/session.rs

@@ -231,7 +231,7 @@ impl QueryExecutionTrackerInner {
 
     fn mark_timed_out(&self) -> Option<u8> {
         self.state
-            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |state| {
+            .try_update(Ordering::AcqRel, Ordering::Acquire, |state| {
                 (state < EXECUTION_FINISHED).then_some(EXECUTION_TIMED_OUT)
             })
             .ok()

crates/scanner/src/lib.rs

@@ -39,11 +39,11 @@ use storage_api::owner::{
     EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts, EcstoreReplicationConfigurationExt,
     EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore,
     EcstoreVersioningApi, HTTPPreconditions, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete,
-    ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
-    ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
-    ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_invalidate_admin_data_usage_snapshot_cache,
-    ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure, ecstore_is_erasure_sd,
-    ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
+    ScannerPublicationCommitScope, ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission,
+    ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr,
+    ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config,
+    ecstore_invalidate_admin_data_usage_snapshot_cache, ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure,
+    ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
     ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
     ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
     scanner_replication_config_for_lifecycle_eval,
@@ -55,6 +55,7 @@ use storage_api::owner::{
     ecstore_new_disk,
 };
 use tokio_util::sync::CancellationToken;
+use uuid::Uuid;
 
 pub mod data_usage_define;
 pub mod error;
@@ -176,7 +177,7 @@ impl Default for ForegroundReadGuard {
 impl Drop for ForegroundReadGuard {
     fn drop(&mut self) {
         let _ =
-            SCANNER_FOREGROUND_STREAM_READS.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| current.checked_sub(1));
+            SCANNER_FOREGROUND_STREAM_READS.try_update(Ordering::Relaxed, Ordering::Relaxed, |current| current.checked_sub(1));
     }
 }
 
@@ -206,7 +207,7 @@ impl ScannerRuntimeGuard {
 
 impl Drop for ScannerRuntimeGuard {
     fn drop(&mut self) {
-        let _ = SCANNER_RUNTIME_INSTANCES.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| current.checked_sub(1));
+        let _ = SCANNER_RUNTIME_INSTANCES.try_update(Ordering::Relaxed, Ordering::Relaxed, |current| current.checked_sub(1));
     }
 }
 
@@ -217,8 +218,8 @@ fn reset_scanner_runtime_instances_for_test() {
 
 impl Drop for ScannerActivityGuard {
     fn drop(&mut self) {
-        let _ = SCANNER_ACTIVE_WORK_UNITS
-            .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1)));
+        let _ =
+            SCANNER_ACTIVE_WORK_UNITS.try_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1)));
     }
 }
 
@@ -499,7 +500,7 @@ pub(crate) async fn runtime_tier_registry() -> TierRegistrySnapshot {
 
 fn next_tier_registry_generation() -> u64 {
     TIER_REGISTRY_GENERATION
-        .fetch_update(Ordering::AcqRel, Ordering::Relaxed, |current| Some(current.saturating_add(1)))
+        .try_update(Ordering::AcqRel, Ordering::Relaxed, |current| Some(current.saturating_add(1)))
         .unwrap_or(u64::MAX)
 }
 
@@ -752,6 +753,32 @@ pub(crate) fn scanner_publication_epoch_changed(error: &EcstoreError) -> bool {
     )
 }
 
+pub(crate) async fn delete_config_with_publication_scope_for_epoch<S>(
+    api: Arc<S>,
+    bucket: &str,
+    object: &str,
+    mut opts: ScannerObjectOptions,
+    expected_epoch: u64,
+    scanner_publication_commit_scope: Option<ScannerPublicationCommitScope>,
+) -> EcstoreResult<ScannerObjectInfo>
+where
+    S: ScannerObjectIO + ScannerConfigObjectDelete,
+{
+    let legacy_admission = if scanner_publication_commit_scope.is_none() {
+        Some(
+            scanner_publication_admission_for_epoch(api.clone(), expected_epoch)
+                .await
+                .ok_or_else(|| EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED))?,
+        )
+    } else {
+        None
+    };
+    opts.scanner_publication_commit_scope = scanner_publication_commit_scope;
+    let result = api.delete_config_object(bucket, object, opts).await;
+    drop(legacy_admission);
+    result
+}
+
 pub(crate) async fn delete_config_with_publication_admission_for_epoch<S>(
     api: Arc<S>,
     bucket: &str,
@@ -762,10 +789,7 @@ pub(crate) async fn delete_config_with_publication_admission_for_epoch<S>(
 where
     S: ScannerObjectIO + ScannerConfigObjectDelete,
 {
-    let Some(_admission) = scanner_publication_admission_for_epoch(api.clone(), expected_epoch).await else {
-        return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
-    };
-    api.delete_config_object(bucket, object, opts).await
+    delete_config_with_publication_scope_for_epoch(api, bucket, object, opts, expected_epoch, None).await
 }
 
 /// Capture the storage-owned publication epoch without retaining the read
@@ -796,13 +820,14 @@ where
     Some(admission)
 }
 
-pub(crate) async fn save_config_shared_with_preconditions_and_lease_fence<S>(
+pub(crate) async fn save_config_shared_with_preconditions_and_lease_fence_and_scope<S>(
     api: Arc<S>,
     file: &str,
     data: Bytes,
     sha256hex: Option<String>,
     preconditions: HTTPPreconditions,
     scanner_publication_lease_fence: Option<&str>,
+    scanner_publication_commit_scope: Option<ScannerPublicationCommitScope>,
 ) -> EcstoreResult<ScannerObjectInfo>
 where
     S: ScannerObjectIO,
@@ -822,6 +847,7 @@ where
         &ScannerObjectOptions {
             max_parity: true,
             http_preconditions: Some(preconditions),
+            scanner_publication_commit_scope,
             user_defined,
             ..Default::default()
         },
@@ -886,6 +912,27 @@ pub trait ScannerConfigObjectDelete: Send + Sync + std::fmt::Debug + 'static {
     async fn scanner_data_usage_publication_admission(&self) -> Option<ScannerDataUsagePublicationAdmission> {
         None
     }
+
+    /// Acquire a storage-owned scope for a fenced scanner metadata mutation.
+    /// Implementations without a storage movement owner fail closed.
+    async fn scanner_data_usage_publication_commit_scope(
+        &self,
+        _expected_movement_epoch: u64,
+        _safe_deadline: tokio::time::Instant,
+        _remote_lease_tokens: Vec<Uuid>,
+    ) -> Option<ScannerPublicationCommitScope> {
+        None
+    }
+
+    async fn scanner_data_usage_publication_commit_scope_with_release_flag(
+        &self,
+        _expected_movement_epoch: u64,
+        _safe_deadline: tokio::time::Instant,
+        _remote_lease_tokens: Vec<Uuid>,
+        _lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
+    ) -> Option<ScannerPublicationCommitScope> {
+        None
+    }
 }
 
 pub struct ScannerDataUsagePublicationAdmission {
@@ -929,6 +976,32 @@ impl ScannerConfigObjectDelete for ECStore {
         let (read_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
         Some(ScannerDataUsagePublicationAdmission::fenced(read_guard, epoch))
     }
+
+    async fn scanner_data_usage_publication_commit_scope(
+        &self,
+        expected_movement_epoch: u64,
+        safe_deadline: tokio::time::Instant,
+        remote_lease_tokens: Vec<Uuid>,
+    ) -> Option<ScannerPublicationCommitScope> {
+        self.scanner_data_usage_publication_commit_scope(expected_movement_epoch, safe_deadline, remote_lease_tokens)
+            .await
+    }
+
+    async fn scanner_data_usage_publication_commit_scope_with_release_flag(
+        &self,
+        expected_movement_epoch: u64,
+        safe_deadline: tokio::time::Instant,
+        remote_lease_tokens: Vec<Uuid>,
+        lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
+    ) -> Option<ScannerPublicationCommitScope> {
+        self.scanner_data_usage_publication_commit_scope_with_release_flag(
+            expected_movement_epoch,
+            safe_deadline,
+            remote_lease_tokens,
+            lease_release_safe,
+        )
+        .await
+    }
 }
 
 #[async_trait::async_trait]
@@ -946,6 +1019,32 @@ impl ScannerConfigObjectDelete for SetDisks {
         let (read_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
         Some(ScannerDataUsagePublicationAdmission::fenced(read_guard, epoch))
     }
+
+    async fn scanner_data_usage_publication_commit_scope(
+        &self,
+        expected_movement_epoch: u64,
+        safe_deadline: tokio::time::Instant,
+        remote_lease_tokens: Vec<Uuid>,
+    ) -> Option<ScannerPublicationCommitScope> {
+        self.scanner_data_usage_publication_commit_scope(expected_movement_epoch, safe_deadline, remote_lease_tokens)
+            .await
+    }
+
+    async fn scanner_data_usage_publication_commit_scope_with_release_flag(
+        &self,
+        expected_movement_epoch: u64,
+        safe_deadline: tokio::time::Instant,
+        remote_lease_tokens: Vec<Uuid>,
+        lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
+    ) -> Option<ScannerPublicationCommitScope> {
+        self.scanner_data_usage_publication_commit_scope_with_release_flag(
+            expected_movement_epoch,
+            safe_deadline,
+            remote_lease_tokens,
+            lease_release_safe,
+        )
+        .await
+    }
 }
 
 #[cfg(test)]

crates/scanner/src/scanner.rs

@@ -16,6 +16,7 @@ use std::collections::BTreeMap;
 use std::future::Future;
 #[cfg(test)]
 use std::sync::Mutex as StdMutex;
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, LazyLock, RwLock};
 
 use self::heal_info::{BackgroundHealInfoReadStatus, read_background_heal_info_with_epoch, save_background_heal_info_for_epoch};
@@ -62,8 +63,8 @@ use tokio::time::{Duration, Instant};
 use tokio_util::sync::CancellationToken;
 use tokio_util::task::AbortOnDropHandle;
 use tracing::{debug, error, info, instrument, warn};
+use uuid::Uuid;
 
-use crate::storage_api::owner::SCANNER_PUBLICATION_LEASE_TTL_MS;
 use crate::storage_api::scan::{
     BucketOperations, BucketOptions, NamespaceLocking as _, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
     SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
@@ -72,7 +73,7 @@ use crate::{
     ECStore, EcstoreError, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerLifecycleConfigExt as _,
     ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch, get_lifecycle_config,
     get_replication_config, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, read_config,
-    replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions_and_lease_fence,
+    replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions_and_lease_fence_and_scope,
     save_config_with_preconditions, save_config_with_publication_admission_for_epoch, scanner_is_erasure_sd,
     scanner_publication_admission_for_epoch, scanner_publication_epoch, scanner_publication_epoch_changed,
 };
@@ -454,6 +455,7 @@ fn data_usage_backup_due(data_usage_info: &DataUsageInfo) -> bool {
 }
 
 #[cfg(test)]
+#[allow(dead_code)]
 async fn sync_data_usage_backup_from_primary(
     ctx: &CancellationToken,
     storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
@@ -461,12 +463,34 @@ async fn sync_data_usage_backup_from_primary(
     sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(ctx, storeapi, None, None, None).await
 }
 
+#[allow(dead_code)]
 async fn sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
     ctx: &CancellationToken,
     storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
     expected_publication_epoch: Option<u64>,
     remote_lease_deadline: Option<std::time::Instant>,
     scanner_publication_lease_fence: Option<&str>,
+) -> Result<(), EcstoreError> {
+    sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence_and_scope(
+        ctx,
+        storeapi,
+        expected_publication_epoch,
+        remote_lease_deadline,
+        scanner_publication_lease_fence,
+        Vec::new(),
+        Arc::new(AtomicBool::new(true)),
+    )
+    .await
+}
+
+async fn sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence_and_scope(
+    ctx: &CancellationToken,
+    storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
+    expected_publication_epoch: Option<u64>,
+    remote_lease_deadline: Option<std::time::Instant>,
+    scanner_publication_lease_fence: Option<&str>,
+    remote_lease_tokens: Vec<Uuid>,
+    lease_release_safe: Arc<AtomicBool>,
 ) -> Result<(), EcstoreError> {
     let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
     for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
@@ -531,15 +555,48 @@ async fn sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
                 }
                 return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
             };
-            save_config_shared_with_preconditions_and_lease_fence(
+            let publication_scope = match expected_publication_epoch {
+                Some(expected_epoch) => {
+                    storeapi
+                        .scanner_data_usage_publication_commit_scope_with_release_flag(
+                            expected_epoch,
+                            usage_store::scanner_publication_scope_deadline(data_usage_persist_timeout(), remote_lease_deadline),
+                            remote_lease_tokens.clone(),
+                            Arc::clone(&lease_release_safe),
+                        )
+                        .await
+                }
+                None => None,
+            };
+            if expected_publication_epoch.is_some() && publication_scope.is_none() {
+                if retry < SCANNER_PERSIST_CAS_RETRIES {
+                    continue;
+                }
+                return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
+            }
+            let save_result = save_config_shared_with_preconditions_and_lease_fence_and_scope(
                 storeapi.clone(),
                 &backup_path,
                 primary.clone(),
                 sha256hex,
                 revision.preconditions(),
                 scanner_publication_lease_fence,
+                publication_scope.clone(),
             )
-            .await
+            .await;
+            if let Some(scope) = publication_scope {
+                match scope.wait_for_completion().await {
+                    crate::storage_api::owner::ScannerPublicationCommitState::Committed
+                    | crate::storage_api::owner::ScannerPublicationCommitState::AbortedBeforeCommit => save_result,
+                    crate::storage_api::owner::ScannerPublicationCommitState::Indeterminate
+                    | crate::storage_api::owner::ScannerPublicationCommitState::Admitted
+                    | crate::storage_api::owner::ScannerPublicationCommitState::InFlight => Err(EcstoreError::other(
+                        "scanner backup publication commit scope did not reach a safe terminal state",
+                    )),
+                }
+            } else {
+                save_result
+            }
         };
 
         match save_result {
@@ -1416,13 +1473,10 @@ async fn run_data_scanner_cycle_with_budget(
         mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
         return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
     };
-    let usage_persist_baseline_result = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await;
+    let usage_persist_baseline_result = read_data_usage_persist_baseline(storeapi.clone()).await;
     drop(baseline_publication_guard);
     let usage_persist_baseline = match usage_persist_baseline_result {
-        Ok((data, revision)) => DataUsagePersistBaseline {
-            data: data.map(Bytes::from),
-            revision,
-        },
+        Ok(baseline) => baseline,
         Err(err) => {
             error!(
                 target: "rustfs::scanner",
@@ -1462,10 +1516,23 @@ async fn run_data_scanner_cycle_with_budget(
         {
             Some(ScannerCycleDeferReason::DataMovement)
         }
+        // A complete walk can still be retained as an observational snapshot
+        // when only the final activity proof was unavailable.  It must not
+        // block the observation receiver: the authoritative publication
+        // fence remains enforced by the usage store and the cycle is advanced
+        // as partial without acknowledging dirty usage.
+        Ok(result)
+            if result.has_observational_snapshot()
+                && matches!(
+                    result.status,
+                    ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
+                ) =>
+        {
+            None
+        }
         Ok(result) => final_data_usage_publication_defer_reason(storeapi.as_ref(), result.status).await,
         Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
     };
-    let publication_deferred = publication_defer_reason.is_some();
     let publication_epoch = scan_result.as_ref().ok().and_then(ScannerCycleResult::publication_epoch);
     let remote_publication_lease_targets = if publication_defer_reason.is_none() {
         scan_result
@@ -1479,21 +1546,24 @@ async fn run_data_scanner_cycle_with_budget(
     let mut remote_publication_leases = None;
     let remote_lease_defer_reason = if remote_publication_lease_targets.is_empty() {
         None
-    } else if usage_persist_timeout >= Duration::from_millis(SCANNER_PUBLICATION_LEASE_TTL_MS) {
-        // The lease is intentionally fixed-duration and has no renewal path.
-        // Refuse a persistence budget that could outlive it instead of
-        // allowing the peer to admit movement while a local PUT is in flight.
-        Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
     } else if let Some(notification_system) = storeapi.notification_system() {
-        match notification_system
-            .acquire_scanner_publication_leases(remote_publication_lease_targets.clone())
-            .await
-        {
-            Ok(grants) => {
+        let publication_proof_ctx = cycle_budget.token();
+        let lease_result = await_scanner_publication_proof(
+            &publication_proof_ctx,
+            cycle_info.current,
+            "lease_acquire",
+            || notification_system.acquire_scanner_publication_leases(remote_publication_lease_targets.clone()),
+            |err| scanner_publication_lease_error_is_retryable(&err.to_string()),
+        )
+        .await;
+        match lease_result {
+            ScannerPublicationProofWait::Ready(grants) => {
                 remote_publication_leases = Some((notification_system, grants));
                 None
             }
-            Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
+            ScannerPublicationProofWait::Rejected(_) | ScannerPublicationProofWait::Cancelled => {
+                Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
+            }
         }
     } else {
         Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
@@ -1534,21 +1604,16 @@ async fn run_data_scanner_cycle_with_budget(
         remote_lease_fence.is_some(),
     ))
     .then_some(ScannerCycleDeferReason::ActivityBaselineUnavailable);
-    let remote_lease_covers_persistence = remote_lease_deadline.is_none_or(|deadline| {
-        std::time::Instant::now()
-            .checked_add(usage_persist_timeout)
-            .is_some_and(|latest_finish| latest_finish < deadline)
-    });
     let publication_defer_reason = publication_defer_reason
         .or(remote_lease_defer_reason)
         .or(remote_lease_fence_defer_reason);
-    let publication_defer_reason = (!remote_lease_covers_persistence)
-        .then_some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
-        .or(publication_defer_reason);
+    // Include reasons discovered while acquiring or validating remote leases.
+    let publication_deferred = publication_defer_reason.is_some();
     let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
     let remote_lease_probe = remote_publication_leases
         .as_ref()
         .map(|(notification_system, grants)| (Arc::clone(notification_system), grants.clone()));
+    let remote_lease_release_safe = Arc::new(AtomicBool::new(true));
     let mut usage_persist_outcome = match publication_defer_reason {
         Some(reason) => {
             drop(receiver);
@@ -1562,6 +1627,11 @@ async fn run_data_scanner_cycle_with_budget(
             let ctx_clone = ctx.clone();
             let route_probe_store = storeapi.clone();
             let remote_lease_fence = remote_lease_fence.clone();
+            let remote_lease_release_safe_for_task = Arc::clone(&remote_lease_release_safe);
+            let remote_lease_tokens = remote_publication_leases
+                .as_ref()
+                .map(|(_, grants)| grants.iter().map(|grant| grant.lease.token).collect())
+                .unwrap_or_default();
             let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
                 store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
                     ctx_clone,
@@ -1573,7 +1643,9 @@ async fn run_data_scanner_cycle_with_budget(
                         publication_epoch,
                         remote_lease_deadline,
                         remote_lease_fence,
-                    ),
+                    )
+                    .with_remote_lease_tokens(remote_lease_tokens)
+                    .with_lease_release_flag(remote_lease_release_safe_for_task),
                     move || {
                         let storeapi = route_probe_store.clone();
                         let remote_lease_probe = remote_lease_probe.clone();
@@ -1638,16 +1710,28 @@ async fn run_data_scanner_cycle_with_budget(
     let lease_expired = remote_publication_leases
         .as_ref()
         .is_some_and(|(_, grants)| grants.iter().any(|grant| !grant.lease.is_valid()));
-    if let Some((notification_system, grants)) = remote_publication_leases.take() {
+    if !remote_lease_release_safe.load(Ordering::Acquire) {
+        // A cancelled or detached storage mutation did not report a safe
+        // terminal state. Keep remote grants until their own expiry rather
+        // than releasing movement admission while a commit may be unknown.
+        usage_persist_outcome = if usage_persist_outcome == DataUsagePersistOutcome::Failed {
+            DataUsagePersistOutcome::Failed
+        } else {
+            DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded)
+        };
+    } else if let Some((notification_system, grants)) = remote_publication_leases.take() {
         let release_result = notification_system.release_scanner_publication_leases(grants).await;
-        if lease_expired || release_result.is_err() {
+        let lease_release_failed = release_result.is_err();
+        if lease_expired || lease_release_failed {
             // A lease that expired or could not be released is never treated
             // as a successful authoritative publication. The peer may have
             // admitted movement immediately after the lease ended.
             usage_persist_outcome = if usage_persist_outcome == DataUsagePersistOutcome::Failed {
                 DataUsagePersistOutcome::Failed
+            } else if lease_release_failed {
+                DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseReleaseFailed)
             } else {
-                DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
+                DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded)
             };
         }
     }
@@ -2803,12 +2887,7 @@ fn finalize_scanner_cycle_result(
     scan_cycle_result: crate::scanner_io::ScannerCycleResult,
     usage_persist_outcome: DataUsagePersistOutcome,
 ) -> (ScannerCycleOutcome, bool, Vec<ScannerDirtyUsageAcknowledgement>) {
-    let completion_outcome = scanner_cycle_completion_outcome(
-        scan_cycle_result.status,
-        usage_persist_outcome,
-        scan_cycle_result.has_dirty_usage_to_acknowledge(),
-        scan_cycle_result.has_failed_dirty_usage(),
-    );
+    let completion_outcome = scanner_cycle_completion_outcome_for_result(&scan_cycle_result, usage_persist_outcome);
     let pending_maintenance_work = scan_cycle_result.has_pending_maintenance_work();
     let durable_complete_snapshot = scan_cycle_result.status == ScannerCycleStatus::Complete
         && matches!(
@@ -2823,6 +2902,34 @@ fn finalize_scanner_cycle_result(
     (completion_outcome, pending_maintenance_work, remote_dirty_usage_acknowledgements)
 }
 
+fn scanner_cycle_completion_outcome_for_result(
+    scan_cycle_result: &crate::scanner_io::ScannerCycleResult,
+    usage_persist_outcome: DataUsagePersistOutcome,
+) -> ScannerCycleOutcome {
+    let has_dirty_usage = scan_cycle_result.has_dirty_usage_to_acknowledge();
+    let has_failed_dirty_usage = scan_cycle_result.has_failed_dirty_usage();
+    if scan_cycle_result.has_observational_snapshot()
+        && matches!(
+            scan_cycle_result.status,
+            ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
+        )
+    {
+        return match usage_persist_outcome {
+            DataUsagePersistOutcome::Saved
+            | DataUsagePersistOutcome::AlreadyDurable
+            | DataUsagePersistOutcome::PriorCycleDurable
+            | DataUsagePersistOutcome::Current
+                if !has_failed_dirty_usage =>
+            {
+                ScannerCycleOutcome::Partial
+            }
+            DataUsagePersistOutcome::Deferred(reason) => ScannerCycleOutcome::Deferred(reason),
+            _ => ScannerCycleOutcome::Failed,
+        };
+    }
+    scanner_cycle_completion_outcome(scan_cycle_result.status, usage_persist_outcome, has_dirty_usage, has_failed_dirty_usage)
+}
+
 /// Decide whether an incoming usage snapshot must be skipped as stale, given the local
 /// wall clock `now`. Mirrors `stale_data_usage_persist_reason` in
 /// `crates/ecstore/src/data_usage/mod.rs` — keep the two consistent.

crates/scanner/src/scanner/activity.rs

@@ -108,6 +108,149 @@ impl ScannerRetryBackoff {
     }
 }
 
+const SCANNER_PUBLICATION_PROOF_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30);
+
+pub(crate) fn scanner_publication_proof_retry_delay(consecutive_failures: u32) -> Duration {
+    let exponent = consecutive_failures.saturating_sub(1).min(31);
+    let multiplier = 1u32.checked_shl(exponent).unwrap_or(u32::MAX);
+    SCANNER_RETRY_BASE_INTERVAL
+        .saturating_mul(multiplier)
+        .min(SCANNER_PUBLICATION_PROOF_RETRY_MAX_INTERVAL)
+}
+
+pub(crate) fn scanner_publication_activity_error_is_retryable(error: &str) -> bool {
+    crate::storage_api::scanner_peer_transport_error_message_is_retryable(error)
+}
+
+pub(crate) fn scanner_publication_lease_error_is_retryable(error: &str) -> bool {
+    scanner_publication_activity_error_is_retryable(error)
+        || error.ends_with("scanner publication lease capacity is exhausted")
+        || error.ends_with("scanner publication lease response arrived after its safety window")
+}
+
+pub(crate) enum ScannerPublicationProofWait<T, E> {
+    Ready(T),
+    Rejected(E),
+    Cancelled,
+}
+
+pub(crate) async fn await_scanner_publication_proof<T, E, F, Fut, Retryable>(
+    ctx: &CancellationToken,
+    cycle: u64,
+    stage: &'static str,
+    mut proof: F,
+    retryable: Retryable,
+) -> ScannerPublicationProofWait<T, E>
+where
+    F: FnMut() -> Fut,
+    Fut: Future<Output = Result<T, E>>,
+    E: std::fmt::Display,
+    Retryable: Fn(&E) -> bool,
+{
+    let started_at = Instant::now();
+    let mut consecutive_failures = 0u32;
+
+    loop {
+        if ctx.is_cancelled() {
+            return ScannerPublicationProofWait::Cancelled;
+        }
+
+        match proof().await {
+            Ok(value) => {
+                if consecutive_failures > 0 {
+                    info!(
+                        target: "rustfs::scanner",
+                        event = EVENT_SCANNER_CYCLE_STATE,
+                        component = LOG_COMPONENT_SCANNER,
+                        subsystem = LOG_SUBSYSTEM_RUNTIME,
+                        state = "publication_proof_recovered",
+                        cycle,
+                        stage,
+                        attempts = consecutive_failures.saturating_add(1),
+                        pending_duration = ?started_at.elapsed(),
+                        "Scanner publication proof recovered"
+                    );
+                }
+                return ScannerPublicationProofWait::Ready(value);
+            }
+            Err(err) if !retryable(&err) => {
+                warn!(
+                    target: "rustfs::scanner",
+                    event = EVENT_SCANNER_CYCLE_STATE,
+                    component = LOG_COMPONENT_SCANNER,
+                    subsystem = LOG_SUBSYSTEM_RUNTIME,
+                    state = "publication_proof_rejected",
+                    cycle,
+                    stage,
+                    error = %err,
+                    "Scanner publication proof failed with a non-retryable cluster state"
+                );
+                return ScannerPublicationProofWait::Rejected(err);
+            }
+            Err(err) => {
+                consecutive_failures = consecutive_failures.saturating_add(1);
+                let retry_delay = scanner_publication_proof_retry_delay(consecutive_failures);
+                if consecutive_failures == 1 || consecutive_failures.is_multiple_of(20) {
+                    warn!(
+                        target: "rustfs::scanner",
+                        event = EVENT_SCANNER_CYCLE_STATE,
+                        component = LOG_COMPONENT_SCANNER,
+                        subsystem = LOG_SUBSYSTEM_RUNTIME,
+                        state = "publication_proof_pending",
+                        cycle,
+                        stage,
+                        attempt = consecutive_failures,
+                        retry_delay = ?retry_delay,
+                        error = %err,
+                        "Scanner retained a completed scan while publication proof is unavailable"
+                    );
+                } else {
+                    debug!(
+                        target: "rustfs::scanner",
+                        event = EVENT_SCANNER_CYCLE_STATE,
+                        component = LOG_COMPONENT_SCANNER,
+                        subsystem = LOG_SUBSYSTEM_RUNTIME,
+                        state = "publication_proof_retry",
+                        cycle,
+                        stage,
+                        attempt = consecutive_failures,
+                        retry_delay = ?retry_delay,
+                        error = %err,
+                        "Scanner publication activity proof retry scheduled"
+                    );
+                }
+
+                tokio::select! {
+                    _ = ctx.cancelled() => return ScannerPublicationProofWait::Cancelled,
+                    _ = tokio::time::sleep(retry_delay) => {}
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+pub(crate) async fn await_scanner_publication_activity<F, Fut>(
+    ctx: &CancellationToken,
+    cycle: u64,
+    stage: &'static str,
+    probe: F,
+) -> Result<ScannerActivitySnapshot, String>
+where
+    F: FnMut() -> Fut,
+    Fut: Future<Output = Result<ScannerActivitySnapshot, String>>,
+{
+    match await_scanner_publication_proof(ctx, cycle, stage, probe, |err: &String| {
+        scanner_publication_activity_error_is_retryable(err)
+    })
+    .await
+    {
+        ScannerPublicationProofWait::Ready(snapshot) => Ok(snapshot),
+        ScannerPublicationProofWait::Rejected(err) => Err(err),
+        ScannerPublicationProofWait::Cancelled => Err(format!("scanner publication activity proof was cancelled during {stage}")),
+    }
+}
+
 impl Default for ScannerCleanIdleBackoff {
     fn default() -> Self {
         Self { interval_multiplier: 1 }

crates/scanner/src/scanner/cycle_state.rs

@@ -1327,6 +1327,13 @@ pub(super) async fn persisted_usage_floor_for_startup(
     let mut floor = PersistedUsageFloor::default();
     let mut found_any = false;
     let mut bootstrap_pending = false;
+    // A valid JSON object without a baseline identity is not a floor and must
+    // never be treated as an empty one.  It can, however, be a partially
+    // written v2 primary left behind during an upgrade.  Keep its epoch as a
+    // fence while looking for a durable companion snapshot; if no companion
+    // is new enough, the caller still fails closed below.
+    let mut invalid_baseline_path: Option<String> = None;
+    let mut invalid_baseline_epoch: Option<u64> = None;
     let update_floor = |floor: &mut PersistedUsageFloor, usage: &DataUsageInfo, path: &str| -> Result<(), ScannerError> {
         floor.leader_epoch = floor.leader_epoch.max(usage.scanner_epoch.unwrap_or_default());
         if let Some(completed_cycle) = usage.scanner_cycle {
@@ -1340,6 +1347,7 @@ pub(super) async fn persisted_usage_floor_for_startup(
     };
     for primary_path in [DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()] {
         let backup_path = format!("{primary_path}.bkp");
+        let is_v2_path = primary_path == DATA_USAGE_OBJ_NAME_PATH.as_str();
         let primary_epoch = match read_config_with_revision(storeapi.clone(), primary_path).await {
             Ok((Some(data), _)) => {
                 let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| {
@@ -1353,13 +1361,20 @@ pub(super) async fn persisted_usage_floor_for_startup(
                     update_floor(&mut floor, &usage, primary_path)?;
                     None
                 } else if !data_usage_info_has_persisted_baseline_identity(&usage) {
-                    return Err(ScannerError::Other(format!(
-                        "scanner usage floor from {primary_path} has no persisted baseline identity"
-                    )));
+                    invalid_baseline_path.get_or_insert_with(|| primary_path.to_string());
+                    invalid_baseline_epoch = invalid_baseline_epoch.max(usage.scanner_epoch);
+                    None
                 } else {
                     let epoch = usage.scanner_epoch.unwrap_or_default();
-                    update_floor(&mut floor, &usage, primary_path)?;
-                    Some(epoch)
+                    // A legacy snapshot may be structurally valid but older
+                    // than an incomplete v2 snapshot left by a newer leader.
+                    // Do not let that candidate regress the startup floor.
+                    if !is_v2_path && invalid_baseline_epoch.is_some_and(|fenced_epoch| epoch < fenced_epoch) {
+                        None
+                    } else {
+                        update_floor(&mut floor, &usage, primary_path)?;
+                        Some(epoch)
+                    }
                 }
             }
             Ok((None, _)) => None,
@@ -1377,21 +1392,26 @@ pub(super) async fn persisted_usage_floor_for_startup(
                         "scanner usage bootstrap conflicts with a persisted backup".to_string(),
                     ));
                 }
-                any_found = true;
                 let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| {
                     ScannerError::Other(format!("failed to decode scanner usage floor from {backup_path}: {err}"))
                 })?;
                 if !data_usage_info_has_persisted_baseline_identity(&usage) {
-                    return Err(ScannerError::Other(format!(
-                        "scanner usage floor from {backup_path} has no persisted baseline identity"
-                    )));
-                }
-                let backup_epoch = usage.scanner_epoch.unwrap_or_default();
-                // A backup write from an older leader may complete after the
-                // primary epoch has been fenced. It must not advance the startup
-                // floor unless its epoch is at least as new as the primary.
-                if primary_epoch.is_none_or(|epoch| backup_epoch >= epoch) {
-                    update_floor(&mut floor, &usage, &backup_path)?;
+                    invalid_baseline_path.get_or_insert_with(|| backup_path.clone());
+                    invalid_baseline_epoch = invalid_baseline_epoch.max(usage.scanner_epoch);
+                    // This is still persisted state, so it must not enable a
+                    // missing-state bootstrap.  Continue to a legacy pair in
+                    // case it contains a complete, fenced snapshot.
+                } else {
+                    let backup_epoch = usage.scanner_epoch.unwrap_or_default();
+                    // A backup write from an older leader may complete after the
+                    // primary epoch has been fenced. It must not advance the startup
+                    // floor unless its epoch is at least as new as the primary.
+                    if primary_epoch.is_none_or(|epoch| backup_epoch >= epoch)
+                        && invalid_baseline_epoch.is_none_or(|epoch| backup_epoch >= epoch)
+                    {
+                        update_floor(&mut floor, &usage, &backup_path)?;
+                        any_found = true;
+                    }
                 }
             }
             Ok((None, _)) => {}
@@ -1413,6 +1433,11 @@ pub(super) async fn persisted_usage_floor_for_startup(
     }
 
     if !found_any && !bootstrap_pending {
+        if let Some(path) = invalid_baseline_path {
+            return Err(ScannerError::Other(format!(
+                "persisted scanner usage floor from {path} has no authoritative baseline or newer valid backup"
+            )));
+        }
         if !allow_missing_for_bootstrap {
             return Err(ScannerError::Other(
                 "persisted scanner usage floor has no authoritative baseline".to_string(),

crates/scanner/src/scanner/leadership.rs

@@ -82,9 +82,30 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
     primary: Option<&[u8]>,
     allow_bootstrap_pending: bool,
 ) -> Result<Option<DataUsageInfo>, ScannerError> {
+    // A partially written v2 primary is not itself a baseline, but a durable
+    // companion may still provide one after an interrupted upgrade. Keep the
+    // primary epoch as a fence while checking those companions; malformed
+    // bytes and bootstrap markers retain their fail-closed behavior.
+    let mut invalid_primary_epoch = None;
     if let Some(primary) = primary {
-        return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str(), allow_bootstrap_pending)
-            .map(Some);
+        let usage: DataUsageInfo = serde_json::from_slice(primary).map_err(|err| {
+            ScannerError::Other(format!(
+                "failed to decode scanner usage epoch fence from {}: {err}",
+                DATA_USAGE_OBJ_NAME_PATH.as_str()
+            ))
+        })?;
+        if data_usage_info_has_persisted_baseline_identity(&usage)
+            || (allow_bootstrap_pending && data_usage_info_is_bootstrap_pending(&usage))
+        {
+            return Ok(Some(usage));
+        }
+        if data_usage_info_is_bootstrap_pending(&usage) {
+            return Err(ScannerError::Other(format!(
+                "scanner usage epoch fence from {} has no persisted baseline identity",
+                DATA_USAGE_OBJ_NAME_PATH.as_str()
+            )));
+        }
+        invalid_primary_epoch = usage.scanner_epoch;
     }
 
     let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
@@ -92,7 +113,10 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
         .await
         .map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence backup: {err}")))?;
     if let Some(backup) = backup.as_deref() {
-        return decode_usage_snapshot_for_epoch_fence(backup, &backup_path, false).map(Some);
+        let usage = decode_usage_snapshot_for_epoch_fence(backup, &backup_path, false)?;
+        if invalid_primary_epoch.is_none_or(|epoch| usage.scanner_epoch.unwrap_or_default() >= epoch) {
+            return Ok(Some(usage));
+        }
     }
 
     for path in [
@@ -103,7 +127,10 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
             .await
             .map_err(|err| ScannerError::Other(format!("failed to read legacy scanner usage epoch fence: {err}")))?;
         if let Some(legacy) = legacy.as_deref() {
-            return decode_usage_snapshot_for_epoch_fence(legacy, &path, false).map(Some);
+            let usage = decode_usage_snapshot_for_epoch_fence(legacy, &path, false)?;
+            if invalid_primary_epoch.is_none_or(|epoch| usage.scanner_epoch.unwrap_or_default() >= epoch) {
+                return Ok(Some(usage));
+            }
         }
     }
     // A missing usage snapshot is an uninitialized state, not an empty

crates/scanner/src/scanner/tests.rs

@@ -352,6 +352,7 @@ struct MemoryConfigStore {
     objects: Mutex<HashMap<String, Vec<u8>>>,
     revisions: Mutex<HashMap<String, u64>>,
     insert_after_gets: Mutex<HashMap<String, Vec<u8>>>,
+    delayed_gets: Mutex<HashMap<String, Duration>>,
     non_regular_objects: Mutex<HashSet<String>>,
     fail_put_number: Mutex<HashMap<String, usize>>,
     object_not_found_put_number: Mutex<HashMap<String, usize>>,
@@ -399,6 +400,9 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
         _opts: &ObjectOptions,
     ) -> EcstoreResult<GetObjectReader> {
         let key = memory_config_key(bucket, object);
+        if let Some(delay) = self.delayed_gets.lock().await.remove(&key) {
+            tokio::time::sleep(delay).await;
+        }
         let inserted_data = self.insert_after_gets.lock().await.remove(&key);
         let data = {
             let mut objects = self.objects.lock().await;
@@ -1923,6 +1927,176 @@ async fn scanner_startup_uses_primary_and_backup_usage_floor() {
     assert_eq!(epoch, 11);
 }
 
+#[tokio::test]
+async fn scanner_usage_floor_keeps_valid_primary_when_backup_has_no_identity() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
+    let mut primary = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
+    primary.scanner_epoch = Some(8);
+    primary.scanner_cycle = Some(100);
+    let backup = DataUsageInfo {
+        scanner_epoch: Some(9),
+        scanner_cycle: Some(101),
+        usage_snapshot_complete: false,
+        ..Default::default()
+    };
+
+    for (path, usage) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), primary), (backup_path.as_str(), backup)] {
+        store.objects.lock().await.insert(
+            memory_config_key(RUSTFS_META_BUCKET, path),
+            serde_json::to_vec(&usage).expect("usage snapshot should encode"),
+        );
+    }
+
+    assert_eq!(
+        persisted_usage_floor(store)
+            .await
+            .expect("valid primary should remain authoritative"),
+        PersistedUsageFloor {
+            next_cycle: 101,
+            leader_epoch: 8,
+        }
+    );
+}
+
+#[tokio::test]
+async fn scanner_usage_floor_recovers_from_incomplete_v2_primary_using_fenced_backup() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
+
+    // This shape is valid JSON from an interrupted v2 publication, but it is
+    // not a durable baseline because the snapshot is incomplete. It must not
+    // be converted into an empty floor.
+    let primary = DataUsageInfo {
+        scanner_epoch: Some(7),
+        scanner_cycle: Some(100),
+        usage_snapshot_complete: false,
+        ..Default::default()
+    };
+    let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
+    backup.scanner_epoch = Some(7);
+    backup.scanner_cycle = Some(103);
+
+    for (path, usage) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), primary), (backup_path.as_str(), backup)] {
+        store.objects.lock().await.insert(
+            memory_config_key(RUSTFS_META_BUCKET, path),
+            serde_json::to_vec(&usage).expect("usage snapshot should encode"),
+        );
+    }
+
+    assert_eq!(
+        persisted_usage_floor(store)
+            .await
+            .expect("valid backup should recover the usage floor"),
+        PersistedUsageFloor {
+            next_cycle: 104,
+            leader_epoch: 7,
+        }
+    );
+}
+
+#[tokio::test]
+async fn scanner_usage_floor_does_not_bootstrap_over_incomplete_v2_primary() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let primary = DataUsageInfo {
+        scanner_epoch: Some(7),
+        scanner_cycle: Some(100),
+        usage_snapshot_complete: false,
+        ..Default::default()
+    };
+    store.objects.lock().await.insert(
+        memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
+        serde_json::to_vec(&primary).expect("usage snapshot should encode"),
+    );
+
+    let err = persisted_usage_floor_for_startup(store, true)
+        .await
+        .expect_err("an existing incomplete primary must remain fail-closed");
+    assert!(err.to_string().contains("no authoritative baseline"));
+}
+
+#[tokio::test]
+async fn scanner_usage_floor_rejects_backup_older_than_incomplete_v2_primary() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
+    let primary = DataUsageInfo {
+        scanner_epoch: Some(7),
+        scanner_cycle: Some(100),
+        usage_snapshot_complete: false,
+        ..Default::default()
+    };
+    let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
+    backup.scanner_epoch = Some(6);
+    backup.scanner_cycle = Some(10_000);
+
+    for (path, usage) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), primary), (backup_path.as_str(), backup)] {
+        store.objects.lock().await.insert(
+            memory_config_key(RUSTFS_META_BUCKET, path),
+            serde_json::to_vec(&usage).expect("usage snapshot should encode"),
+        );
+    }
+
+    let err = persisted_usage_floor_for_startup(store, true)
+        .await
+        .expect_err("an older backup must not cross the incomplete primary epoch fence");
+    assert!(err.to_string().contains("no authoritative baseline"));
+}
+
+#[tokio::test]
+async fn scanner_usage_floor_rejects_older_legacy_primary_after_incomplete_v2_primary() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let primary = DataUsageInfo {
+        scanner_epoch: Some(7),
+        scanner_cycle: Some(100),
+        usage_snapshot_complete: false,
+        ..Default::default()
+    };
+    let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
+    legacy.scanner_epoch = Some(6);
+    legacy.scanner_cycle = Some(103);
+    for (path, usage) in [
+        (DATA_USAGE_OBJ_NAME_PATH.as_str(), primary),
+        (LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), legacy),
+    ] {
+        store.objects.lock().await.insert(
+            memory_config_key(RUSTFS_META_BUCKET, path),
+            serde_json::to_vec(&usage).expect("usage snapshot should encode"),
+        );
+    }
+
+    let err = persisted_usage_floor_for_startup(store, true)
+        .await
+        .expect_err("an older legacy baseline must not cross the incomplete v2 epoch fence");
+    assert!(err.to_string().contains("no authoritative baseline"));
+}
+
+#[tokio::test]
+async fn scanner_leadership_fencing_recovers_incomplete_v2_primary_from_backup() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
+    let primary = serde_json::to_vec(&DataUsageInfo {
+        scanner_epoch: Some(7),
+        scanner_cycle: Some(100),
+        usage_snapshot_complete: false,
+        ..Default::default()
+    })
+    .expect("incomplete usage snapshot should encode");
+    let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
+    backup.scanner_epoch = Some(7);
+    backup.scanner_cycle = Some(103);
+    store.objects.lock().await.insert(
+        memory_config_key(RUSTFS_META_BUCKET, &backup_path),
+        serde_json::to_vec(&backup).expect("backup usage snapshot should encode"),
+    );
+
+    let recovered = usage_snapshot_for_epoch_fence(store, Some(&primary), false)
+        .await
+        .expect("a valid backup should provide the fencing baseline")
+        .expect("the fencing baseline should be present");
+    assert_eq!(recovered.scanner_epoch, Some(7));
+    assert_eq!(recovered.scanner_cycle, Some(103));
+}
+
 #[tokio::test]
 async fn scanner_usage_floor_ignores_older_backup_after_primary_epoch_fence() {
     let store = Arc::new(MemoryConfigStore::default());
@@ -2481,7 +2655,7 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
         }
         if self
             .block_publication_after_admissions
-            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| remaining.checked_sub(1))
+            .try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| remaining.checked_sub(1))
             == Ok(1)
         {
             self.publication_admission_blocked.store(true, Ordering::Release);
@@ -3182,6 +3356,79 @@ async fn test_observational_usage_defers_when_authoritative_baseline_is_missing(
     );
 }
 
+#[tokio::test]
+async fn test_observational_usage_uses_fenced_backup_when_v2_primary_has_no_identity() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let primary = DataUsageInfo {
+        scanner_epoch: Some(7),
+        scanner_cycle: Some(100),
+        usage_snapshot_complete: false,
+        ..Default::default()
+    };
+    let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
+    backup.scanner_epoch = Some(7);
+    backup.scanner_cycle = Some(103);
+    store.objects.lock().await.insert(
+        memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
+        serde_json::to_vec(&primary).expect("incomplete primary should encode"),
+    );
+    store.objects.lock().await.insert(
+        memory_config_key(RUSTFS_META_BUCKET, &format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str())),
+        serde_json::to_vec(&backup).expect("backup baseline should encode"),
+    );
+
+    let (sender, receiver) = mpsc::channel(1);
+    let mut observation = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1);
+    observation.usage_snapshot_converged = Some(false);
+    sender.send(observation).await.expect("observation should enqueue");
+    drop(sender);
+
+    let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
+        CancellationToken::new(),
+        store.clone(),
+        receiver,
+        None,
+        None,
+        || async { false },
+    )
+    .await;
+
+    assert_eq!(outcome, DataUsagePersistOutcome::Saved);
+    let observed = read_config(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("observational snapshot should be persisted");
+    let observed = serde_json::from_slice::<DataUsageInfo>(&observed).expect("observational snapshot should decode");
+    assert_eq!(observed.usage_snapshot_authoritative_baseline, Some(backup.snapshot_identity()));
+}
+
+#[tokio::test]
+async fn usage_baseline_does_not_fall_back_to_older_legacy_snapshot() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let primary = DataUsageInfo {
+        scanner_epoch: Some(7),
+        scanner_cycle: Some(100),
+        usage_snapshot_complete: false,
+        ..Default::default()
+    };
+    let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
+    legacy.scanner_epoch = Some(6);
+    legacy.scanner_cycle = Some(103);
+    let primary_data = serde_json::to_vec(&primary).expect("incomplete primary should encode");
+    store.objects.lock().await.insert(
+        memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
+        primary_data.clone(),
+    );
+    store.objects.lock().await.insert(
+        memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
+        serde_json::to_vec(&legacy).expect("legacy baseline should encode"),
+    );
+
+    let baseline = read_data_usage_persist_baseline(store)
+        .await
+        .expect("baseline inspection should complete");
+    assert_eq!(baseline.data.as_deref(), Some(primary_data.as_slice()));
+}
+
 #[tokio::test]
 #[serial]
 async fn test_usage_route_barrier_precedes_durable_reconciliation() {
@@ -3251,6 +3498,85 @@ async fn coordinator_does_not_put_after_remote_generation_flip() {
     assert_eq!(store.put_counts.lock().await.get(&key), None);
 }
 
+#[tokio::test]
+async fn coordinator_classifies_an_expired_publication_lease() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let (sender, receiver) = mpsc::channel(1);
+    sender
+        .send(complete_usage_with_bucket_count(
+            Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
+            1,
+        ))
+        .await
+        .expect("usage snapshot should enqueue");
+    drop(sender);
+
+    let expired = std::time::Instant::now()
+        .checked_sub(std::time::Duration::from_secs(1))
+        .expect("test instant should support a one-second subtraction");
+    let outcome =
+        store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
+            CancellationToken::new(),
+            store.clone(),
+            receiver,
+            None,
+            Some(DataUsagePersistBaseline {
+                data: None,
+                revision: DataUsageCacheRevision::Missing,
+            }),
+            ScannerPublicationFence::new(None, Some(expired), None),
+            || async { false },
+        )
+        .await;
+
+    assert_eq!(
+        outcome,
+        DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded)
+    );
+    assert!(store.put_counts.lock().await.is_empty(), "expired lease must prevent a PUT");
+}
+
+#[tokio::test]
+async fn backup_sync_checks_the_lease_deadline_after_a_slow_backup_read() {
+    let store = Arc::new(MemoryConfigStore::default());
+    let primary_path = DATA_USAGE_OBJ_NAME_PATH.as_str();
+    let backup_path = format!("{primary_path}.bkp");
+    let primary_key = memory_config_key(RUSTFS_META_BUCKET, primary_path);
+    let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path);
+    let primary = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
+    store
+        .objects
+        .lock()
+        .await
+        .insert(primary_key, serde_json::to_vec(&primary).expect("primary usage snapshot should encode"));
+    store
+        .delayed_gets
+        .lock()
+        .await
+        .insert(backup_key.clone(), Duration::from_millis(20));
+
+    // The primary read is allowed to start, but the backup read consumes the
+    // remaining lease window. The second deadline check must prevent a stale
+    // backup PUT after that window has elapsed.
+    let deadline = std::time::Instant::now()
+        .checked_add(std::time::Duration::from_millis(5))
+        .expect("test deadline should support a five-millisecond window");
+    let result = sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
+        &CancellationToken::new(),
+        store.clone(),
+        None,
+        Some(deadline),
+        None,
+    )
+    .await;
+
+    assert!(scanner_publication_epoch_changed(
+        &result.expect_err("an expired backup lease must defer publication")
+    ));
+    assert!(!store.objects.lock().await.contains_key(&backup_key));
+    assert_eq!(store.put_counts.lock().await.get(&backup_key), None);
+}
+
 #[tokio::test]
 #[serial]
 async fn test_deferred_usage_save_keeps_last_real_save_metric() {
@@ -4302,6 +4628,8 @@ fn scanner_cycle_cache_floor_stays_pending_during_deferred_usage_publication() {
     for reason in [
         ScannerCycleDeferReason::DataMovement,
         ScannerCycleDeferReason::ActivityBaselineUnavailable,
+        ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded,
+        ScannerCycleDeferReason::PublicationLeaseReleaseFailed,
     ] {
         let deferred = DataUsagePersistOutcome::Deferred(reason);
         assert_eq!(
@@ -4368,6 +4696,26 @@ fn finalizing_a_deferred_usage_save_keeps_dirty_work_pending() {
     crate::scanner_io::clear_dirty_usage_bucket("photos");
 }
 
+#[test]
+#[serial]
+fn finalizing_post_scan_observation_advances_partially_without_dirty_ack() {
+    crate::scanner_io::clear_dirty_usage_bucket("photos");
+    crate::scanner_io::record_dirty_usage_bucket("photos");
+    let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
+    let observed = crate::scanner_io::ScannerCycleResult::new(
+        ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
+        Some(dirty_snapshot),
+    )
+    .with_observational_snapshot_published(true);
+
+    let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(observed, DataUsagePersistOutcome::Saved);
+
+    assert_eq!(outcome, ScannerCycleOutcome::Partial);
+    assert!(acknowledgements.is_empty());
+    assert!(crate::scanner_io::dirty_usage_buckets_pending());
+    crate::scanner_io::clear_dirty_usage_bucket("photos");
+}
+
 #[tokio::test]
 async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
     let pending = remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::<bool, std::io::Error>(true))).await;
@@ -4480,6 +4828,29 @@ async fn data_usage_persist_wait_aborts_after_timeout() {
     assert!(task.is_finished());
 }
 
+#[tokio::test(start_paused = true)]
+async fn data_usage_persist_timeout_drops_owned_task_without_a_late_commit() {
+    let ctx = CancellationToken::new();
+    let commit_started = Arc::new(AtomicBool::new(false));
+    let commit_started_by_task = commit_started.clone();
+    let task_ready = Arc::new(tokio::sync::Notify::new());
+    let task_ready_by_task = task_ready.clone();
+    let mut task = AbortOnDropHandle::new(tokio::spawn(async move {
+        task_ready_by_task.notify_one();
+        std::future::pending::<()>().await;
+        commit_started_by_task.store(true, Ordering::Release);
+        DataUsagePersistOutcome::Saved
+    }));
+    task_ready.notified().await;
+
+    let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(1)).await;
+
+    assert!(matches!(result, DataUsagePersistTaskResult::TimedOut));
+    assert!(task.is_finished(), "the timed-out persistence task must be drained before return");
+    tokio::task::yield_now().await;
+    assert!(!commit_started.load(Ordering::Acquire), "an owned task must not commit after its timeout");
+}
+
 #[tokio::test(start_paused = true)]
 async fn maintenance_feature_inspection_preserves_base_cycle_after_timeout() {
     let ctx = CancellationToken::new();
@@ -4705,6 +5076,140 @@ fn superseded_retry_backoff_grows_from_the_default_cycle() {
     }
 }
 
+#[test]
+fn publication_proof_retry_backoff_reaches_its_short_cap() {
+    for (failures, expected) in [(1, 5), (2, 10), (3, 20), (4, 30), (20, 30)] {
+        assert_eq!(scanner_publication_proof_retry_delay(failures), Duration::from_secs(expected));
+    }
+}
+
+#[test]
+fn publication_proof_retry_classifies_availability_without_masking_protocol_errors() {
+    for error in [
+        "peer node3 is temporarily offline",
+        "scanner activity peer node3 timed out after 5s",
+        "transport error: connection refused",
+    ] {
+        assert!(scanner_publication_activity_error_is_retryable(error), "{error}");
+    }
+
+    for error in [
+        "scanner activity peer node3 uses protocol 6, expected 7",
+        "scanner activity peer node3 has a different storage topology",
+        "scanner activity peer node3 omitted its movement generation",
+        "duplicate scanner activity peer: node3",
+        "scanner activity peer[2] is unreachable",
+        "scanner publication lease peer node3 is unavailable",
+    ] {
+        assert!(!scanner_publication_activity_error_is_retryable(error), "{error}");
+    }
+}
+
+#[test]
+fn publication_lease_retry_preserves_only_recoverable_candidates() {
+    for error in [
+        "scanner publication lease acquisition failed: scanner publication lease capacity is exhausted",
+        "scanner publication lease acquisition failed: scanner publication lease response arrived after its safety window",
+        "scanner publication lease acquisition failed: peer node3 is temporarily offline",
+    ] {
+        assert!(scanner_publication_lease_error_is_retryable(error), "{error}");
+    }
+
+    for error in [
+        "scanner publication lease acquisition failed: scanner publication lease generation is stale",
+        "scanner publication lease acquisition failed: peer returned a different scanner publication lease session",
+        "scanner publication lease acquisition failed: scanner publication lease is blocked by data movement",
+        "scanner publication lease acquisition failed: peer returned an invalid scanner publication lease proof",
+    ] {
+        assert!(!scanner_publication_lease_error_is_retryable(error), "{error}");
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn publication_proof_retains_candidate_until_activity_recovers() {
+    let ctx = CancellationToken::new();
+    let attempts = Arc::new(AtomicUsize::new(0));
+    let probe_attempts = attempts.clone();
+    let started_at = Instant::now();
+
+    let snapshot = await_scanner_publication_activity(&ctx, 17, "postscan", move || {
+        let attempt = probe_attempts.fetch_add(1, Ordering::SeqCst);
+        async move {
+            if attempt == 0 {
+                Err("peer temporarily offline".to_string())
+            } else {
+                Ok(ScannerActivitySnapshot::new())
+            }
+        }
+    })
+    .await
+    .expect("a retained publication candidate should survive one transient probe failure");
+
+    assert!(snapshot.is_empty());
+    assert_eq!(attempts.load(Ordering::SeqCst), 2);
+    assert_eq!(started_at.elapsed(), Duration::from_secs(5));
+}
+
+#[tokio::test(start_paused = true)]
+async fn publication_proof_does_not_retry_a_protocol_mismatch() {
+    let ctx = CancellationToken::new();
+    let attempts = Arc::new(AtomicUsize::new(0));
+    let probe_attempts = attempts.clone();
+
+    let err = await_scanner_publication_activity(&ctx, 17, "postscan", move || {
+        probe_attempts.fetch_add(1, Ordering::SeqCst);
+        async { Err("scanner activity peer node3 uses protocol 6, expected 7".to_string()) }
+    })
+    .await
+    .expect_err("a protocol mismatch must not be hidden behind availability retries");
+
+    assert!(err.contains("uses protocol"));
+    assert_eq!(attempts.load(Ordering::SeqCst), 1);
+}
+
+#[tokio::test]
+async fn publication_proof_stops_waiting_when_the_cycle_is_cancelled() {
+    let ctx = CancellationToken::new();
+    ctx.cancel();
+    let attempts = Arc::new(AtomicUsize::new(0));
+    let probe_attempts = attempts.clone();
+
+    let err = await_scanner_publication_activity(&ctx, 17, "postscan", move || {
+        probe_attempts.fetch_add(1, Ordering::SeqCst);
+        async { Ok(ScannerActivitySnapshot::new()) }
+    })
+    .await
+    .expect_err("a cancelled cycle must release its retained publication candidate");
+
+    assert!(err.contains("cancelled"));
+    assert_eq!(attempts.load(Ordering::SeqCst), 0);
+}
+
+#[tokio::test(start_paused = true)]
+async fn publication_proof_releases_candidate_when_cancelled_during_backoff() {
+    let ctx = CancellationToken::new();
+    let cancel_ctx = ctx.clone();
+    let attempts = Arc::new(AtomicUsize::new(0));
+    let probe_attempts = attempts.clone();
+    let started_at = Instant::now();
+
+    let cancel = tokio::spawn(async move {
+        tokio::time::sleep(Duration::from_secs(1)).await;
+        cancel_ctx.cancel();
+    });
+    let err = await_scanner_publication_activity(&ctx, 17, "postscan", move || {
+        probe_attempts.fetch_add(1, Ordering::SeqCst);
+        async { Err("peer temporarily offline".to_string()) }
+    })
+    .await
+    .expect_err("cycle cancellation must release a candidate waiting to retry publication proof");
+    cancel.await.expect("cancellation task should complete");
+
+    assert!(err.contains("cancelled"));
+    assert_eq!(attempts.load(Ordering::SeqCst), 1);
+    assert_eq!(started_at.elapsed(), Duration::from_secs(1));
+}
+
 #[tokio::test(start_paused = true)]
 async fn corrupt_cycle_state_backoff_uses_virtual_clock() {
     let mut backoff = ScannerRetryBackoff::default();

crates/scanner/src/scanner/usage_store.rs

@@ -13,7 +13,10 @@
 // limitations under the License.
 /// Data-usage snapshot persistence: CAS store pipeline, epoch baselines, and observed-snapshot cleanup.
 use super::*;
+use crate::storage_api::owner::ScannerPublicationCommitState;
 use std::collections::HashMap;
+use std::sync::atomic::AtomicBool;
+use uuid::Uuid;
 
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
 pub(super) enum DataUsagePersistOutcome {
@@ -34,12 +37,100 @@ fn remote_lease_expired(deadline: Option<std::time::Instant>) -> bool {
     deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline)
 }
 
+pub(super) fn scanner_publication_scope_deadline(
+    persist_timeout: Duration,
+    remote_lease_deadline: Option<std::time::Instant>,
+) -> tokio::time::Instant {
+    let configured_deadline = tokio::time::Instant::now() + persist_timeout;
+    remote_lease_deadline
+        .map(tokio::time::Instant::from_std)
+        .map_or(configured_deadline, |lease_deadline| configured_deadline.min(lease_deadline))
+}
+
 #[derive(Clone, Debug)]
 pub(super) struct DataUsagePersistBaseline {
     pub(super) data: Option<Bytes>,
     pub(super) revision: DataUsageCacheRevision,
 }
 
+/// Read the bytes used as the baseline for a usage publication while keeping
+/// the v2 primary revision as the CAS fence. During an interrupted upgrade the
+/// primary can be valid JSON without a baseline identity; in that case a
+/// same-or-newer durable companion may still be used, but an older legacy
+/// snapshot must not cross the primary's epoch fence.
+pub(super) async fn read_data_usage_persist_baseline(
+    storeapi: Arc<impl ScannerObjectIO>,
+) -> Result<DataUsagePersistBaseline, EcstoreError> {
+    let (primary, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?;
+    let Some(primary) = primary else {
+        for path in [
+            format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()),
+            LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
+            format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
+        ] {
+            let (candidate, _) = read_config_with_revision(storeapi.clone(), &path).await?;
+            let Some(candidate) = candidate else {
+                continue;
+            };
+            let Ok(usage) = serde_json::from_slice::<DataUsageInfo>(&candidate) else {
+                continue;
+            };
+            if data_usage_info_has_persisted_baseline_identity(&usage) {
+                return Ok(DataUsagePersistBaseline {
+                    data: Some(Bytes::from(candidate)),
+                    revision,
+                });
+            }
+        }
+        return Ok(DataUsagePersistBaseline { data: None, revision });
+    };
+
+    let Ok(primary_info) = serde_json::from_slice::<DataUsageInfo>(&primary) else {
+        // Preserve the original bytes and revision. A completed scan may
+        // replace the invalid primary under this CAS fence; an observation
+        // will still reject it below because it has no verifiable identity.
+        return Ok(DataUsagePersistBaseline {
+            data: Some(Bytes::from(primary)),
+            revision,
+        });
+    };
+    if data_usage_info_has_persisted_baseline_identity(&primary_info) || data_usage_info_is_bootstrap_pending(&primary_info) {
+        return Ok(DataUsagePersistBaseline {
+            data: Some(Bytes::from(primary)),
+            revision,
+        });
+    }
+
+    let invalid_primary_epoch = primary_info.scanner_epoch;
+    for path in [
+        format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()),
+        LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
+        format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
+    ] {
+        let (candidate, _) = read_config_with_revision(storeapi.clone(), &path).await?;
+        let Some(candidate) = candidate else {
+            continue;
+        };
+        let Ok(usage) = serde_json::from_slice::<DataUsageInfo>(&candidate) else {
+            continue;
+        };
+        let candidate_epoch = usage.scanner_epoch.unwrap_or_default();
+        if data_usage_info_has_persisted_baseline_identity(&usage)
+            && invalid_primary_epoch.is_none_or(|epoch| candidate_epoch >= epoch)
+        {
+            return Ok(DataUsagePersistBaseline {
+                data: Some(Bytes::from(candidate)),
+                revision,
+            });
+        }
+    }
+
+    Ok(DataUsagePersistBaseline {
+        data: Some(Bytes::from(primary)),
+        revision,
+    })
+}
+
 /// Short-lived publication inputs captured for one usage persistence attempt.
 /// Keeping the movement epoch, lease deadline, and target fence together makes
 /// it explicit that they are one proof rather than independent options.
@@ -48,6 +139,8 @@ pub(super) struct ScannerPublicationFence {
     pub(super) expected_publication_epoch: Option<u64>,
     pub(super) remote_lease_deadline: Option<std::time::Instant>,
     pub(super) scanner_publication_lease_fence: Option<String>,
+    pub(super) remote_lease_tokens: Vec<Uuid>,
+    pub(super) lease_release_safe: Arc<AtomicBool>,
 }
 
 impl ScannerPublicationFence {
@@ -60,8 +153,20 @@ impl ScannerPublicationFence {
             expected_publication_epoch,
             remote_lease_deadline,
             scanner_publication_lease_fence,
+            remote_lease_tokens: Vec::new(),
+            lease_release_safe: Arc::new(AtomicBool::new(true)),
         }
     }
+
+    pub(super) fn with_remote_lease_tokens(mut self, remote_lease_tokens: Vec<Uuid>) -> Self {
+        self.remote_lease_tokens = remote_lease_tokens;
+        self
+    }
+
+    pub(super) fn with_lease_release_flag(mut self, lease_release_safe: Arc<AtomicBool>) -> Self {
+        self.lease_release_safe = lease_release_safe;
+        self
+    }
 }
 
 #[derive(Debug)]
@@ -212,6 +317,8 @@ where
         expected_publication_epoch,
         remote_lease_deadline,
         scanner_publication_lease_fence,
+        remote_lease_tokens,
+        lease_release_safe,
     } = publication_fence;
     let mut outcome = DataUsagePersistOutcome::NoUpdate;
     let mut next_baseline = initial_baseline;
@@ -225,7 +332,7 @@ where
             data_usage_info.scanner_epoch = Some(leader_epoch);
         }
         if remote_lease_expired(remote_lease_deadline) {
-            outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
+            outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded);
             break 'updates;
         }
         if let Some(expected_epoch) = expected_publication_epoch
@@ -281,8 +388,8 @@ where
             publication_epoch = Some(read_epoch);
             let authoritative_data = match next_baseline.as_ref() {
                 Some(baseline) => baseline.data.clone(),
-                None => match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await {
-                    Ok((data, _)) => data.map(Bytes::from),
+                None => match read_data_usage_persist_baseline(storeapi.clone()).await {
+                    Ok(baseline) => baseline.data,
                     Err(err) => {
                         error!(
                             target: "rustfs::scanner",
@@ -497,30 +604,59 @@ where
                 break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
             }
             if remote_lease_expired(remote_lease_deadline) {
-                break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
+                break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded);
             }
 
             let done_save = Metrics::time(Metric::SaveUsage);
             let save_result = {
-                let Some(_publication_admission) =
-                    scanner_publication_admission_for_epoch(storeapi.clone(), publication_epoch_for_save).await
-                else {
-                    done_save();
-                    break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
+                let publication_scope = storeapi
+                    .scanner_data_usage_publication_commit_scope_with_release_flag(
+                        publication_epoch_for_save,
+                        scanner_publication_scope_deadline(data_usage_persist_timeout(), remote_lease_deadline),
+                        remote_lease_tokens.clone(),
+                        Arc::clone(&lease_release_safe),
+                    )
+                    .await;
+                let legacy_publication_admission = if publication_scope.is_none() {
+                    let Some(admission) =
+                        scanner_publication_admission_for_epoch(storeapi.clone(), publication_epoch_for_save).await
+                    else {
+                        done_save();
+                        break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
+                    };
+                    Some(admission)
+                } else {
+                    None
                 };
                 if remote_lease_expired(remote_lease_deadline) {
                     done_save();
-                    break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
+                    break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded);
                 }
-                save_config_shared_with_preconditions_and_lease_fence(
+                let save_result = crate::save_config_shared_with_preconditions_and_lease_fence_and_scope(
                     storeapi.clone(),
                     target_path,
                     data.clone(),
                     sha256hex.clone(),
                     revision.preconditions(),
                     scanner_publication_lease_fence.as_deref(),
+                    publication_scope.clone(),
                 )
-                .await
+                .await;
+                drop(legacy_publication_admission);
+                if let Some(scope) = publication_scope {
+                    match scope.wait_for_completion().await {
+                        ScannerPublicationCommitState::Committed | ScannerPublicationCommitState::AbortedBeforeCommit => {
+                            save_result
+                        }
+                        ScannerPublicationCommitState::Indeterminate
+                        | ScannerPublicationCommitState::Admitted
+                        | ScannerPublicationCommitState::InFlight => Err(EcstoreError::other(
+                            "scanner publication commit scope did not reach a safe terminal state",
+                        )),
+                    }
+                } else {
+                    save_result
+                }
             };
             done_save();
 
@@ -618,6 +754,8 @@ where
                         expected_publication_epoch,
                         remote_lease_deadline,
                         scanner_publication_lease_fence.as_deref(),
+                        &remote_lease_tokens,
+                        Arc::clone(&lease_release_safe),
                     )
                     .await;
                     if expected_publication_epoch.is_some() && !cleanup_ok {
@@ -641,6 +779,8 @@ where
                         expected_publication_epoch,
                         remote_lease_deadline,
                         scanner_publication_lease_fence.as_deref(),
+                        &remote_lease_tokens,
+                        Arc::clone(&lease_release_safe),
                     )
                     .await;
                     if expected_publication_epoch.is_some() && !cleanup_ok {
@@ -683,6 +823,8 @@ where
                         expected_publication_epoch,
                         remote_lease_deadline,
                         scanner_publication_lease_fence.as_deref(),
+                        &remote_lease_tokens,
+                        Arc::clone(&lease_release_safe),
                     )
                     .await;
                     if expected_publication_epoch.is_some() && !cleanup_ok {
@@ -700,12 +842,14 @@ where
 
         if backup_due {
             let done_save = Metrics::time(Metric::SaveUsage);
-            let backup_result = sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
+            let backup_result = sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence_and_scope(
                 &ctx,
                 storeapi.clone(),
                 expected_publication_epoch,
                 remote_lease_deadline,
                 scanner_publication_lease_fence.as_deref(),
+                remote_lease_tokens.clone(),
+                Arc::clone(&lease_release_safe),
             )
             .await;
             done_save();
@@ -739,6 +883,8 @@ async fn cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
     expected_publication_epoch: Option<u64>,
     remote_lease_deadline: Option<std::time::Instant>,
     scanner_publication_lease_fence: Option<&str>,
+    remote_lease_tokens: &[Uuid],
+    lease_release_safe: Arc<AtomicBool>,
 ) -> bool {
     if remote_lease_expired(remote_lease_deadline) {
         return false;
@@ -807,7 +953,15 @@ async fn cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
         return false;
     }
 
-    let result = delete_config_with_publication_admission_for_epoch(
+    let publication_scope = storeapi
+        .scanner_data_usage_publication_commit_scope_with_release_flag(
+            read_epoch,
+            scanner_publication_scope_deadline(data_usage_persist_timeout(), remote_lease_deadline),
+            remote_lease_tokens.to_vec(),
+            Arc::clone(&lease_release_safe),
+        )
+        .await;
+    let result = crate::delete_config_with_publication_scope_for_epoch(
         storeapi,
         RUSTFS_META_BUCKET,
         DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
@@ -826,9 +980,23 @@ async fn cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
             ..Default::default()
         },
         read_epoch,
+        publication_scope.clone(),
     )
     .await;
 
+    let result = if let Some(scope) = publication_scope {
+        match scope.wait_for_completion().await {
+            ScannerPublicationCommitState::Committed | ScannerPublicationCommitState::AbortedBeforeCommit => result,
+            ScannerPublicationCommitState::Indeterminate
+            | ScannerPublicationCommitState::Admitted
+            | ScannerPublicationCommitState::InFlight => Err(EcstoreError::other(
+                "scanner publication cleanup scope did not reach a safe terminal state",
+            )),
+        }
+    } else {
+        result
+    };
+
     match result {
         Ok(_)
         | Err(

crates/scanner/src/scanner_io.rs

@@ -269,6 +269,10 @@ fn should_publish_usage_snapshot(status: ScannerCycleStatus) -> bool {
     matches!(status, ScannerCycleStatus::Complete | ScannerCycleStatus::Superseded)
 }
 
+fn should_publish_observational_snapshot(status: ScannerCycleStatus) -> bool {
+    matches!(status, ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable))
+}
+
 fn prepare_usage_snapshot_for_publication(
     status: ScannerCycleStatus,
     mut data_usage_info: DataUsageInfo,
@@ -570,6 +574,13 @@ pub(crate) async fn scanner_set_disk_inventory(set: &SetDisks) -> Vec<Arc<Disk>>
 pub(crate) enum ScannerCycleDeferReason {
     ActivityBaselineUnavailable,
     DataMovement,
+    /// A granted lease's absolute deadline cannot cover the persistence
+    /// operation. This can occur even when the configured budget fits the
+    /// nominal TTL because lease acquisition consumed part of the window.
+    PublicationLeaseDeadlineExceeded,
+    /// A remote lease could not be released after the persistence attempt.
+    /// Keep the cycle deferred because the peer may still admit movement.
+    PublicationLeaseReleaseFailed,
 }
 
 impl ScannerCycleDeferReason {
@@ -577,6 +588,8 @@ impl ScannerCycleDeferReason {
         match self {
             Self::ActivityBaselineUnavailable => "activity_baseline_unavailable",
             Self::DataMovement => "data_movement",
+            Self::PublicationLeaseDeadlineExceeded => "publication_lease_deadline_exceeded",
+            Self::PublicationLeaseReleaseFailed => "publication_lease_release_failed",
         }
     }
 }
@@ -611,6 +624,7 @@ fn scanner_activity_preflight(
 pub(crate) struct ScannerCycleResult {
     pub(crate) status: ScannerCycleStatus,
     publication_epoch: Option<u64>,
+    observational_snapshot_published: bool,
     dirty_usage_clear: Option<DirtyUsageBuckets>,
     remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
     remote_publication_lease_targets: Vec<(String, String, u64)>,
@@ -624,6 +638,7 @@ impl ScannerCycleResult {
         Self {
             status,
             publication_epoch: None,
+            observational_snapshot_published: false,
             dirty_usage_clear,
             remote_dirty_usage_acknowledgements: Vec::new(),
             remote_publication_lease_targets: Vec::new(),
@@ -642,6 +657,15 @@ impl ScannerCycleResult {
         self.publication_epoch
     }
 
+    pub(crate) fn with_observational_snapshot_published(mut self, published: bool) -> Self {
+        self.observational_snapshot_published = published;
+        self
+    }
+
+    pub(crate) fn has_observational_snapshot(&self) -> bool {
+        self.observational_snapshot_published
+    }
+
     fn with_failed_dirty_usage(mut self, failed_dirty_usage: bool) -> Self {
         self.failed_dirty_usage = failed_dirty_usage;
         self

crates/scanner/src/scanner_io/dirty_usage.rs

@@ -45,7 +45,7 @@ pub(super) fn usize_to_u64_saturated(value: usize) -> u64 {
 
 pub(super) fn advance_generation(generation: &AtomicU64) -> u64 {
     generation
-        .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(1)))
+        .try_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(1)))
         .map_or_else(|current| current, |previous| previous.saturating_add(1))
 }
 

crates/scanner/src/scanner_io/guards.rs

@@ -195,14 +195,14 @@ impl Drop for DiskBucketScanGaugeReset {
 
 pub(super) fn decrement_atomic_usize(counter: &AtomicUsize) -> usize {
     counter
-        .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1)))
+        .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1)))
         .map(|previous| previous.saturating_sub(1))
         .unwrap_or_else(|current| current)
 }
 
 pub(super) fn increment_atomic_usize(counter: &AtomicUsize) -> usize {
     counter
-        .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(1)))
+        .try_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(1)))
         .map(|previous| previous.saturating_add(1))
         .unwrap_or_else(|current| current)
 }
@@ -286,6 +286,16 @@ pub(super) fn scanner_task_join_error(stage: &str, err: tokio::task::JoinError)
 mod tests {
     use super::*;
     use rustfs_scanner_contracts::metrics::{ScannerWorkSource, global_metrics};
+    use tokio::sync::oneshot;
+
+    fn active_bucket_drive_count(source: ScannerWorkSource, bucket: &str, drive: &str) -> u64 {
+        global_metrics()
+            .scanner_runtime_details_report()
+            .active_bucket_drive_scans
+            .into_iter()
+            .find(|active| active.source == source.as_str() && active.bucket == bucket && active.drive == drive)
+            .map_or(0, |active| active.count)
+    }
 
     #[test]
     fn bucket_drive_failure_guard_retires_active_scan_on_drop() {
@@ -305,4 +315,85 @@ mod tests {
                 .any(|active| active.source == source.as_str() && active.bucket == bucket && active.drive == drive)
         );
     }
+
+    #[tokio::test]
+    async fn bucket_drive_failure_guard_retires_active_scan_after_cancellation() {
+        let source = ScannerWorkSource::Usage;
+        let bucket = "__guard_cancel_lifecycle_test__";
+        let drive = "/__guard_cancel_lifecycle_test__";
+        global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
+
+        let cancellation = CancellationToken::new();
+        let worker_cancellation = cancellation.clone();
+        let worker = tokio::spawn(async move {
+            let mut guard = BucketDriveFailureGuard::new(source, bucket, drive);
+            worker_cancellation.cancelled().await;
+            guard.mark_not_failed();
+        });
+
+        cancellation.cancel();
+        worker.await.expect("cancelled scanner worker should finish");
+
+        assert_eq!(active_bucket_drive_count(source, bucket, drive), 0);
+    }
+
+    #[tokio::test]
+    async fn bucket_drive_failure_guard_retires_active_scan_when_worker_is_aborted() {
+        let source = ScannerWorkSource::Bitrot;
+        let bucket = "__guard_abort_lifecycle_test__";
+        let drive = "/__guard_abort_lifecycle_test__";
+        global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
+
+        let (started_sender, started_receiver) = oneshot::channel();
+        let worker = tokio::spawn(async move {
+            let _guard = BucketDriveFailureGuard::new(source, bucket, drive);
+            started_sender.send(()).expect("test should observe worker start");
+            std::future::pending::<()>().await;
+        });
+        started_receiver.await.expect("scanner worker should start");
+        assert_eq!(active_bucket_drive_count(source, bucket, drive), 1);
+
+        worker.abort();
+        worker.await.expect_err("aborted scanner worker should report cancellation");
+
+        assert_eq!(active_bucket_drive_count(source, bucket, drive), 0);
+    }
+
+    #[tokio::test]
+    async fn bucket_drive_failure_guards_track_overlapping_scans_independently() {
+        let source = ScannerWorkSource::Usage;
+        let bucket = "__guard_overlap_lifecycle_test__";
+        let drive = "/__guard_overlap_lifecycle_test__";
+        global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
+        global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
+
+        let (first_release_sender, first_release_receiver) = oneshot::channel();
+        let (second_release_sender, second_release_receiver) = oneshot::channel();
+        let (first_started_sender, first_started_receiver) = oneshot::channel();
+        let (second_started_sender, second_started_receiver) = oneshot::channel();
+        let first = tokio::spawn(async move {
+            let _guard = BucketDriveFailureGuard::new(source, bucket, drive);
+            first_started_sender.send(()).expect("test should observe first worker start");
+            first_release_receiver.await.expect("first worker should be released");
+        });
+        let second = tokio::spawn(async move {
+            let _guard = BucketDriveFailureGuard::new(source, bucket, drive);
+            second_started_sender
+                .send(())
+                .expect("test should observe second worker start");
+            second_release_receiver.await.expect("second worker should be released");
+        });
+
+        first_started_receiver.await.expect("first scanner worker should start");
+        second_started_receiver.await.expect("second scanner worker should start");
+        assert_eq!(active_bucket_drive_count(source, bucket, drive), 2);
+
+        first_release_sender.send(()).expect("first worker should be released");
+        first.await.expect("first scanner worker should finish");
+        assert_eq!(active_bucket_drive_count(source, bucket, drive), 1);
+
+        second_release_sender.send(()).expect("second worker should be released");
+        second.await.expect("second scanner worker should finish");
+        assert_eq!(active_bucket_drive_count(source, bucket, drive), 0);
+    }
 }

crates/scanner/src/scanner_io/io_cycle.rs

@@ -162,18 +162,18 @@ impl ScannerIOCycle for ECStore {
                 dirty_usage_status,
                 activity_status,
             );
-            if !publish_usage_snapshot(
-                &updates,
-                status,
-                DataUsageInfo {
-                    last_update: Some(SystemTime::now()),
-                    scanner_cycle: Some(want_cycle),
-                    usage_snapshot_complete: true,
-                    ..Default::default()
-                },
-            )
-            .await?
-            {
+            let empty_usage = DataUsageInfo {
+                last_update: Some(SystemTime::now()),
+                scanner_cycle: Some(want_cycle),
+                usage_snapshot_complete: true,
+                ..Default::default()
+            };
+            let observational_snapshot_published = if should_publish_observational_snapshot(status) {
+                publish_observational_snapshot(&updates, empty_usage).await?
+            } else {
+                publish_usage_snapshot(&updates, status, empty_usage).await?
+            };
+            if !observational_snapshot_published {
                 return Ok(ScannerCycleResult::new(status, None).with_publication_epoch(publication_epoch));
             }
             if status == ScannerCycleStatus::Complete {
@@ -188,6 +188,7 @@ impl ScannerIOCycle for ECStore {
             };
             return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
                 .with_publication_epoch(publication_epoch)
+                .with_observational_snapshot_published(observational_snapshot_published)
                 .with_remote_publication_lease_targets(remote_publication_lease_targets)
                 .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements));
         }
@@ -437,13 +438,19 @@ impl ScannerIOCycle for ECStore {
             dirty_usage_status,
             activity_status,
         );
-        if let Some((data_usage_info, _)) = completed_usage {
-            publish_usage_snapshot(&updates, cycle_status, data_usage_info).await?;
+        let observational_snapshot_published = if let Some((data_usage_info, _)) = completed_usage {
+            if should_publish_observational_snapshot(cycle_status) {
+                publish_observational_snapshot(&updates, data_usage_info).await?
+            } else {
+                publish_usage_snapshot(&updates, cycle_status, data_usage_info).await?
+            }
         } else if !ctx.is_cancelled()
             && let Some((data_usage_info, _)) = observational_usage
         {
-            publish_observational_snapshot(&updates, data_usage_info).await?;
-        }
+            publish_observational_snapshot(&updates, data_usage_info).await?
+        } else {
+            false
+        };
         let dirty_usage_clear = should_clear_dirty_usage_snapshot(
             result.is_ok(),
             structurally_complete_snapshot,
@@ -463,6 +470,7 @@ impl ScannerIOCycle for ECStore {
         };
         Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
             .with_publication_epoch(publication_epoch)
+            .with_observational_snapshot_published(observational_snapshot_published)
             .with_remote_publication_lease_targets(remote_publication_lease_targets)
             .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)
             .with_failed_dirty_usage(!failed_buckets.is_empty())

crates/scanner/src/scanner_io/tests.rs

@@ -909,6 +909,47 @@ async fn structurally_complete_superseded_cycles_publish_without_claiming_conver
     );
 }
 
+#[tokio::test]
+async fn post_scan_activity_failure_retains_complete_usage_as_observation() {
+    let (updates, mut receiver) = mpsc::channel(1);
+    let status = ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
+
+    assert!(should_publish_observational_snapshot(status));
+    assert!(
+        publish_observational_snapshot(
+            &updates,
+            DataUsageInfo {
+                last_update: Some(SystemTime::now()),
+                scanner_cycle: Some(7),
+                objects_total_count: 3,
+                objects_total_size: 12,
+                usage_snapshot_complete: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("post-scan activity failure should retain an observation")
+    );
+
+    let observed = receiver.recv().await.expect("observational update should be queued");
+    assert!(!observed.usage_snapshot_complete);
+    assert!(observed.usage_snapshot_partial);
+    assert_eq!(observed.usage_snapshot_converged, Some(false));
+    assert_eq!(observed.objects_total_count, 3);
+    assert_eq!(observed.objects_total_size, 12);
+}
+
+#[test]
+fn only_unverified_activity_allows_post_scan_observation() {
+    assert!(should_publish_observational_snapshot(ScannerCycleStatus::Deferred(
+        ScannerCycleDeferReason::ActivityBaselineUnavailable
+    )));
+    assert!(!should_publish_observational_snapshot(ScannerCycleStatus::Deferred(
+        ScannerCycleDeferReason::DataMovement
+    )));
+    assert!(!should_publish_observational_snapshot(ScannerCycleStatus::Incomplete));
+}
+
 #[test]
 fn scanner_cycle_fails_closed_for_namespace_disappearance() {
     for activity_status in [ScannerCycleActivityStatus::Changed, ScannerCycleActivityStatus::Unchanged] {

crates/scanner/src/storage_api.rs

@@ -92,7 +92,10 @@ pub(crate) use rustfs_ecstore::api::event::{EventArgs as EcstoreEventArgs, send_
 pub(crate) use rustfs_ecstore::api::layout::{
     EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints,
 };
-pub(crate) use rustfs_ecstore::api::object::SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY;
+pub(crate) use rustfs_ecstore::api::notification::scanner_peer_transport_error_message_is_retryable;
+pub(crate) use rustfs_ecstore::api::object::{
+    SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitState,
+};
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::rebalance::{
     RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta,
@@ -106,9 +109,9 @@ pub(crate) use rustfs_ecstore::api::runtime::{
     setup_is_erasure_sd as ecstore_is_erasure_sd,
 };
 pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
+pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::storage::init_local_disks_with_instance_ctx as ecstore_init_local_disks_with_instance_ctx;
-pub(crate) use rustfs_ecstore::api::storage::{ECStore as EcstoreStore, SCANNER_PUBLICATION_LEASE_TTL_MS};
 use rustfs_storage_api as storage_contracts;
 
 pub(crate) mod owner {
@@ -125,7 +128,7 @@ pub(crate) mod owner {
         EcstoreNsScannerOpenRequest, EcstoreObjectLockConfiguration, EcstoreObjectOpts, EcstoreReplicationConfigurationExt,
         EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore,
         EcstoreVersioningApi, EcstoreVersioningConfiguration, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY,
-        SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerReplicationHealObject, ScannerReplicationHealResult,
+        ScannerPublicationCommitScope, ScannerPublicationCommitState, ScannerReplicationHealObject, ScannerReplicationHealResult,
         ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_expiry_state_handle,
         ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config, ecstore_get_object_lock_config,
         ecstore_get_replication_config, ecstore_invalidate_admin_data_usage_snapshot_cache,

crates/targets/src/store.rs

@@ -641,7 +641,7 @@ impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
         // The closure always returns Some, so the update never fails and the Result is discarded.
         let _ = self
             .failed_count
-            .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |current| Some(current.saturating_sub(1)));
+            .try_update(Ordering::SeqCst, Ordering::SeqCst, |current| Some(current.saturating_sub(1)));
     }
 
     /// Maps a per-entry stat outcome inside the ordered failed scan. A NotFound error means the file

crates/targets/src/testkit.rs

@@ -53,7 +53,7 @@ impl Drop for HealthDropGuard {
 /// A budget of `usize::MAX` behaves as "always fail" for any realistic call count.
 fn consume_failure_budget(budget: &AtomicUsize) -> bool {
     budget
-        .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| remaining.checked_sub(1))
+        .try_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| remaining.checked_sub(1))
         .is_ok()
 }
 

docs/operations/kms-admin-contract.md

@@ -10,6 +10,7 @@ The wire prefix is `/rustfs/admin/v3`. Request and response field names for the
 | `POST /kms/reconfigure` | `kms:Configure` / high | no | supported | supported | none |
 | `POST /kms/start` | `kms:ServiceControl` / high | no | supported | supported | none |
 | `POST /kms/stop` | `kms:ServiceControl` / high | no | supported | supported | none |
+| `POST /kms/reload` | `kms:ServiceControl` / high | no | pending | pending | Re-reads the cluster-persisted configuration without resubmitting secrets; response reuses the configure shape. |
 | `GET /kms/config` | `kms:Configure` / sensitive | no | no | supported | Redact operational paths before display. |
 | `POST /kms/clear-cache` | `kms:ClearCache` / high | no | no | supported | Keep the current `{status,message}` response stable. |
 | `POST /kms/keys` | `kms:Configure` / high | no | supported | supported | none |
@@ -52,7 +53,7 @@ One case is deliberately an error rather than a report: a listing that covered t
 
 ## Server-side snapshot coverage
 
-The merged #5626 producer snapshots cover the nine modern/legacy key response types and the metadata response type served by `kms_keys.rs` and `kms_key_metadata.rs`: create, describe, list, generate-data-key, delete, cancel-deletion, update-description, tag, and untag. The four dynamic responses served verbatim by `kms_dynamic.rs` are covered in `crates/kms/src/snapshots/`: configure, start, stop, and the `service-status` response.
+The merged #5626 producer snapshots cover the nine modern/legacy key response types and the metadata response type served by `kms_keys.rs` and `kms_key_metadata.rs`: create, describe, list, generate-data-key, delete, cancel-deletion, update-description, tag, and untag. The four dynamic responses served verbatim by `kms_dynamic.rs` are covered in `crates/kms/src/snapshots/`: configure, start, stop, and the `service-status` response. `POST /kms/reload` serves the same `ConfigureKmsResponse` type the configure snapshot pins; it adds no new wire shape.
 
 `POST /kms/clear-cache` now has a named `KmsClearCacheResponse` and a producer snapshot beside the others; its serialized bytes are unchanged from the inline JSON it replaced.
 
@@ -60,7 +61,7 @@ The remaining wire-shape gaps are intentionally documented rather than duplicate
 
 ## Client handoff gaps
 
-The `rc` client currently has status, key list/status/create/delete/cancel-deletion, configure/reconfigure/start/restart/stop, and diagnostic/roundtrip entry points. It has no lifecycle enable/disable/rotate, key metadata, or backup/restore commands. The console currently calls service-status, configure/reconfigure/start/stop/config, clear-cache, status, and the modern key CRUD routes. It has no lifecycle, metadata, or backup/restore UI. These pending cells are delivery items for `rustfs/cli` and `rustfs/console`; they are not implemented in this repository. A read-only issue search on 2026-08-02 found no matching KMS issue in either client repository, so the client handoff still needs issue creation there.
+The `rc` client currently has status, key list/status/create/delete/cancel-deletion, configure/reconfigure/start/restart/stop, and diagnostic/roundtrip entry points. It has no lifecycle enable/disable/rotate, key metadata, backup/restore, or reload commands; `POST /kms/reload` is the recovery path when a restarted server reports not-configured while a persisted configuration exists, so it is a client delivery item alongside the lifecycle gaps. The console currently calls service-status, configure/reconfigure/start/stop/config, clear-cache, status, and the modern key CRUD routes. It has no lifecycle, metadata, or backup/restore UI. These pending cells are delivery items for `rustfs/cli` and `rustfs/console`; they are not implemented in this repository. A read-only issue search on 2026-08-02 found no matching KMS issue in either client repository, so the client handoff still needs issue creation there.
 
 `POST /kms/generate-data-key` is deliberately marked “do not expose” for both clients: its response contains a base64 plaintext data key. `GET /kms/config` and backup status/restore responses contain operational paths and identifiers, not key material, but still require UI/CLI redaction and confirmation handling.
 

docs/operations/kms-observability-runbook.md

@@ -212,6 +212,17 @@ Investigation:
 
 Related signals: `rotation_due` / `rotation_due_reason` on the key listing; `rustfs_kms_deletion_sweep_keys_total{outcome=~"unreadable|failed"}` (a frozen gauge is stale, not healthy); the [rotation drivers and scheduling matrix](kms-backend-security.md#rotation-drivers-and-scheduling-per-backend) and pre-rotation checklist in the backend security properties document.
 
+## Startup persisted-configuration load
+
+KMS configured through the admin API is persisted to cluster storage and restored on every startup. The load result is visible in two places; check both before concluding that KMS "was never configured":
+
+- **Startup log**, `event="kms_persisted_config_lookup"` (`target: rustfs::init`): `state="found"` means the persisted configuration was loaded and applied; `state="not_found"` means no persisted configuration exists on disk; `state="load_failed"` means one exists but reading, unsealing, or decoding it failed.
+- **`GET /rustfs/admin/v3/kms/service-status`**: `"NotConfigured"` matches `not_found` (nothing persisted — configuring from scratch is the correct response), while a status of `Error("Failed to load persisted KMS configuration: ...")` or `Error("Failed to apply persisted KMS configuration: ...")` matches `load_failed`. The two states call for different operator actions; do not resubmit a full configuration to recover from `load_failed`.
+
+To recover from `load_failed` — or from any state where the server runs but its in-memory KMS lags the persisted configuration — call `POST /rustfs/admin/v3/kms/reload` (requires `kms:ServiceControl`). It re-reads the persisted configuration from cluster storage and reconfigures the service without resubmitting secrets, then broadcasts the reload to peer nodes. If reload keeps failing, check cluster storage health first (the read needs quorum), then `RUSTFS_KMS_CONFIG_SECRET`: an unseal error means the secret is missing or differs from the one that sealed the persisted copy — it must be identical on every node.
+
+A separate event, `kms_config_load_skipped` with `reason="storage_uninitialized"`, comes from the ambient loader used by the peer-reload RPC path; during normal startup the loader receives the store explicitly, so seeing this event outside a peer reload indicates a request arrived before storage initialization finished.
+
 ## Threshold calibration
 
 Every numeric traffic or latency threshold in `rustfs-kms-alerts.yml` (5% error ratio, 2s p99, 0.5/s attempt failures, 0.05/s budget exhaustion) is a conservative default chosen without a production baseline, biased toward not paging on healthy-but-busy systems. Before relying on these alerts for paging: run the workload in staging for at least a week, record the steady-state values of the expressions above, then tighten thresholds to sit clearly above observed peaks. `KmsBackendCircuitOpen` is different: its gauge is direct state, and the one-minute hold only suppresses a circuit that recovers immediately. `KmsKeyRotationOverdue` is different in the other direction: its 400-day threshold is a policy default (sitting above a common one-year rotation period), not a traffic default — calibrate it against the rotation period your compliance policy requires and against `RUSTFS_KMS_ROTATION_MAX_AGE_SECS`, not against a staging baseline. Once a stable baseline exists, consider converting `KmsBackendAttemptFailureSpike` to a baseline-relative form (`offset 1d` ratio, see `.docker/observability/prometheus-rules/rustfs-get-optimization-alerts.yaml` for the pattern). Formal SLO targets for KMS operations are deliberately out of scope until that baseline exists (rustfs/backlog#1584).

flake.nix

@@ -35,6 +35,23 @@
       forAllSystems = nixpkgs.lib.genAttrs systems;
     in
     {
+      nixosModules.rustfs = import ./nix/rustfs-module.nix {
+        defaultPackage = system: self.packages.${system}.rustfs;
+      };
+      nixosModules.default = self.nixosModules.rustfs;
+
+      overlays.default = final: prev:
+        let
+          packages = self.packages.${prev.stdenv.hostPlatform.system};
+        in
+        {
+          rustfs = packages.rustfs;
+        }
+        // prev.lib.optionalAttrs (builtins.hasAttr "rustfs-client" packages) {
+          rustfs-client = packages.rustfs-client;
+          rc = packages.rustfs-client;
+        };
+
       packages = forAllSystems (
         system:
         let
@@ -55,9 +72,10 @@
             cargo = rustToolchain;
             rustc = rustToolchain;
           };
-        in
-        {
-          default = rustPlatform.buildRustPackage {
+
+          clientVersion = "0.1.32";
+
+          rustfs = rustPlatform.buildRustPackage {
             pname = "rustfs";
             version = "1.0.0-rc.4";
 
@@ -82,7 +100,6 @@
               "rustfs"
             ];
 
-            # Set environment variables for build
             PROTOC = "${pkgs.protobuf}/bin/protoc";
 
             doCheck = false;
@@ -94,6 +111,55 @@
               mainProgram = "rustfs";
             };
           };
+
+          clientAssets = {
+            "x86_64-linux" = {
+              name = "rustfs-cli-linux-amd64-v${clientVersion}.tar.gz";
+              hash = "sha256-qwDZNwedy28ce0HTS7+q0OsL1PchhnLLy3wzZS0cRt8=";
+            };
+            "aarch64-linux" = {
+              name = "rustfs-cli-linux-arm64-v${clientVersion}.tar.gz";
+              hash = "sha256-1T1M9Q3lcy9IJo/n5eQezmbTaEgHVJbx1QFCaTX3BYY=";
+            };
+          };
+
+          clientSupported = builtins.hasAttr system clientAssets;
+
+          clientPackage =
+            if clientSupported then
+              let
+                asset = clientAssets.${system};
+              in
+              pkgs.stdenvNoCC.mkDerivation {
+                pname = "rustfs-cli";
+                version = clientVersion;
+                src = pkgs.fetchurl {
+                  url = "https://github.com/rustfs/cli/releases/download/v${clientVersion}/${asset.name}";
+                  inherit (asset) hash;
+                };
+                sourceRoot = ".";
+                installPhase = ''
+                  runHook preInstall
+                  install -Dm755 rc "$out/bin/rc"
+                  runHook postInstall
+                '';
+                meta = {
+                  description = "RustFS S3-compatible command-line client";
+                  homepage = "https://github.com/rustfs/cli";
+                  license = pkgs.lib.licenses.asl20;
+                  mainProgram = "rc";
+                };
+              }
+            else
+              null;
+        in
+        {
+          inherit rustfs;
+          default = rustfs;
+        }
+        // pkgs.lib.optionalAttrs clientSupported {
+          rustfs-client = clientPackage;
+          rc = clientPackage;
         }
       );