fix(scanner): retain cleanup scope after cancellation

Co-Authored-By: heihutu <[email protected]>

.agents/skills/arch-checks/SKILL.md

@@ -50,11 +50,10 @@ consider adding it to the script's `checked_files` list.
 
 ## `check_doc_paths.sh`
 
-Instruction docs (`AGENTS.md`, `CLAUDE.md`, `ARCHITECTURE.md`) and every
-Markdown file under `docs/` (architecture, operations, testing, index) must not
-reference repo file paths that no longer exist. If your refactor moved code,
-update the docs that point at it — the error message lists `doc -> stale-path`
-pairs. Cite paths plus symbol names, never line numbers (see `docs/README.md`).
+Instruction/architecture docs (`AGENTS.md`, `CLAUDE.md`, `ARCHITECTURE.md`,
+`docs/architecture/*.md`) must not reference repo file paths that no longer
+exist. If your refactor moved code, update the docs that point at it — the
+error message lists `doc -> stale-path` pairs.
 
 ## `check_no_planning_docs.sh`
 

.agents/skills/security-advisory-lessons/references/advisory-patterns.md

@@ -48,7 +48,6 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
 
 ### S3 object actions, copy, multipart, and upload policy validation
 
-- `GHSA-g8w9-qw9q-fghr`: a valid presigned `PutObject` accepted extra `x-amz-tagging`, website redirect, and storage-class headers omitted from `SignedHeaders`. Lesson: a presigned URL is a bounded capability; reject `x-amz-*` headers that are not cryptographically bound by the signature so unsigned metadata cannot change authorization, lifecycle, redirect, cost, or durability semantics.
 - `GHSA-3ppv-fx5m-m749`: explicit `versionId` reads and copy sources authorized `s3:GetObject` instead of `s3:GetObjectVersion`. Lesson: version-specific object access must select version-specific actions for direct reads, `CopyObject`, and `UploadPartCopy`, with tests proving the backend is not reached on denial.
 - `GHSA-x298-9x87-fvjq`: anonymous `ListObjectVersions` fell back to `ListBucket` and returned before public-access-block gates. Lesson: compatibility fallbacks must converge on the same post-authorization checks as direct grants, especially `RestrictPublicBuckets` and anonymous data-plane denies.
 - `GHSA-mx42-j6wv-px98`: `UploadPartCopy` missed source authorization and allowed cross-bucket object exfiltration. Lesson: multipart copy must enforce the same source and destination contract as `CopyObject`.
@@ -120,7 +119,7 @@ Use these targeted searches when a diff touches security-sensitive code:
 ```bash
 rg -n "validate_admin_request|check_permissions|AdminAction::|deny_only|is_allowed" rustfs crates
 rg -n "authorize_operation|FtpsDriver|SftpDriver|RETR|MKD|SIZE|MDTM|CreateBucket|GetObject|HeadObject" crates/protocols rustfs
-rg -n "UploadPartCopy|upload_part_copy|CompleteMultipart|PostObject|presign|SignedHeaders|content-length-range|starts-with" rustfs crates
+rg -n "UploadPartCopy|upload_part_copy|CompleteMultipart|PostObject|content-length-range|starts-with" rustfs crates
 rg -n "ListBucketVersions|GetObjectVersion|versionId|VersionId|ExistingObjectTag|ForAllValues|ForAnyValue|POLICY_PLUGIN|opa" rustfs crates
 rg -n "normalize_extract_entry_key|Snowball|auto-extract|PathBuf::join|canonicalize|\\.\\.|x-forwarded-for|x-real-ip|SourceIp" rustfs crates
 rg -n "DEFAULT_SECRET|DEFAULT_ACCESS|TEST_PRIVATE_KEY|rustfs rpc|RUSTFS_RPC_SECRET" rustfs crates
@@ -137,7 +136,6 @@ rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
 - Protocol frontend authz fixes: include denied `RETR`, `SIZE`/`MDTM`, `MKD`, bucket probe, and sibling allowed-operation cases, and assert denied paths do not reach the storage backend.
 - IAM fixes: include import/update/list service-account cases with attacker-controlled parent, claims, access key, secret key, and policy.
 - Copy/upload fixes: include cross-bucket, cross-user, source-denied, destination-denied, copy-source-condition, and multipart completion cases.
-- Presigned upload fixes: include a valid presign with extra unsigned tagging, redirect, and storage-class headers; require rejection before storage access, and verify explicitly signed equivalents still work.
 - Version-action fixes: include historical UUID, explicit current version, `null`, range, partNumber, presigned, STS/session, service-account, anonymous bucket-policy, copy source, and multipart-copy source cases.
 - Policy-condition fixes: include reserved-key header collisions, missing keys, partially overlapping multi-value sets, plugin mode, and built-in policy mode.
 - Path fixes: include encoded traversal, absolute path, nested traversal, archive entries with `..`, valid object keys that resemble traversal text but should be rejected, and canonical bucket/prefix boundary checks.

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=ef914ec0b8daa9c2c5e52f501d339914662f42d6f6ed9d33877d56b97adf16f9
-sha256-linux=a8a816d7bb0e7cb5632b1863b33794bcb9fc7e765f150aa5e1bf16518e28dfb4
+sha256-darwin=d6aa36cfaae2c4d8590482c7e47138c5965b335b34a75f50d11ffc3366e9021e
+sha256-linux=96db8060fce98addda4f69092d297ca236bec4892d820617a26a261eedac61b0

.config/e2e-nightly-selection.txt

@@ -1 +1 @@
-sha256=d06524b44de97ed8f62b0fd8cf9fa504e3cd520ffcaacc32691d6f890ebe7f20
+sha256=9b9bc336b43b70d0e06e0adb5455bf035bb18945d85d60936eb6fe4d48e0e680

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=db9bd8cdcb0abe43461aa6b36499b17cabd4098e5b34e300b1a0f0d0f34d9884
+sha256=294350518743cac8d7c41880a2835216e4b697908d7b0b1bc92b62816d94c59d

.config/make/coverage.mak

@@ -23,4 +23,4 @@ coverage: core-deps ## Workspace line coverage (cargo-llvm-cov + nextest; slow,
 	@mkdir -p target/llvm-cov
 	cargo llvm-cov report --lcov --output-path target/llvm-cov/lcov.info
 	cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
-	$(RUSTFS_PYTHON_BIN) scripts/coverage_per_crate.py target/llvm-cov/coverage.json
+	python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json

.config/make/lint-fmt.mak

@@ -88,7 +88,7 @@ offline-enrollment-e2e-check: core-deps ## Build and exercise the dedicated offl
 .PHONY: test-wiring-check
 test-wiring-check: ## Check tests stay registered and selected by their intended runners
 	@echo "🧪 Checking test wiring..."
-	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py
+	python3 ./scripts/check_test_wiring.py
 
 .PHONY: log-analyzer-rules-check
 log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source

.config/make/tests.mak

@@ -35,14 +35,13 @@ script-tests: ## Run shell script tests
 	./scripts/test_pinned_paired_abba_bench.sh
 	./scripts/test_manual_transition_runbooks.sh
 	./scripts/test_fuzz_runner.sh
-	./scripts/test_python_bin.sh
 	./scripts/check_embedded_secrets.sh --self-test
-	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
-	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
-	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
-	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
+	python3 ./scripts/check_test_wiring.py --self-test
+	python3 ./scripts/check_security_coverage.py --self-test
+	python3 ./scripts/check_scheduled_validation_freshness.py --self-test
+	python3 ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
-	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
+	python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
 	./scripts/validate_object_data_cache_cold_stampede.sh --self-test
 
 .PHONY: test

.config/nextest.toml

@@ -46,11 +46,6 @@ e2e-reliability = { max-threads = 1 }
 e2e-inline-boundaries = { max-threads = 1 }
 e2e-cluster-nightly = { max-threads = 1 }
 
-# Deep async storage futures are composed into tests across several crates.
-# Keep the test stack bounded but above libtest's 2 MiB default.
-[scripts.setup.ecstore-base-stack]
-command = ['sh', '-c', 'echo RUST_MIN_STACK=4194304 >> "$NEXTEST_ENV"']
-
 # These exact regression scenarios build deep async storage futures that exceed
 # libtest's 2 MiB spawned-thread stack on Linux. Give only their test processes
 # the same 32 MiB stack already used by the crate's dedicated large-stack tests.
@@ -65,13 +60,9 @@ command = ['sh', '-c', 'echo RUST_MIN_STACK=33554432 >> "$NEXTEST_ENV"']
 
 # --- default profile (local): serialize the flaky groups, never retry --------
 [[profile.default.scripts]]
-filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
+filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
 setup = 'ecstore-large-stack'
 
-[[profile.default.scripts]]
-filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
-setup = 'ecstore-base-stack'
-
 [[profile.default.scripts]]
 filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
 setup = 'lifecycle-large-stack'
@@ -89,29 +80,6 @@ test-group = 'ecstore-serial-flaky'
 filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
 test-group = 'ecstore-serial-flaky'
 
-# Serialize the heal result-report tests. Every test in the module builds a
-# real-disk (TempDir-backed) hermetic erasure set and drives MiB-scale writes
-# plus deep-scan heal — the same load-sensitive cross-disk IO shape as the
-# crash_consistency scenarios above. Under a heavily parallel run a single
-# disk's IO can fail while write quorum still holds, which flips per-disk
-# readback and aggregate-outcome assertions nondeterministically (different
-# tests each round; all pass standalone). Preventive serialization only, no
-# retries. The matching ci-profile override is after [profile.ci].
-[[profile.default.overrides]]
-filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::heal::heal_result_report_tests::/)'
-test-group = 'ecstore-serial-flaky'
-
-# Serialize the metadata-cache generation-retirement pair. Both carry
-# #[serial(metadata_cache_invalidation_probe)] — a no-op across nextest's
-# process boundary — and assert get_object_metadata_cache generation
-# semantics on a 4-disk hermetic set, the same load-sensitive shape that
-# forced the transition matrix tests into this group. Preventive
-# serialization only, no retries. The matching ci-profile override is after
-# [profile.ci].
-[[profile.default.overrides]]
-filter = 'package(rustfs-ecstore) & test(retires_cached_snapshot)'
-test-group = 'ecstore-serial-flaky'
-
 # The production-handler relocation regression builds an isolated 8-disk,
 # 2-pool store and commits a 72 MiB multipart object. Keep that cross-disk IO
 # from overlapping the ecstore commit fixtures above.
@@ -132,29 +100,12 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'
 
-# Serialize the transition matrix tests. They build a 4-disk hermetic erasure
-# set, populate the get_object_metadata_cache, and assert generation lifecycle
-# semantics. serial_test's #[serial] has no effect across nextest's process
-# boundary, so concurrent execution races the shared metadata-cache generation
-# counter and causes spurious "metadata read should publish the generation"
-# panics. Preventive serialization, no retries.
-[[profile.default.overrides]]
-filter = 'package(rustfs-ecstore) & test(set_disk::transition_matrix_tests::)'
-test-group = 'ecstore-serial-flaky'
-
 # The durable ILM decommission regressions build isolated multi-pool stores and
 # deliberately take source or target disks offline while checking fencing.
 [[profile.default.overrides]]
 filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
 test-group = 'ecstore-serial-flaky'
 
-# Decommission entry and marker/barrier tests share process-wide fault hooks and
-# deterministic commit barriers. Keep the whole init decommission family in one
-# nextest group; serial_test alone cannot isolate separate test processes.
-[[profile.default.overrides]]
-filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_|suspended_.*decommission)$/)'
-test-group = 'ecstore-serial-flaky'
-
 # Serialize the bucket-incarnation / lifecycle-fence tests. They drive
 # init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
 # OnceLock state that serial_test's #[serial] cannot protect across nextest's
@@ -206,13 +157,9 @@ fail-fast = false
 path = "junit.xml"
 
 [[profile.ci.scripts]]
-filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
+filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
 setup = 'ecstore-large-stack'
 
-[[profile.ci.scripts]]
-filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
-setup = 'ecstore-base-stack'
-
 [[profile.ci.scripts]]
 filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
 setup = 'lifecycle-large-stack'
@@ -273,20 +220,6 @@ test-group = 'e2e-reliability'
 filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::multipart::tests::crash_consistency::/)'
 test-group = 'ecstore-serial-flaky'
 
-# Serialize the heal result-report tests under the ci profile too (see the
-# matching default-profile override near the top). Not a quarantine: no
-# retries, just serialized real-disk heal IO.
-[[profile.ci.overrides]]
-filter = 'package(rustfs-ecstore) & test(/^set_disk::ops::heal::heal_result_report_tests::/)'
-test-group = 'ecstore-serial-flaky'
-
-# Serialize the metadata-cache generation-retirement pair under the ci
-# profile too (see the matching default-profile override near the top). Not a
-# quarantine: no retries.
-[[profile.ci.overrides]]
-filter = 'package(rustfs-ecstore) & test(retires_cached_snapshot)'
-test-group = 'ecstore-serial-flaky'
-
 # Match the default-profile embedded test isolation without quarantining or
 # retrying failures in CI.
 [[profile.ci.overrides]]
@@ -299,20 +232,10 @@ test-group = 'embedded-test-ports'
 filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
 test-group = 'ecstore-serial-flaky'
 
-# Serialize the transition matrix tests under the ci profile too (see the
-# matching default-profile override near the top). No retries.
-[[profile.ci.overrides]]
-filter = 'package(rustfs-ecstore) & test(set_disk::transition_matrix_tests::)'
-test-group = 'ecstore-serial-flaky'
-
 [[profile.ci.overrides]]
 filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
 test-group = 'ecstore-serial-flaky'
 
-[[profile.ci.overrides]]
-filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_|suspended_.*decommission)$/)'
-test-group = 'ecstore-serial-flaky'
-
 # Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
 # too (see the matching default-profile override near the top). No retries.
 [[profile.ci.overrides]]
@@ -355,8 +278,7 @@ test-group = 'ecstore-serial-flaky'
 # allowlist", so any new replication test lands in nightly by default (never
 # silently unrun) until it is explicitly blessed as fast here. Keep the two
 # regexes byte-identical. The committed profile selection digests make changes
-# visible in CI; list current membership with `cargo nextest list -p e2e_test
-# --profile <profile>` (platform-dependent; see docs/testing/README.md).
+# visible in CI; current counts live in docs/testing/e2e-suite-inventory.md.
 # HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
 # (#4724) because they set a loopback (127.0.0.1) replication target that the
 # SSRF egress guard rejected on every PR after repl-1 (#4712). That is fixed —
@@ -478,7 +400,7 @@ path = "junit.xml"
 [profile.e2e-nightly]
 default-filter = """
     package(e2e_test)
-    & test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
+    & test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
 """
 fail-fast = false
 
@@ -509,7 +431,7 @@ path = "junit.xml"
 # quota, checksum, encryption,
 # security-boundary, ... — that the fast PR `e2e-smoke` subset deliberately
 # skips. Budget <= 45 min; authority for the suite count is `cargo nextest list
-# --profile e2e-full -p e2e_test` (platform-dependent; see docs/testing/README.md).
+# --profile e2e-full` (see docs/testing/e2e-suite-inventory.md).
 #
 # The filter is "the whole e2e_test crate MINUS the sets owned by other lanes":
 #   * protocols:: — FTPS/SFTP/WebDAV, run from the dedicated protocol profile
@@ -530,12 +452,23 @@ path = "junit.xml"
 # parallel-safe — the same property e2e-smoke relies on. The exceptions are the
 # 4-disk reliability / degraded-read fault-injection tests and the fixed-port
 # Vault tests, both serialized below.
+# KNOWN-FAILURE EXCLUSIONS (characterization run 29381309848, 2026-07-15:
+# 341 ran / 32 failed on the suites' first automated run ever). Deterministic
+# product failures cannot be quarantined away with retries, so each family is
+# excluded here with its tracking issue, under the same discipline as the
+# ci-profile quarantine (docs/testing/README.md): every entry MUST cite one
+# OPEN issue, and the fixing PR MUST delete the exclusion. The passing
+# negative-path siblings of each family stay in as regression guards.
+#   * rustfs#4843 — over-limit archive entry paths hard-reject the whole
+#     archive even under ignore-errors semantics.
 [profile.e2e-full]
 default-filter = """
     package(e2e_test)
     & !test(/^protocols::/)
-    & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
+    & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
     & !test(/^replication_extension_test::/)
+    & !test(/^multipart_auth_test::test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_enabled$/)
+    & !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(ignores_invalid_entries_when_requested|supports_standard_headers_with_combined_extract_options)$/)
 """
 fail-fast = false
 

.github/workflows/audit.yml

@@ -24,11 +24,8 @@ on:
       - '.github/actions/**'
       - '.github/workflows/**'
       - 'scripts/release/create_or_update_release.sh'
-      - 'scripts/release/package_versions.sh'
-      - 'scripts/test_package_versions.sh'
       - 'scripts/security/check_performance_ab_workflow.sh'
       - 'scripts/security/check_preview_release_workflow.sh'
-      - 'scripts/security/check_tier_artifact_workflow.sh'
       - 'scripts/security/check_workflow_pins.sh'
   pull_request:
     types: [ opened, synchronize, reopened, closed ]
@@ -40,11 +37,8 @@ on:
       - '.github/actions/**'
       - '.github/workflows/**'
       - 'scripts/release/create_or_update_release.sh'
-      - 'scripts/release/package_versions.sh'
-      - 'scripts/test_package_versions.sh'
       - 'scripts/security/check_performance_ab_workflow.sh'
       - 'scripts/security/check_preview_release_workflow.sh'
-      - 'scripts/security/check_tier_artifact_workflow.sh'
       - 'scripts/security/check_workflow_pins.sh'
   schedule:
     # Daily, not weekly. This schedule exists to catch RustSec advisories
@@ -152,12 +146,6 @@ jobs:
       - name: Check performance A/B workflow trust boundary
         run: ./scripts/security/check_performance_ab_workflow.sh
 
-      - name: Check tier evidence workflow isolation
-        run: ./scripts/security/check_tier_artifact_workflow.sh
-
-      - name: Check package version contract
-        run: ./scripts/test_package_versions.sh
-
   dependency-review:
     name: Dependency Review
     runs-on: ubuntu-latest

.github/workflows/build.yml

@@ -244,7 +244,7 @@ jobs:
     needs: [ build-check, prepare-platform-matrix ]
     if: needs.build-check.outputs.should_build == 'true' && needs.prepare-platform-matrix.result == 'success'
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 180
+    timeout-minutes: 150
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
       # Release binaries ship without dial9 telemetry and therefore do not need
@@ -408,9 +408,9 @@ jobs:
 
           if [[ "${{ matrix.cross }}" == "true" ]]; then
             # All cross targets in the matrix are Linux; zigbuild handles them.
-            cargo zigbuild --release --target ${{ matrix.target }} -p rustfs --bin rustfs
+            cargo zigbuild --release --target ${{ matrix.target }} -p rustfs --bins
           else
-            cargo build --release --target ${{ matrix.target }} -p rustfs --bin rustfs
+            cargo build --release --target ${{ matrix.target }} -p rustfs --bins
           fi
 
       - name: Create release package
@@ -1065,7 +1065,7 @@ jobs:
           while IFS= read -r preview_tag; do
             [[ -n "$preview_tag" ]] || continue
             echo "🧹 Deleting preview release $preview_tag (tag kept)"
-            gh release delete "$preview_tag" --repo "${GITHUB_REPOSITORY}" --yes
+            gh release delete "$preview_tag" --yes
             DELETED=$((DELETED + 1))
           done < <(
             jq -r --arg tag "$TAG" '

.github/workflows/e2e-upgrade.yml

@@ -49,20 +49,8 @@ env:
   UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
 
 jobs:
-  upgrade:
-    name: ${{ matrix.name }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - name: Direct upgrade from rc.2
-            cache_key: e2e-direct-upgrade
-            test: direct_upgrade_from_rc2_preserves_object_contracts
-            artifact: direct-upgrade
-          - name: Mixed-version rolling upgrade from rc.2
-            cache_key: e2e-mixed-version-upgrade
-            test: rolling_upgrade_from_rc2_preserves_mixed_version_contracts
-            artifact: mixed-version-upgrade
+  direct-upgrade:
+    name: Direct upgrade from rc.2
     runs-on: ubuntu-latest
     timeout-minutes: 60
     env:
@@ -76,7 +64,7 @@ jobs:
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
-          cache-shared-key: ${{ matrix.cache_key }}
+          cache-shared-key: e2e-direct-upgrade
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
           install-build-packaging-tools: "false"
 
@@ -101,17 +89,17 @@ jobs:
           cargo build --locked -p rustfs --bin rustfs
           : > target/debug/rustfs.features
 
-      - name: Run upgrade compatibility test
+      - name: Run direct-upgrade compatibility test
         run: |
           cargo test --locked -p e2e_test \
-            "upgrade_compatibility_test::${{ matrix.test }}" \
+            upgrade_compatibility_test::direct_upgrade_from_rc2_preserves_object_contracts \
             -- --ignored --exact --nocapture
 
       - name: Upload server logs
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: ${{ matrix.artifact }}-server-logs-${{ github.run_number }}
+          name: direct-upgrade-server-logs-${{ github.run_number }}
           path: ${{ runner.temp }}/rustfs-upgrade-logs
           if-no-files-found: warn
           retention-days: 14

.github/workflows/package.yml

@@ -21,10 +21,10 @@
 # - workflow_run: automatically package after "Build and Release" completes
 #   for a release tag (the mac/windows/linux binaries are already uploaded
 #   to the GitHub release before packaging starts)
-# - workflow_dispatch: manual fallback with a release tag and/or exact build run ID
+# - workflow_dispatch: manual fallback (backfill / re-run) with optional tag/run_id
 #
 # Flow:
-# 1. Resolve and validate the selected Build workflow run and source identity
+# 1. Resolve the triggering Build workflow run for the release tag
 # 2. Download Linux binaries (x86_64-gnu, aarch64-gnu) from build artifacts
 # 3. Build DEB packages for amd64 and arm64
 # 4. Build RPM packages for x86_64 and aarch64
@@ -51,7 +51,7 @@ on:
         required: false
         type: string
       build_run_id:
-        description: "Build workflow run ID (when combined with tag, both must identify the same release commit)"
+        description: "Build workflow run ID (overrides tag lookup)"
         required: false
         type: string
 
@@ -82,9 +82,6 @@ jobs:
       version: ${{ steps.resolve.outputs.version }}
       build_type: ${{ steps.resolve.outputs.build_type }}
       build_run_id: ${{ steps.resolve.outputs.build_run_id }}
-      build_run_number: ${{ steps.resolve.outputs.build_run_number }}
-      head_sha: ${{ steps.resolve.outputs.head_sha }}
-      dev_sequence: ${{ steps.resolve.outputs.dev_sequence }}
       tag: ${{ steps.resolve.outputs.tag }}
     steps:
       - name: Resolve build run
@@ -92,129 +89,90 @@ jobs:
         shell: bash
         env:
           GH_TOKEN: ${{ github.token }}
-          EVENT_NAME: ${{ github.event_name }}
-          REPOSITORY: ${{ github.repository }}
           INPUT_TAG: ${{ github.event.inputs.tag }}
           INPUT_RUN_ID: ${{ github.event.inputs.build_run_id }}
         run: |
           set -euo pipefail
 
-          fail() {
-            echo "❌ $1" >&2
-            exit 1
-          }
-
-          TAG=""
-          BUILD_RUN_ID=""
-          case "$EVENT_NAME" in
-            workflow_run)
-              TAG="$HEAD_BRANCH"
-              BUILD_RUN_ID="$WORKFLOW_RUN_ID"
-              ;;
-            workflow_dispatch)
-              TAG="$INPUT_TAG"
-              BUILD_RUN_ID="$INPUT_RUN_ID"
-              ;;
-            *) fail "unsupported event: $EVENT_NAME" ;;
-          esac
-
-          # Validate and classify tags before using them in API paths or logs.
-          semver_core='(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)'
-          prerelease_id='(alpha|beta|rc)\.(0|[1-9][0-9]*)'
-          if [[ -n "$TAG" ]]; then
-            if [[ "$TAG" =~ ^${semver_core}-${prerelease_id}-preview\.(0|[1-9][0-9]*)$ ]]; then
-              BUILD_TYPE=preview
-            elif [[ "$TAG" =~ ^${semver_core}-${prerelease_id}$ ]]; then
-              BUILD_TYPE=prerelease
-            elif [[ "$TAG" =~ ^${semver_core}$ ]]; then
-              BUILD_TYPE=release
-            else
-              fail "tag is not a supported strict package version"
-            fi
+          # Determine tag
+          if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
+            TAG="${HEAD_BRANCH}"
+          elif [[ -n "$INPUT_TAG" ]]; then
+            TAG="$INPUT_TAG"
           else
-            BUILD_TYPE=development
+            TAG=""
           fi
 
-          if [[ -n "$BUILD_RUN_ID" ]]; then
-            [[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ ]] || fail "build run ID must be a positive decimal integer"
-            echo "Using selected build run: $BUILD_RUN_ID"
-          elif [[ -n "$TAG" ]]; then
-            echo "Looking for build run for tag: $TAG"
-            BUILD_RUN_ID=$(gh api --method GET \
-              "repos/${REPOSITORY}/actions/workflows/build.yml/runs" \
-              -f branch="$TAG" -f status=success -F per_page=1 \
-              --jq '.workflow_runs[0].id // empty' 2>/dev/null || true)
+          echo "Tag: ${TAG:-<none>}"
 
-            if [[ -z "$BUILD_RUN_ID" ]]; then
-              BUILD_RUN_ID=$(gh api --method GET \
-                "repos/${REPOSITORY}/actions/workflows/build.yml/runs" \
-                -f event=push -f status=success -F per_page=100 2>/dev/null |
-                jq -r --arg tag "$TAG" \
-                  '[.workflow_runs[] | select(.head_branch == $tag)][0].id // empty' || true)
+          # Determine build run ID
+          BUILD_RUN_ID=""
+
+          if [[ -n "$INPUT_RUN_ID" ]]; then
+            # Explicit run ID takes priority
+            BUILD_RUN_ID="$INPUT_RUN_ID"
+            echo "Using explicit build run ID: $BUILD_RUN_ID"
+
+          elif [[ "${{ github.event_name }}" == "workflow_run" ]]; then
+            # Use the Build and Release run that triggered this workflow
+            BUILD_RUN_ID="${WORKFLOW_RUN_ID}"
+            echo "Using triggering workflow run: $BUILD_RUN_ID"
+
+          elif [[ -n "$TAG" ]]; then
+            # Find the build run that produced this tag
+            echo "Looking for build run for tag: $TAG"
+            BUILD_RUN_ID=$(gh api \
+              "repos/${{ github.repository }}/actions/workflows/build.yml/runs?branch=${TAG}&status=success&per_page=1" \
+              --jq '.workflow_runs[0].id' 2>/dev/null || echo "")
+
+            if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
+              # Tag might not be a branch; try event=push with head_branch matching
+              BUILD_RUN_ID=$(gh api \
+                "repos/${{ github.repository }}/actions/workflows/build.yml/runs?event=push&status=success&per_page=100" \
+                --jq ".workflow_runs[] | select(.head_branch == \"$TAG\") | .id" 2>/dev/null | head -1 || echo "")
+            fi
+
+            if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
+              echo "❌ No successful build run found for tag: $TAG"
+              exit 1
             fi
-            [[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ ]] || fail "no successful build run found for tag"
             echo "Found build run: $BUILD_RUN_ID"
+
           else
+            # No tag — latest successful main build
             echo "No tag specified, looking for latest main build"
-            BUILD_RUN_ID=$(gh api --method GET \
-              "repos/${REPOSITORY}/actions/workflows/build.yml/runs" \
-              -f branch=main -f status=success -F per_page=1 \
-              --jq '.workflow_runs[0].id // empty' 2>/dev/null || true)
-            [[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ ]] || fail "no successful main build found"
+            BUILD_RUN_ID=$(gh api \
+              "repos/${{ github.repository }}/actions/workflows/build.yml/runs?branch=main&status=success&per_page=1" \
+              --jq '.workflow_runs[0].id' 2>/dev/null || echo "")
+
+            if [[ -z "$BUILD_RUN_ID" || "$BUILD_RUN_ID" == "null" ]]; then
+              echo "❌ No successful main build found"
+              exit 1
+            fi
             echo "Latest main build: $BUILD_RUN_ID"
           fi
 
-          # Fetch once and use the same immutable run metadata for identity,
-          # ordering, workflow provenance, and release-channel validation.
-          RUN_JSON=$(gh api "repos/${REPOSITORY}/actions/runs/${BUILD_RUN_ID}") ||
-            fail "cannot read selected build run"
-          RUN_ID=$(jq -r '.id // empty' <<<"$RUN_JSON")
-          RUN_NUMBER=$(jq -r '.run_number // empty' <<<"$RUN_JSON")
-          RUN_STATUS=$(jq -r '.status // empty' <<<"$RUN_JSON")
-          RUN_CONCLUSION=$(jq -r '.conclusion // empty' <<<"$RUN_JSON")
-          RUN_PATH=$(jq -r '.path // empty' <<<"$RUN_JSON")
-          HEAD_SHA=$(jq -r '.head_sha // empty' <<<"$RUN_JSON")
-          RUN_HEAD_BRANCH=$(jq -r '.head_branch // empty' <<<"$RUN_JSON")
-
-          [[ "$RUN_ID" == "$BUILD_RUN_ID" ]] || fail "run metadata ID mismatch"
-          [[ "$RUN_NUMBER" =~ ^[1-9][0-9]*$ ]] || fail "build run number must be a positive decimal integer"
-          [[ "$RUN_STATUS" == completed && "$RUN_CONCLUSION" == success ]] || fail "selected build run is not successful"
-          [[ "$RUN_PATH" == .github/workflows/build.yml ]] || fail "selected run is not Build and Release"
-          [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] || fail "selected build run has an invalid head SHA"
-          [[ "$RUN_HEAD_BRANCH" != *$'\n'* && -n "$RUN_HEAD_BRANCH" ]] || fail "selected build run has an invalid head branch"
-
+          # Determine version and build type
           if [[ -n "$TAG" ]]; then
-            [[ "$RUN_HEAD_BRANCH" == "$TAG" ]] || fail "tag and build run head branch do not match"
-
-            TAG_REF_JSON=$(gh api "repos/${REPOSITORY}/git/ref/tags/${TAG}") ||
-              fail "cannot resolve release tag ref"
-            TAG_OBJECT_TYPE=$(jq -r '.object.type // empty' <<<"$TAG_REF_JSON")
-            TAG_OBJECT_SHA=$(jq -r '.object.sha // empty' <<<"$TAG_REF_JSON")
-            depth=0
-            while [[ "$TAG_OBJECT_TYPE" == tag && $depth -lt 5 ]]; do
-              TAG_OBJECT_JSON=$(gh api "repos/${REPOSITORY}/git/tags/${TAG_OBJECT_SHA}") ||
-                fail "cannot peel annotated release tag"
-              TAG_OBJECT_TYPE=$(jq -r '.object.type // empty' <<<"$TAG_OBJECT_JSON")
-              TAG_OBJECT_SHA=$(jq -r '.object.sha // empty' <<<"$TAG_OBJECT_JSON")
-              depth=$((depth + 1))
-            done
-            [[ "$TAG_OBJECT_TYPE" == commit && "$TAG_OBJECT_SHA" =~ ^[0-9a-f]{40}$ ]] ||
-              fail "release tag does not resolve to a commit"
-            [[ "$TAG_OBJECT_SHA" == "$HEAD_SHA" ]] || fail "release tag commit and build run head SHA do not match"
             VERSION="$TAG"
-            DEV_SEQUENCE=""
+            if [[ "$TAG" == *"-preview"* ]]; then
+              BUILD_TYPE="preview"
+            elif [[ "$TAG" == *"alpha"* || "$TAG" == *"beta"* || "$TAG" == *"rc"* ]]; then
+              BUILD_TYPE="prerelease"
+            else
+              BUILD_TYPE="release"
+            fi
           else
-            VERSION="dev-${HEAD_SHA}"
-            DEV_SEQUENCE="$RUN_NUMBER"
+            SHORT_SHA=$(gh api "repos/${{ github.repository }}/actions/runs/${BUILD_RUN_ID}" \
+              --jq '.head_sha' 2>/dev/null | head -c 7)
+            VERSION="dev-${SHORT_SHA}"
+            BUILD_TYPE="development"
           fi
 
           {
             echo "version=$VERSION"
             echo "build_type=$BUILD_TYPE"
             echo "build_run_id=$BUILD_RUN_ID"
-            echo "build_run_number=$RUN_NUMBER"
-            echo "head_sha=$HEAD_SHA"
-            echo "dev_sequence=$DEV_SEQUENCE"
             echo "tag=${TAG}"
           } >> "$GITHUB_OUTPUT"
 
@@ -222,7 +180,6 @@ jobs:
           echo "  Version: $VERSION"
           echo "  Build type: $BUILD_TYPE"
           echo "  Build run ID: $BUILD_RUN_ID"
-          echo "  Build run number: $RUN_NUMBER"
 
   # Build DEB and RPM packages for each architecture
   package:
@@ -249,22 +206,6 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Normalize package metadata
-        id: versions
-        shell: bash
-        env:
-          BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
-          SOURCE_VERSION: ${{ needs.resolve.outputs.version }}
-          DEV_SEQUENCE: ${{ needs.resolve.outputs.dev_sequence }}
-          DEB_ARCH: ${{ matrix.deb_arch }}
-          RPM_ARCH: ${{ matrix.rpm_arch }}
-        run: |
-          set -euo pipefail
-
-          normalized=$(./scripts/release/package_versions.sh \
-            "$BUILD_TYPE" "$SOURCE_VERSION" "$DEV_SEQUENCE" "$DEB_ARCH" "$RPM_ARCH")
-          printf '%s\n' "$normalized" >> "$GITHUB_OUTPUT"
-
       - name: Download binary artifact from build run
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
         with:
@@ -304,16 +245,18 @@ jobs:
       - name: Build DEB package
         id: deb
         shell: bash
-        env:
-          DEB_VERSION: ${{ steps.versions.outputs.deb_version }}
-          DEB_ARCH: ${{ matrix.deb_arch }}
-          DEB_FILE: ${{ steps.versions.outputs.deb_file }}
         run: |
           set -euo pipefail
 
-          PKG_DIR="${DEB_FILE%.deb}"
+          VERSION="${{ needs.resolve.outputs.version }}"
+          DEB_ARCH="${{ matrix.deb_arch }}"
+          # DEB version: replace - with ~ (1.0.0-beta.12 -> 1.0.0~beta.12)
+          # Use a variable for ~ to prevent tilde expansion by bash
+          TILDE='~'
+          DEB_VERSION="${VERSION/-/$TILDE}"
+          PKG_DIR="rustfs_${DEB_VERSION}_${DEB_ARCH}"
 
-          echo "Building DEB: ${DEB_FILE}"
+          echo "Building DEB: ${PKG_DIR}.deb"
 
           mkdir -p "${PKG_DIR}/DEBIAN"
           mkdir -p "${PKG_DIR}/usr/bin"
@@ -390,12 +333,9 @@ jobs:
           cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
           cp README.md "${PKG_DIR}/usr/share/doc/rustfs/"
 
-          fakeroot dpkg-deb --build "${PKG_DIR}" "$DEB_FILE"
+          fakeroot dpkg-deb --build "${PKG_DIR}"
 
-          [[ $(dpkg-deb -f "$DEB_FILE" Package) == rustfs ]]
-          [[ $(dpkg-deb -f "$DEB_FILE" Version) == "$DEB_VERSION" ]]
-          [[ $(dpkg-deb -f "$DEB_FILE" Architecture) == "$DEB_ARCH" ]]
-          dpkg-deb --fsys-tarfile "$DEB_FILE" | tar -tf - | grep -Fx './usr/bin/rustfs' >/dev/null
+          DEB_FILE="${PKG_DIR}.deb"
           stat --printf='%n %s bytes\n' "$DEB_FILE"
           echo "deb_file=$DEB_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ DEB built: $DEB_FILE"
@@ -403,19 +343,16 @@ jobs:
       - name: Build RPM package
         id: rpm
         shell: bash
-        env:
-          RPM_VERSION: ${{ steps.versions.outputs.rpm_version }}
-          RPM_RELEASE: ${{ steps.versions.outputs.rpm_release }}
-          RPM_ARCH: ${{ matrix.rpm_arch }}
-          RPM_FILE: ${{ steps.versions.outputs.rpm_file }}
         run: |
           set -euo pipefail
 
+          VERSION="${{ needs.resolve.outputs.version }}"
+          RPM_ARCH="${{ matrix.rpm_arch }}"
+
           echo "Building RPM for ${RPM_ARCH}"
 
-          sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential rpm
+          sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential
           sudo gem install fpm
-          ./scripts/test_package_versions.sh --require-package-managers
 
           # Create config file for fpm (DEB build creates it in its package dir structure,
           # but fpm needs the file to exist before packaging)
@@ -430,10 +367,8 @@ jobs:
 
           fpm -s dir -t rpm \
             --name rustfs \
-            --version "$RPM_VERSION" \
-            --iteration "$RPM_RELEASE" \
+            --version "$VERSION" \
             --architecture "$RPM_ARCH" \
-            --package "$RPM_FILE" \
             --depends "glibc >= 2.31" \
             --maintainer "RustFS Team <[email protected]>" \
             --description "High-performance distributed object storage" \
@@ -475,15 +410,13 @@ jobs:
             LICENSE=/usr/share/doc/rustfs/LICENSE \
             README.md=/usr/share/doc/rustfs/README.md
 
-          if [[ ! -f "$RPM_FILE" ]]; then
+            RPM_FILE=$(find . -maxdepth 1 -type f -name 'rustfs-*.rpm' -print | head -1)
+            RPM_FILE="${RPM_FILE#./}"
+          if [[ -z "$RPM_FILE" ]]; then
             echo "❌ RPM build failed"
             exit 1
           fi
 
-          RPM_METADATA=$(rpm -qp --qf '%{NAME}\n%{VERSION}\n%{RELEASE}\n%{ARCH}\n' "$RPM_FILE")
-          EXPECTED_METADATA=$(printf 'rustfs\n%s\n%s\n%s' "$RPM_VERSION" "$RPM_RELEASE" "$RPM_ARCH")
-          [[ "$RPM_METADATA" == "$EXPECTED_METADATA" ]]
-          rpm -qpl "$RPM_FILE" | grep -Fx '/usr/bin/rustfs' >/dev/null
           stat --printf='%n %s bytes\n' "$RPM_FILE"
           echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
           echo "✅ RPM built: $RPM_FILE"
@@ -505,9 +438,6 @@ jobs:
           R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
           R2_BUCKET: ${{ secrets.R2_BUCKET }}
           AWS_EC2_METADATA_DISABLED: true
-          BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
-          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
-          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
         shell: bash
         run: |
           set -euo pipefail
@@ -525,6 +455,7 @@ jobs:
           export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
           export AWS_DEFAULT_REGION="auto"
 
+          BUILD_TYPE="${{ needs.resolve.outputs.build_type }}"
           if [[ "$BUILD_TYPE" == "development" ]]; then
             R2_PREFIX="artifacts/rustfs/packages/dev"
           else
@@ -534,6 +465,9 @@ jobs:
 
           echo "📤 Uploading to $R2_PATH"
 
+          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
+          RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
+
           for f in "$DEB_FILE" "$RPM_FILE"; do
             if [[ -n "$f" && -f "$f" ]]; then
               echo "Uploading: $f"
@@ -559,13 +493,14 @@ jobs:
         if: needs.resolve.outputs.tag != ''
         env:
           GH_TOKEN: ${{ github.token }}
-          TAG: ${{ needs.resolve.outputs.tag }}
-          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
-          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
         shell: bash
         run: |
           set -euo pipefail
 
+          TAG="${{ needs.resolve.outputs.tag }}"
+          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
+          RPM_FILE="${{ steps.rpm.outputs.rpm_file }}"
+
           # Upload the packages, then refresh the release checksums so the new
           # assets are covered, matching the binary release flow.
           for f in "$DEB_FILE" "$RPM_FILE"; do
@@ -617,19 +552,14 @@ jobs:
     steps:
       - name: Print summary
         shell: bash
-        env:
-          SUMMARY_VERSION: ${{ needs.resolve.outputs.version }}
-          SUMMARY_BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
-          SUMMARY_BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
-          SUMMARY_PACKAGE_STATUS: ${{ needs.package.result }}
         run: |
           {
             echo "## 📦 Package Summary"
             echo ""
             echo "| Item | Value |"
             echo "|------|-------|"
-            echo "| Version | \`${SUMMARY_VERSION}\` |"
-            echo "| Build Type | ${SUMMARY_BUILD_TYPE} |"
-            echo "| Build Run | #${SUMMARY_BUILD_RUN_ID} |"
-            echo "| Package Status | ${SUMMARY_PACKAGE_STATUS} |"
+            echo "| Version | \`${{ needs.resolve.outputs.version }}\` |"
+            echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |"
+            echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |"
+            echo "| Package Status | ${{ needs.package.result }} |"
           } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/rustfs-functional-chain.yml

@@ -1,74 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Functional chain driver: runs the ten functional suites in a fixed order
-# (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
-# replication, with performance on its own runner in parallel) and guarantees
-# the chain keeps moving even when individual suites fail.
-#
-# Each suite workflow can still be dispatched standalone (workflow_dispatch);
-# only chain-triggered runs forward to the next suite via repository_dispatch,
-# so a standalone run never drags the rest of the chain behind it.
-#
-# Why not workflow_run chaining: GitHub does not guarantee delivery of
-# workflow_run events (they are fire-and-forget), and the head-SHA filter made
-# newly added suites (storage) unable to trigger at all. Explicit
-# repository_dispatch handoffs are verifiable and re-drivable.
-
-name: RustFS Functional Chain
-
-on:
-  workflow_dispatch:
-  workflow_run:
-    # Entry point: start the chain after the nightly build completes. The
-    # build's own conclusion does not gate the chain; each suite reports its
-    # own result to rustfs/backlog and the dashboard.
-    workflows: ["Nightly GNU Build"]
-    types: [completed]
-
-permissions:
-  contents: read
-
-jobs:
-  start-chain:
-    name: Start functional chain (upgrade first)
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'schedule') }}
-    steps:
-      - name: Dispatch first suite (upgrade)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot start the functional chain" >&2
-            exit 1
-          fi
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-upgrade' \
-            -F 'client_payload[from_suite]=nightly-build'
-
-      - name: Dispatch performance suite (parallel, own runner)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
-            exit 1
-          fi
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-performance' \
-            -F 'client_payload[from_suite]=nightly-build'

.github/workflows/rustfs-heal-test.yml

@@ -23,11 +23,6 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  repository_dispatch:
-    # Chain handoff: dispatched when the storage suite finishes. Heal runs
-    # exactly once per chain; the pool expansion workflow no longer embeds
-    # its own heal pass.
-    types: [rustfs-chain-heal]
 
 permissions:
   contents: read
@@ -48,39 +43,24 @@ env:
   RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
   RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
   RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
-  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
 
 jobs:
   heal-test:
     runs-on: smoke-testing
-    # Requirement: a failing suite must not fail the workflow; failures
-    # are filed to rustfs/backlog and the chain continues.
-    continue-on-error: true
     timeout-minutes: 480
-    # Standalone manual run, or one link of the nightly functional chain
-    # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    # Manual-only standalone run. Nightly chain already runs heal in
+    # rustfs-pool-expand-test.yml to avoid duplicate heal executions.
+    if: ${{ github.event_name == 'workflow_dispatch' }}
     steps:
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
       - name: Show environment
         run: |
@@ -90,24 +70,11 @@ jobs:
           warp --version || true
           df -h /data | tail -1
 
-      - name: Cleanup environment (before)
+      - name: Reset test environment (before)
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
-            '
-          done
+          chmod +x auto-testing/rustfs_heal_test.sh
+          ./auto-testing/rustfs_heal_test.sh --reset -y
 
       - name: Install RustFS package & start cluster
         run: |
@@ -130,129 +97,14 @@ jobs:
           ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
 
       - name: Run heal test (write -> outage -> heal -> verify)
-        id: test
         run: |
           ./auto-testing/rustfs_heal_test.sh \
             --steps "3,4,5,6,7" -y \
             --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
-            --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
-            --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
+            --stop-node-gb "${{ inputs.stop_node_gb }}" \
+            --warp-stop-gb "${{ inputs.warp_stop_gb }}" \
             --log-file /tmp/rustfs-heal-test.log
 
-      - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-heal-test.log
-          REPORT_FILE: /tmp/rustfs-heal-report.md
-        run: |
-          set -euo pipefail
-          PACKAGE_URL='${{ inputs.package_url }}'
-          if [ -n "${PACKAGE_URL}" ]; then
-            PACKAGE_SOURCE="${PACKAGE_URL}"
-          else
-            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
-          fi
-          {
-            echo "# RustFS heal test report"
-            echo ""
-            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Trigger: ${{ github.event_name }}"
-            echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
-          } | tee "${REPORT_FILE}"
-          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-
-      - name: Upload functional report to dashboard
-        if: always()
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-heal-report.md
-          SUITE: heal
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
-            exit 0
-          fi
-          DATE="$(date -u +%Y-%m-%d)"
-          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
-          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          fi
-
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          SUITE: 'heal'
-          SUITE_LABEL: 'Heal'
-          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-heal-report.md'
-          LOG_FILE: '/tmp/rustfs-heal-test.log'
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
-            exit 0
-          fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
-            exit 0
-          fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
-          fi
-          echo "filed backlog issue for suite ${SUITE}"
-
       - name: Upload test logs
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
@@ -263,73 +115,10 @@ jobs:
             /tmp/rustfs-warp.*.log
           if-no-files-found: warn
 
-      - name: Cleanup environment (after)
+      - name: Reset test environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}
         run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
-            '
-          done
-
-      - name: "Continue functional chain (next: Pool expansion)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-pool' \
-              -F 'client_payload[from_suite]=heal'; then
-              echo "dispatched next suite Pool expansion (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Pool expansion after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after heal (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **heal** to **Pool expansion** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-pool'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-pool'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          ./auto-testing/rustfs_heal_test.sh --reset -y
 
       - name: Notify on failure
         if: failure()

.github/workflows/rustfs-kms-test.yml

@@ -11,21 +11,10 @@ on:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
         type: string
-      enforce_sse_key_policy:
-        description: 'Enable RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY (runs KMS-401/402)'
-        type: boolean
-        default: false
-      frame_v2:
-        description: 'Enable RUSTFS_ENCRYPTION_FRAME_V2 (runs KMS-318)'
-        type: boolean
-        default: false
-      config_secret:
-        description: 'Set RUSTFS_KMS_CONFIG_SECRET (runs KMS-107 config sealing)'
-        required: false
-        type: string
-  repository_dispatch:
-    # Chain handoff: dispatched when the S3 compatibility suite finishes.
-    types: [rustfs-chain-kms]
+  workflow_run:
+    # Strict shared-environment order: run after S3 compatibility test succeeds.
+    workflows: ["RustFS S3 Compatibility Test"]
+    types: [completed]
 
 permissions:
   contents: read
@@ -49,29 +38,17 @@ env:
 jobs:
   kms-test:
     runs-on: smoke-testing
-    continue-on-error: true
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     steps:
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
       - name: Show environment
         run: |
@@ -109,7 +86,6 @@ jobs:
 
       - name: Run KMS suite
         id: test
-        continue-on-error: true
         env:
           LOG_FILE: /tmp/rustfs-kms.log
         run: |
@@ -118,19 +94,6 @@ jobs:
           PACKAGE_URL='${{ inputs.package_url }}'
           RUSTFS_VERSION='${{ inputs.rustfs_version }}'
           ARGS=(--all-topologies --backends "local,vault-kv2" -y --log-file "${LOG_FILE}")
-          EXTRA_ENV=""
-          if [ "${{ inputs.enforce_sse_key_policy }}" = "true" ]; then
-            EXTRA_ENV+="RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY=true"$'\n'
-          fi
-          if [ "${{ inputs.frame_v2 }}" = "true" ]; then
-            EXTRA_ENV+="RUSTFS_ENCRYPTION_FRAME_V2=true"$'\n'
-          fi
-          if [ -n "${{ inputs.config_secret }}" ]; then
-            EXTRA_ENV+="RUSTFS_KMS_CONFIG_SECRET=${{ inputs.config_secret }}"$'\n'
-          fi
-          if [ -n "${EXTRA_ENV}" ]; then
-            ARGS+=(--extra-env "${EXTRA_ENV}")
-          fi
           if [ -n "${PACKAGE_URL}" ]; then
             ARGS+=(--package-url "${PACKAGE_URL}")
           elif [ -n "${RUSTFS_VERSION}" ]; then
@@ -156,65 +119,12 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          CASE_TABLE="/tmp/rustfs-kms-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
           {
             echo "# RustFS KMS test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
-            echo ""
-            cat "${CASE_TABLE}" || true
             echo ""
             echo "## Log tail"
             echo '```text'
@@ -223,92 +133,6 @@ jobs:
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
 
-      - name: Upload functional report to dashboard
-        if: always()
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-kms-report.md
-          SUITE: kms
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
-            exit 0
-          fi
-          DATE="$(date -u +%Y-%m-%d)"
-          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
-          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          fi
-
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          SUITE: 'kms'
-          SUITE_LABEL: 'KMS'
-          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-kms-report.md'
-          LOG_FILE: '/tmp/rustfs-kms.log'
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
-            exit 0
-          fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
-            exit 0
-          fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
-          fi
-          echo "filed backlog issue for suite ${SUITE}"
-
       - name: Upload report and logs
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
@@ -338,55 +162,6 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Tier)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-tier' \
-              -F 'client_payload[from_suite]=kms'; then
-              echo "dispatched next suite Tier (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Tier after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after kms (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **kms** to **Tier** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-tier'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-tier'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
-
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-performance-test.yml

@@ -48,10 +48,10 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  repository_dispatch:
-    # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
-    # pf-testing runner, in parallel with the shared-VM chain).
-    types: [rustfs-chain-performance]
+  workflow_run:
+    # Run after the nightly build completes; the nightly deb is what the test installs.
+    workflows: ["Nightly GNU Build"]
+    types: [completed]
 
 permissions:
   contents: read
@@ -84,33 +84,19 @@ env:
 jobs:
   performance-test:
     runs-on: pf-testing
-    # Requirement: a failing benchmark must not fail the workflow;
-    # failures are filed to rustfs/backlog.
-    continue-on-error: true
     timeout-minutes: 900
     # Run on manual dispatch, or when the nightly build completed successfully.
     # Skipped when nightly failed.
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     steps:
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
       - name: Show environment
         run: |
@@ -226,65 +212,6 @@ jobs:
             echo "created ${REPORT_PATH} in rustfs/dashboard"
           fi
 
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.benchmark.outcome == 'failure' || steps.benchmark.outcome == 'cancelled') }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          SUITE: 'performance'
-          SUITE_LABEL: 'Performance'
-          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-perf-report.md'
-          LOG_FILE: '/tmp/rustfs-perf-test.log'
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
-            exit 0
-          fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
-            exit 0
-          fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
-          fi
-          echo "filed backlog issue for suite ${SUITE}"
-
       - name: Upload test logs & results
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6

.github/workflows/rustfs-pool-expand-test.yml

@@ -1,4 +1,4 @@
-name: RustFS Pool Expansion Test
+name: RustFS Pool Expansion / Heal Test
 
 on:
   workflow_dispatch:
@@ -25,14 +25,18 @@ on:
         description: 'warp write duration (e.g. 5m, 10m)'
         required: false
         default: '10m'
-      warp_concurrent:
-        description: 'Pool fill: concurrent warp operations'
-        required: false
-        default: '32'
       run_decommission:
         description: 'Run the pool decommission step (3-pool topology only)'
         type: boolean
         default: true
+      stop_node_gb:
+        description: 'Heal: stop the outage node when surviving nodes reach N GiB'
+        required: false
+        default: '15'
+      warp_stop_gb:
+        description: 'Heal: stop warp when surviving nodes reach N GiB'
+        required: false
+        default: '40'
       cleanup_before:
         description: 'Reset the nodes before the test (DESTROYS existing data/config)'
         type: boolean
@@ -41,16 +45,17 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  repository_dispatch:
-    # Chain handoff: dispatched when the heal suite finishes.
-    types: [rustfs-chain-pool]
+  workflow_run:
+    # Strict shared-environment order: run after tier test succeeds.
+    workflows: ["RustFS Tier Test"]
+    types: [completed]
 
 permissions:
   contents: read
 
-# Only one test run at a time: the job mutates the same shared test
+# Only one test run at a time: every job mutates the same shared test
 # environment (vm000/vm001/vm002), so concurrent runs must not clobber each
-# other.
+# other. Jobs inside a run are chained with needs to serialize them.
 concurrency:
   group: rustfs-shared-functional-tests
   cancel-in-progress: false
@@ -65,55 +70,25 @@ env:
   RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
   RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
   RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
-  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
   # Package used by the nightly run (workflow_dispatch inputs are empty for
   # workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
 
 jobs:
-  # Pool expansion: dispatched by the heal suite's chain handoff. Heal
-  # itself lives in rustfs-heal-test.yml and runs exactly once per chain.
   pool-expansion-test:
     name: Pool expansion / decommission test
     runs-on: smoke-testing
-    # Requirement: a failing suite must not fail the workflow; failures
-    # are filed to rustfs/backlog and the chain continues.
-    continue-on-error: true
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
-    env:
-      RUSTFS_POOL_ADMIN_ENDPOINT: ${{ secrets.RUSTFS_POOL_ADMIN_ENDPOINT || vars.RUSTFS_POOL_ADMIN_ENDPOINT || 'http://rustfs-node1:9000' }}
-      RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
-      RUSTFS_POOL_WARP_ENDPOINT: http://127.0.0.1:19000
-      RUSTFS_SHARED_PROXY_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
-      RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     steps:
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
-
-      - name: Initialize pool test artifacts
-        run: |
-          set -euo pipefail
-          ARTIFACT_DIR="${RUNNER_TEMP}/rustfs-pool-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -p "${ARTIFACT_DIR}"
-          echo "POOL_ARTIFACT_DIR=${ARTIFACT_DIR}" >> "${GITHUB_ENV}"
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
       - name: Show environment
         run: |
@@ -123,32 +98,15 @@ jobs:
           warp --version || true
           df -h /data | tail -1
 
-      - name: Cleanup environment (before)
+      - name: Reset test environment (before)
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
-            '
-          done
+          chmod +x auto-testing/rustfs_pool_expand.sh
+          ./auto-testing/rustfs_pool_expand.sh --reset -y
 
-      - name: Install RustFS package & start cluster
+      - name: Install RustFS package & start first pool
         run: |
-          ARGS=(--steps "1,2,3" -y \
-            --admin-endpoint "${RUSTFS_POOL_ADMIN_ENDPOINT}" \
-            --warp-endpoint "${RUSTFS_POOL_WARP_ENDPOINT}" \
-            --node-endpoints "${RUSTFS_POOL_NODE_ENDPOINTS}" \
-            --log-file "${POOL_ARTIFACT_DIR}/pool-test.log")
+          ARGS=(--steps "1,2,3" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
           if [ -n "${{ inputs.package_url }}" ]; then
             ARGS+=(--package-url "${{ inputs.package_url }}")
           elif [ -n "${{ inputs.rustfs_version }}" ]; then
@@ -160,11 +118,7 @@ jobs:
 
       - name: Preflight checks
         run: |
-          ARGS=(--preflight \
-            --admin-endpoint "${RUSTFS_POOL_ADMIN_ENDPOINT}" \
-            --warp-endpoint "${RUSTFS_POOL_WARP_ENDPOINT}" \
-            --node-endpoints "${RUSTFS_POOL_NODE_ENDPOINTS}" \
-            --log-file "${POOL_ARTIFACT_DIR}/pool-test.log")
+          ARGS=(--preflight --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
           if [ -n "${{ inputs.package_url }}" ]; then
             ARGS+=(--package-url "${{ inputs.package_url }}")
           elif [ -n "${{ inputs.rustfs_version }}" ]; then
@@ -174,60 +128,6 @@ jobs:
           fi
           ./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
 
-      - name: Reset dedicated pool proxy
-        run: |
-          set -euo pipefail
-          RUSTFS_POOL_NGINX_CONFIG_PATH=/etc/nginx/conf.d/rustfs-pool-test.conf \
-          RUSTFS_POOL_NGINX_LISTEN="${RUSTFS_POOL_PROXY_ENDPOINT#http://}" \
-          RUSTFS_POOL_NGINX_ACCESS_LOG=/var/log/nginx/rustfs-pool-test-access.log \
-          RUSTFS_POOL_NGINX_ERROR_LOG=/var/log/nginx/rustfs-pool-test-error.log \
-            ./auto-testing/rustfs_pool_nginx_stage.sh cleanup
-
-      - name: Capture pool test baseline
-        run: |
-          set -uo pipefail
-          BASELINE_FILE="${POOL_ARTIFACT_DIR}/pool-baseline.log"
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          read -r -a DIRECT_ENDPOINTS <<< "${RUSTFS_POOL_NODE_ENDPOINTS}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          failed=0
-          : > "${BASELINE_FILE}"
-
-          if [ "${#DIRECT_ENDPOINTS[@]}" -lt "${#NODES[@]}" ]; then
-            echo "not enough direct endpoints for the configured nodes" | tee -a "${BASELINE_FILE}" >&2
-            exit 1
-          fi
-
-          for index in "${!NODES[@]}"; do
-            node="${NODES[$index]}"
-            endpoint="${DIRECT_ENDPOINTS[$index]}"
-            body_file="${POOL_ARTIFACT_DIR}/ready-baseline-$((index + 1)).body"
-            {
-              echo "--- node=${node} endpoint=${endpoint} ---"
-              if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
-                "${SSH_USER}@${node}" '
-                  set -euo pipefail
-                  SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-                  echo "--- rustfs version ---"
-                  rustfs --version
-                  echo "--- systemd state ---"
-                  ${SUDO} systemctl show rustfs --no-pager \
-                    --property=ActiveState,SubState,Result,ExecMainPID,ExecMainStartTimestamp,NRestarts
-                '; then
-                echo "baseline collection failed for ${node}"
-                failed=1
-              fi
-              curl -sS --connect-timeout 5 --max-time 15 -o "${body_file}" \
-                -w "baseline_ready=${endpoint} http=%{http_code} connect=%{time_connect} ttfb=%{time_starttransfer} total=%{time_total}\n" \
-                "${endpoint%/}/health/ready" || true
-              echo "--- readiness body ---"
-              cat "${body_file}" 2>/dev/null || true
-              echo
-            } >> "${BASELINE_FILE}" 2>&1
-          done
-
-          [ "${failed}" -eq 0 ] || exit 1
-
       - name: Run pool expansion & decommission test
         id: pool_test
         run: |
@@ -240,16 +140,10 @@ jobs:
             fi
           fi
           ARGS=(--steps "$STEPS" --with-warp -y \
-            --admin-endpoint "${RUSTFS_POOL_ADMIN_ENDPOINT}" \
-            --warp-endpoint "${RUSTFS_POOL_WARP_ENDPOINT}" \
-            --node-endpoints "${RUSTFS_POOL_NODE_ENDPOINTS}" \
+            --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
             --storage-threshold "${{ inputs.storage_threshold || '50' }}" \
             --warp-duration "${{ inputs.warp_duration || '10m' }}" \
-            --warp-concurrent "${{ inputs.warp_concurrent || '32' }}" \
-            --log-file "${POOL_ARTIFACT_DIR}/pool-test.log")
-          if [ -n "${RUSTFS_POOL_PROXY_ENDPOINT}" ]; then
-            ARGS+=(--proxy-endpoint "${RUSTFS_POOL_PROXY_ENDPOINT}")
-          fi
+            --log-file /tmp/rustfs-pool-test.log)
           if [ -n "${{ inputs.package_url }}" ]; then
             ARGS+=(--package-url "${{ inputs.package_url }}")
           elif [ -n "${{ inputs.rustfs_version }}" ]; then
@@ -257,391 +151,22 @@ jobs:
           else
             ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
           fi
-          RUSTFS_WARP_LOG_FILE="${POOL_ARTIFACT_DIR}/warp.log" \
-          RUSTFS_PROXY_STAGE_HOOK=./auto-testing/rustfs_pool_nginx_stage.sh \
-          RUSTFS_POOL_NGINX_CONFIG_PATH=/etc/nginx/conf.d/rustfs-pool-test.conf \
-          RUSTFS_POOL_NGINX_LISTEN="${RUSTFS_POOL_PROXY_ENDPOINT#http://}" \
-          RUSTFS_POOL_NGINX_ACCESS_LOG=/var/log/nginx/rustfs-pool-test-access.log \
-          RUSTFS_POOL_NGINX_ERROR_LOG=/var/log/nginx/rustfs-pool-test-error.log \
-            ./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
-
-      - name: Collect pool test diagnostics
-        if: always()
-        run: |
-          set -uo pipefail
-          ARTIFACT_DIR="${POOL_ARTIFACT_DIR:-${RUNNER_TEMP}/rustfs-pool-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}}"
-          mkdir -p "${ARTIFACT_DIR}"
-          echo "POOL_ARTIFACT_DIR=${ARTIFACT_DIR}" >> "${GITHUB_ENV}"
-
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)=).*/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(proxy_set_header[[:space:]]+Authorization[[:space:]]+).*/\1[REDACTED];/Ig' \
-              -e 's/^.*(password|secret|token).*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-
-          if [ "$(id -u)" -eq 0 ]; then
-            SUDO=()
-          else
-            SUDO=(sudo -n)
-          fi
-
-          {
-            echo "captured_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
-            echo "run_id=${GITHUB_RUN_ID}"
-            echo "run_attempt=${GITHUB_RUN_ATTEMPT}"
-            if command -v nginx >/dev/null 2>&1; then
-              "${SUDO[@]}" nginx -T 2>&1 || echo "nginx -T failed"
-            else
-              echo "nginx is not installed on the runner"
-            fi
-          } | redact > "${ARTIFACT_DIR}/nginx-config-redacted.txt"
-
-          for log_path in \
-            /var/log/nginx/access.log \
-            /var/log/nginx/error.log \
-            /var/log/nginx/rustfs-pool-test-access.log \
-            /var/log/nginx/rustfs-pool-test-error.log; do
-            log_name="$(basename "${log_path}")"
-            if "${SUDO[@]}" test -r "${log_path}" 2>/dev/null; then
-              "${SUDO[@]}" cat "${log_path}" 2>&1 | redact \
-                > "${ARTIFACT_DIR}/nginx-${log_name%.log}-redacted.log"
-            else
-              echo "unavailable: ${log_path}" > "${ARTIFACT_DIR}/nginx-${log_name%.log}-redacted.log"
-            fi
-          done
-          "${SUDO[@]}" journalctl -u nginx --no-pager -n 5000 2>&1 | redact \
-            > "${ARTIFACT_DIR}/nginx-journal-redacted.log" || true
-
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            safe_node="${node//[^A-Za-z0-9_.-]/_}"
-            {
-              if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
-                "${SSH_USER}@${node}" '
-                  SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-                  echo "--- rustfs version ---"
-                  rustfs --version 2>&1 || true
-                  echo "--- systemd state ---"
-                  ${SUDO} systemctl show rustfs --no-pager \
-                    --property=ActiveState,SubState,Result,ExecMainPID,ExecMainStartTimestamp,NRestarts 2>&1 || true
-                  echo "--- rustfs journal ---"
-                  ${SUDO} journalctl -u rustfs --no-pager -n 10000 2>&1 || true
-                  echo "--- rustfs file logs ---"
-                  if ${SUDO} test -d /var/log/rustfs; then
-                    ${SUDO} find /var/log/rustfs -maxdepth 2 -type f -print 2>/dev/null | while IFS= read -r file; do
-                      echo "--- ${file} (last 5000 lines) ---"
-                      ${SUDO} tail -n 5000 "${file}" 2>&1 || true
-                    done
-                  else
-                    echo "/var/log/rustfs is unavailable"
-                  fi
-                '; then
-                echo "SSH diagnostics failed for ${node}"
-              fi
-            } 2>&1 | redact > "${ARTIFACT_DIR}/${safe_node}-rustfs-redacted.log"
-          done
-
-          : > "${ARTIFACT_DIR}/endpoint-ready-probes.log"
-          read -r -a DIRECT_ENDPOINTS <<< "${RUSTFS_POOL_NODE_ENDPOINTS}"
-          probe_index=0
-          for endpoint in "${DIRECT_ENDPOINTS[@]}"; do
-            probe_index=$((probe_index + 1))
-            curl -sS --connect-timeout 5 --max-time 15 -o "${ARTIFACT_DIR}/ready-direct-${probe_index}.body" \
-              -w "direct[${probe_index}]=${endpoint} http=%{http_code} connect=%{time_connect} ttfb=%{time_starttransfer} total=%{time_total}\n" \
-              "${endpoint%/}/health/ready" >> "${ARTIFACT_DIR}/endpoint-ready-probes.log" 2>&1 || true
-          done
-          if [ -n "${RUSTFS_POOL_PROXY_ENDPOINT}" ]; then
-            curl -sS --connect-timeout 5 --max-time 15 -o "${ARTIFACT_DIR}/ready-proxy.body" \
-              -w "proxy=${RUSTFS_POOL_PROXY_ENDPOINT} http=%{http_code} connect=%{time_connect} ttfb=%{time_starttransfer} total=%{time_total}\n" \
-              "${RUSTFS_POOL_PROXY_ENDPOINT%/}/health/ready" >> "${ARTIFACT_DIR}/endpoint-ready-probes.log" 2>&1 || true
-          fi
-          if [ -n "${RUSTFS_SHARED_PROXY_ENDPOINT}" ]; then
-            curl -sS --connect-timeout 5 --max-time 15 -o "${ARTIFACT_DIR}/ready-shared-proxy.body" \
-              -w "shared_proxy=${RUSTFS_SHARED_PROXY_ENDPOINT} http=%{http_code} connect=%{time_connect} ttfb=%{time_starttransfer} total=%{time_total}\n" \
-              "${RUSTFS_SHARED_PROXY_ENDPOINT%/}/health/ready" >> "${ARTIFACT_DIR}/endpoint-ready-probes.log" 2>&1 || true
-          fi
-
-      - name: Generate report
-        if: always()
-        run: |
-          set -euo pipefail
-          LOG_FILE="${POOL_ARTIFACT_DIR}/pool-test.log"
-          REPORT_FILE="${POOL_ARTIFACT_DIR}/pool-report.md"
-          PACKAGE_URL='${{ inputs.package_url }}'
-          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
-          if [ -n "${PACKAGE_URL}" ]; then
-            PACKAGE_SOURCE="${PACKAGE_URL}"
-          elif [ -n "${RUSTFS_VERSION}" ]; then
-            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
-          else
-            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
-          fi
-          {
-            echo "# RustFS pool expansion test report"
-            echo ""
-            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Trigger: ${{ github.event_name }}"
-            echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Warp concurrent: ${{ inputs.warp_concurrent || '32' }}"
-            echo "- Test Step Outcome: ${{ steps.pool_test.outcome }}"
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
-          } | tee "${REPORT_FILE}"
-          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-
-      - name: Validate pool diagnostic completeness
-        if: always()
-        run: |
-          set -euo pipefail
-          failed=0
-          require_nonempty() {
-            if [ ! -s "$1" ]; then
-              echo "required diagnostic is missing or empty: $1" >&2
-              failed=1
-            fi
-          }
-          require_available() {
-            if [ ! -e "$1" ]; then
-              echo "required diagnostic is missing: $1" >&2
-              failed=1
-            elif grep -Fq 'unavailable:' "$1" 2>/dev/null; then
-              echo "required diagnostic could not be collected: $1" >&2
-              failed=1
-            fi
-          }
-
-          require_nonempty "${POOL_ARTIFACT_DIR}/pool-test.log"
-          require_nonempty "${POOL_ARTIFACT_DIR}/warp.log"
-          require_nonempty "${POOL_ARTIFACT_DIR}/pool-report.md"
-          require_nonempty "${POOL_ARTIFACT_DIR}/pool-baseline.log"
-          require_nonempty "${POOL_ARTIFACT_DIR}/nginx-config-redacted.txt"
-          require_nonempty "${POOL_ARTIFACT_DIR}/nginx-rustfs-pool-test-access-redacted.log"
-          require_available "${POOL_ARTIFACT_DIR}/nginx-rustfs-pool-test-access-redacted.log"
-          require_available "${POOL_ARTIFACT_DIR}/nginx-rustfs-pool-test-error-redacted.log"
-          require_nonempty "${POOL_ARTIFACT_DIR}/endpoint-ready-probes.log"
-
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          if grep -Fq 'baseline collection failed' "${POOL_ARTIFACT_DIR}/pool-baseline.log" 2>/dev/null; then
-            echo "one or more node baselines could not be collected" >&2
-            failed=1
-          fi
-          for node in "${NODES[@]}"; do
-            safe_node="${node//[^A-Za-z0-9_.-]/_}"
-            node_log="${POOL_ARTIFACT_DIR}/${safe_node}-rustfs-redacted.log"
-            require_nonempty "${node_log}"
-            if grep -Fq "SSH diagnostics failed for ${node}" "${node_log}" 2>/dev/null; then
-              echo "node diagnostics failed: ${node_log}" >&2
-              failed=1
-            fi
-            if ! grep -Eq '^rustfs @' "${node_log}" 2>/dev/null \
-              || ! grep -Eq '^NRestarts=[0-9]+$' "${node_log}" 2>/dev/null; then
-              echo "node version or restart evidence is incomplete: ${node_log}" >&2
-              failed=1
-            elif grep -Eq '^NRestarts=[1-9][0-9]*$' "${node_log}"; then
-              echo "RustFS restarted unexpectedly during the run: ${node_log}" >&2
-              failed=1
-            fi
-          done
-
-          if ! grep -Fq "upstream_status=\"\$upstream_status\"" \
-            "${POOL_ARTIFACT_DIR}/nginx-config-redacted.txt"; then
-            echo "Nginx config does not expose upstream status fields" >&2
-            failed=1
-          fi
-          if ! grep -Eq '^proxy=.* http=200([[:space:]]|$)' "${POOL_ARTIFACT_DIR}/endpoint-ready-probes.log"; then
-            echo "dedicated proxy readiness probe did not return HTTP 200" >&2
-            failed=1
-          fi
-          if grep -Eq 'status=50(2|4)|upstream_status="[^"]*50(2|4)' \
-            "${POOL_ARTIFACT_DIR}/nginx-rustfs-pool-test-access-redacted.log"; then
-            echo "dedicated proxy access log contains a 502/504 response" >&2
-            failed=1
-          fi
-          if grep -Eiq 'upstream prematurely closed connection|upstream timed out|(connect\(\)|recv\(\)|send\(\)) failed.*upstream|connection reset by peer.*upstream' \
-            "${POOL_ARTIFACT_DIR}/nginx-rustfs-pool-test-error-redacted.log"; then
-            echo "dedicated proxy error log contains an upstream timeout or connection failure" >&2
-            failed=1
-          fi
-          [ "${failed}" -eq 0 ] || exit 1
-
-      - name: Upload functional report to dashboard
-        if: always()
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          SUITE: pool
-        run: |
-          set -euo pipefail
-          REPORT_FILE="${POOL_ARTIFACT_DIR}/pool-report.md"
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
-            exit 0
-          fi
-          DATE="$(date -u +%Y-%m-%d)"
-          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
-          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          fi
-
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          SUITE: 'pool'
-          SUITE_LABEL: 'Pool expansion'
-          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-report.md'
-          LOG_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-test.log'
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
-            exit 0
-          fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
-            exit 0
-          fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
-          fi
-          echo "filed backlog issue for suite ${SUITE}"
+          ./auto-testing/rustfs_pool_expand.sh "${ARGS[@]}"
 
       - name: Upload test logs
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-pool-test-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ runner.temp }}/rustfs-pool-${{ github.run_id }}-${{ github.run_attempt }}
+          name: rustfs-pool-test-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-pool-test*.log
+            /tmp/rustfs-warp.*.log
           if-no-files-found: warn
 
-      - name: Restore dedicated pool proxy
-        if: always()
-        run: |
-          set -euo pipefail
-          RUSTFS_POOL_NGINX_CONFIG_PATH=/etc/nginx/conf.d/rustfs-pool-test.conf \
-          RUSTFS_POOL_NGINX_LISTEN="${RUSTFS_POOL_PROXY_ENDPOINT#http://}" \
-          RUSTFS_POOL_NGINX_ACCESS_LOG=/var/log/nginx/rustfs-pool-test-access.log \
-          RUSTFS_POOL_NGINX_ERROR_LOG=/var/log/nginx/rustfs-pool-test-error.log \
-            ./auto-testing/rustfs_pool_nginx_stage.sh cleanup
-
-      - name: Cleanup environment (after)
+      - name: Reset test environment (after)
         if: ${{ always() && inputs.cleanup_after != 'false' }}
         run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
-            '
-          done
-
-      - name: "Continue functional chain (next: Security)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-security' \
-              -F 'client_payload[from_suite]=pool'; then
-              echo "dispatched next suite Security (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Security after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after pool (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **pool** to **Security** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-security'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-security'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          ./auto-testing/rustfs_pool_expand.sh --reset -y
 
       - name: Notify on failure
         if: failure()
@@ -649,3 +174,82 @@ jobs:
           echo "RustFS pool expansion test failed"
           echo "Package source: ${{ inputs.package_url || inputs.rustfs_version || 'nightly (R2 latest)' }}"
           echo "See the uploaded log artifact for details."
+
+  # Heal regression runs after the pool test regardless of its outcome: a pool
+  # failure must be reported (it makes the run red) but must not block heal.
+  heal-test:
+    name: Heal test (after pool test)
+    runs-on: smoke-testing
+    timeout-minutes: 480
+    needs: pool-expansion-test
+    if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
+    steps:
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+
+      - name: Reset test environment (before)
+        if: ${{ inputs.cleanup_before != 'false' }}
+        run: |
+          chmod +x auto-testing/rustfs_heal_test.sh
+          ./auto-testing/rustfs_heal_test.sh --reset -y
+
+      - name: Install RustFS package & start cluster
+        run: |
+          ARGS=(--steps "1,2" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
+          if [ -n "${{ inputs.package_url }}" ]; then
+            ARGS+=(--package-url "${{ inputs.package_url }}")
+          else
+            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
+          fi
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+
+      - name: Preflight checks
+        run: |
+          ARGS=(--preflight --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
+          if [ -n "${{ inputs.package_url }}" ]; then
+            ARGS+=(--package-url "${{ inputs.package_url }}")
+          else
+            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
+          fi
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+
+      - name: Run heal test (write -> outage -> heal -> verify)
+        run: |
+          ARGS=(--steps "3,4,5,6,7" -y \
+            --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
+            --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
+            --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
+            --log-file /tmp/rustfs-heal-test.log)
+          if [ -n "${{ inputs.package_url }}" ]; then
+            ARGS+=(--package-url "${{ inputs.package_url }}")
+          else
+            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
+          fi
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
+
+      - name: Upload test logs
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: rustfs-heal-test-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-heal-test.log
+            /tmp/rustfs-warp.*.log
+          if-no-files-found: warn
+
+      - name: Reset test environment (after)
+        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        run: |
+          ./auto-testing/rustfs_heal_test.sh --reset -y
+
+      - name: Notify on failure
+        if: failure()
+        run: |
+          echo "RustFS heal test failed"
+          echo "See the uploaded log artifact for details."

.github/workflows/rustfs-replication-test.yml

@@ -1,365 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: RustFS Replication Test
-
-on:
-  workflow_dispatch:
-    inputs:
-      rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
-        required: false
-        default: '1.0.0-rc.4-preview.1'
-      package_url:
-        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
-        required: false
-        type: string
-      suite:
-        description: 'Suite to run (all = bucket REP-* then site SITE-*)'
-        type: choice
-        options:
-          - all
-          - bucket
-          - site
-        default: all
-  repository_dispatch:
-    # Chain handoff: dispatched when the security suite finishes. This is the
-    # last link of the functional chain.
-    types: [rustfs-chain-replication]
-
-permissions:
-  contents: read
-
-# The replication suite uses the same shared VMs as the other functional
-# tests, so it must serialize with them instead of running in parallel.
-concurrency:
-  group: rustfs-shared-functional-tests
-  cancel-in-progress: false
-
-defaults:
-  run:
-    shell: bash
-
-env:
-  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
-  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
-  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
-  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
-  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
-  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
-jobs:
-  replication-test:
-    runs-on: smoke-testing
-    # A failed replication run must not break the chain or the workflow: the
-    # failure is reported to rustfs/backlog instead (see the issue step).
-    continue-on-error: true
-    timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
-    steps:
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
-
-      - name: Show environment
-        run: |
-          uname -a
-          jq --version
-          openssl version
-          aws --version
-          df -h /data | tail -1 || true
-
-      - name: Cleanup environment (before)
-        run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs rustfs-rep2 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /data/rustfs-rep2 /var/log/rustfs /var/log/rustfs-rep2 /var/lib/rustfs/kms
-            '
-          done
-
-      - name: Run replication suite
-        id: test
-        continue-on-error: true
-        env:
-          LOG_FILE: /tmp/rustfs-replication.log
-        run: |
-          set -euo pipefail
-          chmod +x auto-testing/rustfs-replication-test.sh
-          PACKAGE_URL='${{ inputs.package_url }}'
-          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
-          SUITE='${{ inputs.suite }}'
-          ARGS=(-y --log-file "${LOG_FILE}")
-          if [ "${SUITE}" = "all" ] || [ -z "${SUITE}" ] || [ "${SUITE}" = "null" ]; then
-            ARGS+=(--suite all)
-          else
-            ARGS+=(--suite "${SUITE}")
-          fi
-          if [ -n "${PACKAGE_URL}" ]; then
-            ARGS+=(--package-url "${PACKAGE_URL}")
-          elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
-            ARGS+=(--version "${RUSTFS_VERSION}")
-          else
-            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
-          fi
-          ./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"
-
-      - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-replication.log
-          REPORT_FILE: /tmp/rustfs-replication-report.md
-        run: |
-          set -euo pipefail
-          PACKAGE_URL='${{ inputs.package_url }}'
-          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
-          if [ -n "${PACKAGE_URL}" ]; then
-            PACKAGE_SOURCE="${PACKAGE_URL}"
-          elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
-            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
-          else
-            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
-          fi
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          RUSTFS_VERSION_INFO="N/A"
-          if [ "${#NODES[@]}" -gt 0 ]; then
-            DETECTED_VERSION="$(ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
-              "${SSH_USER}@${NODES[0]}" 'rustfs --version' 2>/dev/null | tr -d '\r' | head -n 1 || true)"
-            if [ -n "${DETECTED_VERSION}" ]; then
-              RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
-            fi
-          fi
-          CASE_TABLE="/tmp/rustfs-replication-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-                  for raw in fh:
-                      line = ansi.sub('', raw).strip()
-                      m = start_re.match(line)
-                      if m:
-                          case_id, name = m.group(1), m.group(2)
-                          if case_id not in index:
-                              index[case_id] = len(rows)
-                              rows.append([case_id, name, 'RUNNING'])
-                          continue
-                      m = done_re.match(line)
-                      if m:
-                          status, case_id = m.group(1), m.group(2)
-                          if case_id in index:
-                              rows[index[case_id]][2] = status
-                          else:
-                              rows.append([case_id, case_id, status])
-                              index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-              rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-              counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-              out.write('## Case Summary\n\n')
-              out.write(f"- Total: {len(rows)}\\n")
-              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-              out.write('\\n')
-              out.write('| Case | Name | Status |\\n')
-              out.write('| --- | --- | --- |\\n')
-              for case_id, name, status in rows:
-                  out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
-          {
-            echo "# RustFS replication test report"
-            echo ""
-            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Trigger: ${{ github.event_name }}"
-            echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
-            echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
-          } | tee "${REPORT_FILE}"
-          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-
-      - name: Upload functional report to dashboard
-        if: always()
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-replication-report.md
-          SUITE: replication
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
-            exit 0
-          fi
-          DATE="$(date -u +%Y-%m-%d)"
-          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
-          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          fi
-
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          SUITE: 'replication'
-          SUITE_LABEL: 'Replication (bucket + site)'
-          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-replication-report.md'
-          LOG_FILE: '/tmp/rustfs-replication.log'
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
-            exit 0
-          fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
-            exit 0
-          fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
-          fi
-          echo "filed backlog issue for suite ${SUITE}"
-
-      - name: Upload report and logs
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: rustfs-replication-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-replication.log
-            /tmp/rustfs-replication-report.md
-          if-no-files-found: warn
-
-      - name: Cleanup environment (after)
-        if: always()
-        run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs rustfs-rep2 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /data/rustfs-rep2 /var/log/rustfs /var/log/rustfs-rep2
-            '
-          done
-
-      - name: Chain complete
-        # Replication is the last link of the functional chain: nothing to
-        # dispatch after it. This step just records that the chain finished.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        run: |
-          echo "Functional chain complete: replication (final suite) finished."
-          echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}"
-
-      - name: Notify on failure
-        if: failure()
-        run: |
-          echo "RustFS replication suite failed"
-          echo "Package source: ${{ inputs.package_url || inputs.rustfs_version || 'nightly (R2 latest)' }}"
-          echo "See the uploaded report and log artifacts for details."

.github/workflows/rustfs-s3-compat-test.yml

@@ -11,9 +11,10 @@ on:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
         type: string
-  repository_dispatch:
-    # Chain handoff: dispatched when the upgrade suite finishes.
-    types: [rustfs-chain-s3]
+  workflow_run:
+    # Run after the nightly build completes; the nightly deb is what the test installs.
+    workflows: ["Nightly GNU Build"]
+    types: [completed]
 
 permissions:
   contents: read
@@ -32,34 +33,21 @@ env:
   RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
   RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
-  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
 jobs:
   s3-compat-test:
     runs-on: smoke-testing
-    continue-on-error: true
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     steps:
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
       - name: Show environment
         run: |
@@ -88,7 +76,6 @@ jobs:
 
       - name: Run S3 compatibility suite
         id: test
-        continue-on-error: true
         env:
           LOG_FILE: /tmp/rustfs-s3-compat.log
         run: |
@@ -122,79 +109,12 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          RUSTFS_VERSION_INFO="N/A"
-          if [ "${#NODES[@]}" -gt 0 ]; then
-            DETECTED_VERSION="$(ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
-              "${SSH_USER}@${NODES[0]}" 'rustfs --version' 2>/dev/null | tr -d '\r' | head -n 1 || true)"
-            if [ -n "${DETECTED_VERSION}" ]; then
-              RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
-            fi
-          fi
-          CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          current = None
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  current = case_id
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-                  current = None
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
           {
             echo "# RustFS S3 compatibility test report"
             echo ""
             echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
             echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
-            echo ""
-            cat "${CASE_TABLE}" || true
             echo ""
             echo "## Log tail"
             echo '```text'
@@ -203,92 +123,6 @@ jobs:
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
 
-      - name: Upload functional report to dashboard
-        if: always()
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
-          SUITE: s3
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
-            exit 0
-          fi
-          DATE="$(date -u +%Y-%m-%d)"
-          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
-          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          fi
-
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          SUITE: 's3'
-          SUITE_LABEL: 'S3 compatibility'
-          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
-          LOG_FILE: '/tmp/rustfs-s3-compat.log'
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
-            exit 0
-          fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
-            exit 0
-          fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
-          fi
-          echo "filed backlog issue for suite ${SUITE}"
-
       - name: Upload report and logs
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
@@ -318,55 +152,6 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: KMS)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-kms' \
-              -F 'client_payload[from_suite]=s3'; then
-              echo "dispatched next suite KMS (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch KMS after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after s3 (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **s3** to **KMS** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-kms'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-kms'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
-
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-security-test.yml

@@ -1,318 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: RustFS Security Test
-
-on:
-  workflow_dispatch:
-    inputs:
-      rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
-        required: false
-        default: '1.0.0-rc.4-preview.1'
-      package_url:
-        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
-        required: false
-        type: string
-      topology:
-        description: 'Topology to run (all = SNSD, SNMD, MNMD)'
-        type: choice
-        options:
-          - all
-          - single-single
-          - single-multi
-          - multi-multi
-        default: all
-      oidc_live:
-        description: 'Run the live Keycloak OIDC/SSO gate as part of the suite'
-        type: boolean
-        default: true
-      cleanup_before:
-        description: 'Reset the nodes before the test (DESTROYS existing data/config)'
-        type: boolean
-        default: true
-      cleanup_after:
-        description: 'Reset the nodes after the test (DESTROYS test data/config)'
-        type: boolean
-        default: true
-  repository_dispatch:
-    # Chain handoff: dispatched when the pool expansion suite finishes.
-    types: [rustfs-chain-security]
-
-permissions:
-  contents: read
-
-# The security suite uses the same shared VMs as the other functional tests,
-# so it must serialize with them instead of running in parallel.
-concurrency:
-  group: rustfs-shared-functional-tests
-  cancel-in-progress: false
-
-defaults:
-  run:
-    shell: bash
-
-env:
-  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
-  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
-  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
-  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
-  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
-  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
-jobs:
-  security-test:
-    runs-on: smoke-testing
-    continue-on-error: true
-    timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
-    steps:
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
-
-      - name: Checkout repository (for the OIDC live gate script)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Show environment
-        run: |
-          uname -a
-          jq --version
-          openssl version
-          aws --version || true
-          docker --version || true
-          df -h /data | tail -1
-
-      - name: Cleanup environment (before)
-        if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
-        run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
-            '
-          done
-
-      - name: Run security suite
-        id: test
-        continue-on-error: true
-        env:
-          REPORT_FILE: /tmp/rustfs-security-report.md
-          RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
-        run: |
-          set -euo pipefail
-          chmod +x auto-testing/rustfs-security-test.sh
-          PACKAGE_URL='${{ inputs.package_url }}'
-          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
-          TOPOLOGY='${{ inputs.topology }}'
-          ARGS=(-y)
-          if [ "${TOPOLOGY}" = "all" ] || [ -z "${TOPOLOGY}" ] || [ "${TOPOLOGY}" = "null" ]; then
-            ARGS+=(--all-topologies)
-          else
-            ARGS+=(--topology "${TOPOLOGY}")
-          fi
-          if [ "${{ inputs.oidc_live }}" = "true" ] || [ "${{ github.event_name }}" != "workflow_dispatch" ]; then
-            ARGS+=(--oidc-live)
-          fi
-          if [ -n "${PACKAGE_URL}" ]; then
-            ARGS+=(--package-url "${PACKAGE_URL}")
-          elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
-            ARGS+=(--version "${RUSTFS_VERSION}")
-          else
-            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
-          fi
-          ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
-
-      - name: Generate report
-        if: always()
-        run: |
-          set -euo pipefail
-          if [ ! -f /tmp/rustfs-security-report.md ]; then
-            {
-              echo "# RustFS security test report"
-              echo ""
-              echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-              echo "- Trigger: ${{ github.event_name }}"
-              echo "- Test Step Outcome: failure (suite did not produce a report)"
-            } > /tmp/rustfs-security-report.md
-          fi
-          cat /tmp/rustfs-security-report.md >> "${GITHUB_STEP_SUMMARY}"
-
-      - name: Upload functional report to dashboard
-        if: always()
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-security-report.md
-          SUITE: security
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
-            exit 0
-          fi
-          DATE="$(date -u +%Y-%m-%d)"
-          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
-          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          fi
-
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          SUITE: 'security'
-          SUITE_LABEL: 'Security'
-          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-security-report.md'
-          LOG_FILE: ''
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
-            exit 0
-          fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
-            exit 0
-          fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
-          fi
-          echo "filed backlog issue for suite ${SUITE}"
-
-      - name: Upload report and logs
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: rustfs-security-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-security-report.md
-            /tmp/rustfs-security.*/*
-          if-no-files-found: ignore
-          retention-days: 3
-
-      - name: Cleanup environment (after)
-        if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
-        run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
-            '
-          done
-
-      - name: "Continue functional chain (next: Replication)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          echo "Dispatching next functional suite: Replication"
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-replication' \
-            -F 'client_payload[from_suite]=security'
-
-      - name: Notify on failure
-        if: failure()
-        run: |
-          echo "RustFS security test failed"
-          echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
-          echo "See the uploaded report and logs for details."

.github/workflows/rustfs-storage-test.yml

@@ -1,389 +0,0 @@
-name: RustFS Storage Engine Test
-
-on:
-  workflow_dispatch:
-    inputs:
-      rustfs_version:
-        description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
-        required: false
-        default: '1.0.0-rc.4-preview.1'
-      package_url:
-        description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
-        required: false
-        type: string
-      topology:
-        description: 'Topology to run (all = SNSD, SNMD, MNMD)'
-        type: choice
-        options:
-          - all
-          - single-single
-          - single-multi
-          - multi-multi
-        default: all
-  repository_dispatch:
-    # Chain handoff: dispatched when the tier suite finishes.
-    types: [rustfs-chain-storage]
-
-permissions:
-  contents: read
-
-concurrency:
-  group: rustfs-shared-functional-tests
-  cancel-in-progress: false
-
-defaults:
-  run:
-    shell: bash
-
-env:
-  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
-  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
-  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
-  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
-  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
-  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
-jobs:
-  storage-test:
-    runs-on: smoke-testing
-    continue-on-error: true
-    timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
-    steps:
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
-
-      - name: Show environment
-        run: |
-          uname -a
-          jq --version
-          openssl version
-          df -h /data | tail -1
-
-      - name: Cleanup environment (before)
-        run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
-            '
-          done
-
-      - name: Run storage engine suite
-        id: test
-        continue-on-error: true
-        env:
-          LOG_FILE: /tmp/rustfs-storage.log
-        run: |
-          set -euo pipefail
-          chmod +x auto-testing/rustfs-storage-test.sh
-          PACKAGE_URL='${{ inputs.package_url }}'
-          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
-          TOPOLOGY='${{ inputs.topology }}'
-          ARGS=(-y --log-file "${LOG_FILE}")
-          if [ "${TOPOLOGY}" = "all" ] || [ -z "${TOPOLOGY}" ] || [ "${TOPOLOGY}" = "null" ]; then
-            ARGS+=(--all-topologies)
-          else
-            ARGS+=(--topology "${TOPOLOGY}")
-          fi
-          if [ -n "${PACKAGE_URL}" ]; then
-            ARGS+=(--package-url "${PACKAGE_URL}")
-          elif [ -n "${RUSTFS_VERSION}" ]; then
-            ARGS+=(--version "${RUSTFS_VERSION}")
-          else
-            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
-          fi
-          ./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
-
-      - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-storage.log
-          REPORT_FILE: /tmp/rustfs-storage-report.md
-        run: |
-          set -euo pipefail
-          PACKAGE_URL='${{ inputs.package_url }}'
-          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
-          if [ -n "${PACKAGE_URL}" ]; then
-            PACKAGE_SOURCE="${PACKAGE_URL}"
-          elif [ -n "${RUSTFS_VERSION}" ]; then
-            PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
-          else
-            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
-          fi
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          RUSTFS_VERSION_INFO="N/A"
-          if [ "${#NODES[@]}" -gt 0 ]; then
-            DETECTED_VERSION="$(ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
-              "${SSH_USER}@${NODES[0]}" 'rustfs --version' 2>/dev/null | tr -d '\r' | head -n 1 || true)"
-            if [ -n "${DETECTED_VERSION}" ]; then
-              RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
-            fi
-          fi
-          CASE_TABLE="/tmp/rustfs-storage-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          current = None
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  current = case_id
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-                  current = None
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
-          {
-            echo "# RustFS storage engine test report"
-            echo ""
-            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Trigger: ${{ github.event_name }}"
-            echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
-            echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
-          } | tee "${REPORT_FILE}"
-          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-
-      - name: Upload functional report to dashboard
-        if: always()
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-storage-report.md
-          SUITE: storage
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
-            exit 0
-          fi
-          DATE="$(date -u +%Y-%m-%d)"
-          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
-          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          fi
-
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          SUITE: 'storage'
-          SUITE_LABEL: 'Storage engine'
-          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-storage-report.md'
-          LOG_FILE: '/tmp/rustfs-storage.log'
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
-            exit 0
-          fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
-            exit 0
-          fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
-          fi
-          echo "filed backlog issue for suite ${SUITE}"
-
-      - name: Upload report and logs
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: rustfs-storage-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-storage.log
-            /tmp/rustfs-storage-report.md
-          if-no-files-found: warn
-
-      - name: Cleanup environment (after)
-        if: always()
-        run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs
-            '
-          done
-
-      - name: "Continue functional chain (next: Heal)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-heal' \
-              -F 'client_payload[from_suite]=storage'; then
-              echo "dispatched next suite Heal (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Heal after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after storage (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **storage** to **Heal** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-heal'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-heal'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
-
-      - name: Notify on failure
-        if: failure()
-        run: |
-          echo "RustFS storage engine suite failed"
-          echo "See the uploaded report and log artifacts for details."

.github/workflows/rustfs-tier-test.yml

@@ -11,18 +11,10 @@ on:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
         type: string
-      rc_sha256:
-        description: 'Optional SHA-256 for the preinstalled rc binary; mismatch is an infrastructure failure.'
-        required: false
-        type: string
-      force_case_failure:
-        description: 'Diagnostic only: rewrite single-single/TIER-101 to FAIL after execution to verify artifact and final-gate behavior.'
-        required: false
-        default: false
-        type: boolean
-  repository_dispatch:
-    # Chain handoff: dispatched when the KMS suite finishes.
-    types: [rustfs-chain-tier]
+  workflow_run:
+    # Strict shared-environment order: run after KMS test succeeds.
+    workflows: ["RustFS KMS Test"]
+    types: [completed]
 
 permissions:
   contents: read
@@ -41,50 +33,22 @@ env:
   RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
   RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
-  RUSTFS_EXPECTED_RC_SHA256: ${{ inputs.rc_sha256 || vars.RUSTFS_TIER_RC_SHA256 }}
   PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-  TIER_ARTIFACTS_DIR: /tmp/rustfs-tier-artifacts-${{ github.run_id }}-${{ github.run_attempt }}
 
 jobs:
   tier-test:
     runs-on: smoke-testing
-    # Requirement: a failing suite must not fail the workflow; failures
-    # are filed to rustfs/backlog and the chain continues.
-    continue-on-error: true
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     steps:
-      - name: Initialize run evidence directory
-        id: evidence
-        run: |
-          set -euo pipefail
-          umask 077
-          if ! mkdir -- "${TIER_ARTIFACTS_DIR}"; then
-            echo "refusing to reuse tier evidence path: ${TIER_ARTIFACTS_DIR}" >&2
-            exit 1
-          fi
-          test -d "${TIER_ARTIFACTS_DIR}"
-          test ! -L "${TIER_ARTIFACTS_DIR}"
-
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: main
+          path: auto-testing
+          persist-credentials: false
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
 
       - name: Show environment
         run: |
@@ -96,8 +60,6 @@ jobs:
       - name: Cleanup environment (before)
         run: |
           set -euo pipefail
-          sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
-          sudo rm -f /tmp/rustfs-mosquitto.conf
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
           SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
           for node in "${NODES[@]}"; do
@@ -115,55 +77,26 @@ jobs:
 
       - name: Ensure MQTT broker + clients
         run: |
-          set -euo pipefail
           if ! command -v mosquitto_sub >/dev/null 2>&1; then
             sudo apt-get update
-            sudo apt-get install -y mosquitto-clients
+            sudo apt-get install -y mosquitto mosquitto-clients
           fi
-          command -v docker >/dev/null 2>&1 || { echo 'docker not found on runner'; exit 1; }
-          sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
-          cat <<'EOF' | sudo tee /tmp/rustfs-mosquitto.conf >/dev/null
-          listener 1883 0.0.0.0
-          allow_anonymous true
-          EOF
-          sudo docker run -d --name rustfs-test-mqtt -p 1883:1883 \
-            -v /tmp/rustfs-mosquitto.conf:/mosquitto/config/mosquitto.conf:ro \
-            eclipse-mosquitto:2 >/dev/null
-          for _ in {1..10}; do
-            if ss -tln 2>/dev/null | grep -q ':1883'; then
-              break
-            fi
-            sleep 1
-          done
-          ss -tln 2>/dev/null | grep -q ':1883' || {
-            echo 'mosquitto container is not listening on 1883'
-            sudo docker logs rustfs-test-mqtt || true
-            exit 1
-          }
+          sudo mkdir -p /etc/mosquitto/conf.d
+          printf 'listener 1883 0.0.0.0\nallow_anonymous true\n' | sudo tee /etc/mosquitto/conf.d/rustfs-test.conf >/dev/null
+          sudo systemctl restart mosquitto
+          sleep 2
+          ss -tln 2>/dev/null | grep -q ':1883' || { echo 'mosquitto not listening on 1883'; exit 1; }
 
       - name: Run tier suite
         id: test
-        continue-on-error: true
         env:
-          PACKAGE_URL_INPUT: ${{ inputs.package_url }}
-          RUSTFS_VERSION_INPUT: ${{ inputs.rustfs_version }}
+          LOG_FILE: /tmp/rustfs-tier.log
         run: |
           set -euo pipefail
-          LOG_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier.log"
           chmod +x auto-testing/rustfs-tier-test.sh
-          RC_BIN="$(command -v rc)"
-          PACKAGE_URL="${PACKAGE_URL_INPUT}"
-          RUSTFS_VERSION="${RUSTFS_VERSION_INPUT}"
-          ARGS=(
-            --all-topologies
-            -y
-            --log-file "${LOG_FILE}"
-            --rc-bin "${RC_BIN}"
-            --artifacts-dir "${TIER_ARTIFACTS_DIR}"
-          )
-          if [ -n "${RUSTFS_EXPECTED_RC_SHA256}" ]; then
-            ARGS+=(--expected-rc-sha256 "${RUSTFS_EXPECTED_RC_SHA256}")
-          fi
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
+          ARGS=(--all-topologies -y --log-file "${LOG_FILE}")
           if [ -n "${PACKAGE_URL}" ]; then
             ARGS+=(--package-url "${PACKAGE_URL}")
           elif [ -n "${RUSTFS_VERSION}" ]; then
@@ -173,34 +106,15 @@ jobs:
           fi
           ./auto-testing/rustfs-tier-test.sh "${ARGS[@]}"
 
-      - name: Inject diagnostic case failure
-        if: ${{ always() && steps.evidence.outcome == 'success' && inputs.force_case_failure }}
-        run: |
-          set -euo pipefail
-          RESULT_FILE="${TIER_ARTIFACTS_DIR}/cases/single-single--TIER-101.json"
-          test -s "${RESULT_FILE}"
-          TMP_FILE="$(mktemp "${TIER_ARTIFACTS_DIR}/cases/.forced.XXXXXX")"
-          jq '.status = "FAIL" | .case_rc = 97' "${RESULT_FILE}" > "${TMP_FILE}"
-          mv "${TMP_FILE}" "${RESULT_FILE}"
-
       - name: Generate report
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
         env:
-          PACKAGE_URL_INPUT: ${{ inputs.package_url }}
-          RUSTFS_VERSION_INPUT: ${{ inputs.rustfs_version }}
-          TEST_OUTCOME: ${{ steps.test.outcome }}
-          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          TRIGGER_NAME: ${{ github.event_name }}
+          LOG_FILE: /tmp/rustfs-tier.log
+          REPORT_FILE: /tmp/rustfs-tier-report.md
         run: |
           set -euo pipefail
-          test -d "${TIER_ARTIFACTS_DIR}"
-          test ! -L "${TIER_ARTIFACTS_DIR}"
-          LOG_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier.log"
-          REPORT_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier-report.md"
-          CASE_TABLE="${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md"
-          GATE_RC_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier-gate.rc"
-          PACKAGE_URL="${PACKAGE_URL_INPUT}"
-          RUSTFS_VERSION="${RUSTFS_VERSION_INPUT}"
+          PACKAGE_URL='${{ inputs.package_url }}'
+          RUSTFS_VERSION='${{ inputs.rustfs_version }}'
           if [ -n "${PACKAGE_URL}" ]; then
             PACKAGE_SOURCE="${PACKAGE_URL}"
           elif [ -n "${RUSTFS_VERSION}" ]; then
@@ -208,31 +122,12 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          set +e
-          python3 auto-testing/rustfs_tier_report.py \
-            --results-dir "${TIER_ARTIFACTS_DIR}/cases" \
-            --provenance "${TIER_ARTIFACTS_DIR}/provenance.json" \
-            --output "${CASE_TABLE}"
-          CASE_GATE_RC=$?
-          set -e
-          printf '%s\n' "${CASE_GATE_RC}" > "${GATE_RC_FILE}"
-          if [ ! -s "${CASE_TABLE}" ]; then
-            {
-              echo "## Case Summary"
-              echo ""
-              echo "Structured report generation failed before producing output (exit ${CASE_GATE_RC})."
-            } > "${CASE_TABLE}"
-          fi
           {
             echo "# RustFS tier test report"
             echo ""
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${TRIGGER_NAME}"
+            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            echo "- Trigger: ${{ github.event_name }}"
             echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${TEST_OUTCOME}"
-            echo "- Structured Gate Exit: ${CASE_GATE_RC}"
-            echo ""
-            cat "${CASE_TABLE}"
             echo ""
             echo "## Log tail"
             echo '```text'
@@ -241,76 +136,20 @@ jobs:
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
 
-      - name: Upload functional report to dashboard
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md
-          SUITE: tier
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
-            exit 0
-          fi
-          DATE="$(date -u +%Y-%m-%d)"
-          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
-          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          fi
-
-      - name: Verify required tier evidence
-        id: evidence_verify
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
-        run: |
-          set -euo pipefail
-          failed=0
-          for name in \
-            rustfs-tier.log \
-            rustfs-tier-report.md \
-            rustfs-tier-cases.md \
-            rustfs-tier-gate.rc \
-            provenance.json; do
-            if [ ! -s "${TIER_ARTIFACTS_DIR}/${name}" ]; then
-              echo "required tier evidence is missing or empty: ${name}" >&2
-              failed=1
-            fi
-          done
-          for name in cases logs; do
-            if [ ! -d "${TIER_ARTIFACTS_DIR}/${name}" ]; then
-              echo "required tier evidence directory is missing: ${name}" >&2
-              failed=1
-            fi
-          done
-          if ! find "${TIER_ARTIFACTS_DIR}/cases" -maxdepth 1 -type f -name '*.json' -print -quit 2>/dev/null | grep -q .; then
-            echo "no atomic tier case result was produced" >&2
-            failed=1
-          fi
-          [ "${failed}" -eq 0 ]
-
       - name: Upload report and logs
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
-          name: rustfs-tier-test-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ env.TIER_ARTIFACTS_DIR }}/
-          if-no-files-found: error
+          name: rustfs-tier-test-${{ github.run_id }}
+          path: |
+            /tmp/rustfs-tier.log
+            /tmp/rustfs-tier-report.md
+          if-no-files-found: warn
 
       - name: Cleanup environment (after)
         if: always()
         run: |
           set -euo pipefail
-          sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
-          sudo rm -f /tmp/rustfs-mosquitto.conf
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
           SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
           for node in "${NODES[@]}"; do
@@ -326,157 +165,6 @@ jobs:
             '
           done
 
-      - name: Enforce tier suite result
-        id: gate
-        if: always()
-        env:
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
-          TEST_OUTCOME: ${{ steps.test.outcome }}
-          GATE_RC_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-gate.rc
-        run: |
-          set -euo pipefail
-          failed=0
-          if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
-            echo "tier evidence directory initialization is ${EVIDENCE_OUTCOME}, expected success" >&2
-            failed=1
-          fi
-          if [ "${TEST_OUTCOME}" != "success" ]; then
-            echo "tier suite step outcome is ${TEST_OUTCOME}, expected success" >&2
-            failed=1
-          fi
-          if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
-            echo "structured gate result is unavailable because evidence initialization failed" >&2
-          elif [ ! -s "${GATE_RC_FILE}" ]; then
-            echo "structured gate result is missing" >&2
-            failed=1
-          else
-            GATE_RC="$(tr -d '[:space:]' < "${GATE_RC_FILE}")"
-            if ! [[ "${GATE_RC}" =~ ^[0-9]+$ ]] || [ "${GATE_RC}" -ne 0 ]; then
-              echo "structured 56-case gate failed with exit ${GATE_RC:-invalid}" >&2
-              failed=1
-            fi
-          fi
-          [ "${failed}" -eq 0 ]
-
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled' || steps.evidence_verify.outcome == 'failure' || steps.evidence_verify.outcome == 'cancelled' || steps.gate.outcome == 'failure' || steps.gate.outcome == 'cancelled') }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          SUITE: 'tier'
-          SUITE_LABEL: 'Tier'
-          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          EVIDENCE_DIR: ${{ env.TIER_ARTIFACTS_DIR }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
-          VERIFY_OUTCOME: ${{ steps.evidence_verify.outcome }}
-          GATE_OUTCOME: ${{ steps.gate.outcome }}
-          REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md
-          LOG_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier.log
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
-            exit 0
-          fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
-            exit 0
-          fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo "- Evidence initialization: ${EVIDENCE_OUTCOME}"
-            echo "- Evidence verification: ${VERIFY_OUTCOME}"
-            echo "- Final gate: ${GATE_OUTCOME}"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
-              echo "(the run evidence directory was rejected; its contents were not read)"
-            elif [ ! -d "${EVIDENCE_DIR}" ] || [ -L "${EVIDENCE_DIR}" ]; then
-              echo "(the run evidence directory is missing or unsafe; its contents were not read)"
-            elif [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
-          fi
-          echo "filed backlog issue for suite ${SUITE}"
-
-      - name: "Continue functional chain (next: Storage engine)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-storage' \
-              -F 'client_payload[from_suite]=tier'; then
-              echo "dispatched next suite Storage engine (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Storage engine after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after tier (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **tier** to **Storage engine** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-storage'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-storage'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
-
       - name: Notify on failure
         if: failure()
         run: |

.github/workflows/rustfs-upgrade-test.yml

@@ -1,444 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: RustFS Upgrade Test
-
-on:
-  workflow_dispatch:
-    inputs:
-      from_version:
-        description: 'OLD RustFS release tag (e.g. 1.0.0-rc.4-preview.1)'
-        required: false
-        default: '1.0.0-rc.4-preview.1'
-      from_url:
-        description: 'OLD .deb URL. Overrides from_version.'
-        required: false
-        type: string
-      to_version:
-        description: 'NEW RustFS release tag (leave empty for latest nightly)'
-        required: false
-      to_url:
-        description: 'NEW .deb URL. Overrides to_version / nightly default.'
-        required: false
-        type: string
-      topology:
-        description: 'Topology to run (all = SNSD, SNMD, MNMD)'
-        type: choice
-        options:
-          - all
-          - single-single
-          - single-multi
-          - multi-multi
-        default: all
-      backends:
-        description: 'KMS backends to run (local,vault-kv2)'
-        required: false
-        default: 'local,vault-kv2'
-      cleanup_before:
-        description: 'Reset the nodes before the test (DESTROYS existing data/config)'
-        type: boolean
-        default: true
-      cleanup_after:
-        description: 'Reset the nodes after the test (DESTROYS test data/config)'
-        type: boolean
-        default: true
-  repository_dispatch:
-    # Functional-chain entry: dispatched by rustfs-functional-chain.yml.
-    types: [rustfs-chain-upgrade]
-
-permissions:
-  contents: read
-
-concurrency:
-  group: rustfs-shared-functional-tests
-  cancel-in-progress: false
-
-defaults:
-  run:
-    shell: bash
-
-env:
-  RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
-  RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
-  RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
-  RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
-  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
-  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
-jobs:
-  upgrade-test:
-    runs-on: smoke-testing
-    continue-on-error: true
-    timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
-    steps:
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
-
-      - name: Show environment
-        run: |
-          uname -a
-          jq --version
-          openssl version
-          aws --version || true
-          docker --version || true
-          df -h /data | tail -1
-
-      - name: Cleanup environment (before)
-        if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
-        run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
-            '
-          done
-
-      - name: Ensure docker (Vault container)
-        run: |
-          if ! command -v docker >/dev/null 2>&1; then
-            sudo apt-get update
-            sudo apt-get install -y docker.io
-          fi
-          sudo systemctl enable --now docker
-          docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1
-
-      - name: Run upgrade compatibility suite
-        id: test
-        continue-on-error: true
-        env:
-          LOG_FILE: /tmp/rustfs-upgrade.log
-        run: |
-          set -euo pipefail
-          chmod +x auto-testing/rustfs-upgrade-test.sh
-          FROM_URL='${{ inputs.from_url }}'
-          FROM_VERSION='${{ inputs.from_version }}'
-          TO_URL='${{ inputs.to_url }}'
-          TO_VERSION='${{ inputs.to_version }}'
-          TOPOLOGY='${{ inputs.topology }}'
-          BACKENDS='${{ inputs.backends }}'
-          ARGS=(-y --log-file "${LOG_FILE}")
-          if [ "${TOPOLOGY}" = "all" ] || [ -z "${TOPOLOGY}" ] || [ "${TOPOLOGY}" = "null" ]; then
-            ARGS+=(--all-topologies)
-          else
-            ARGS+=(--topology "${TOPOLOGY}")
-          fi
-          if [ -n "${BACKENDS}" ] && [ "${BACKENDS}" != "null" ]; then
-            ARGS+=(--backends "${BACKENDS}")
-          fi
-          if [ -n "${FROM_URL}" ]; then
-            ARGS+=(--from-url "${FROM_URL}")
-          elif [ -n "${FROM_VERSION}" ] && [ "${FROM_VERSION}" != "null" ]; then
-            ARGS+=(--from-version "${FROM_VERSION}")
-          fi
-          if [ -n "${TO_URL}" ]; then
-            ARGS+=(--to-url "${TO_URL}")
-          elif [ -n "${TO_VERSION}" ] && [ "${TO_VERSION}" != "null" ]; then
-            ARGS+=(--to-version "${TO_VERSION}")
-          else
-            ARGS+=(--to-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
-          fi
-          ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
-
-      - name: Generate report
-        if: always()
-        env:
-          LOG_FILE: /tmp/rustfs-upgrade.log
-          REPORT_FILE: /tmp/rustfs-upgrade-report.md
-        run: |
-          set -euo pipefail
-          FROM_URL='${{ inputs.from_url }}'
-          FROM_VERSION='${{ inputs.from_version }}'
-          TO_URL='${{ inputs.to_url }}'
-          TO_VERSION='${{ inputs.to_version }}'
-          if [ -n "${FROM_URL}" ]; then
-            FROM_SOURCE="${FROM_URL}"
-          elif [ -n "${FROM_VERSION}" ]; then
-            FROM_SOURCE="version ${FROM_VERSION}"
-          else
-            FROM_SOURCE="release (default)"
-          fi
-          if [ -n "${TO_URL}" ]; then
-            TO_SOURCE="${TO_URL}"
-          elif [ -n "${TO_VERSION}" ]; then
-            TO_SOURCE="version ${TO_VERSION}"
-          else
-            TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
-          fi
-          CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
-          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-
-          rows = []
-          index = {}
-          try:
-            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-              for raw in fh:
-                line = ansi.sub('', raw).strip()
-                m = start_re.match(line)
-                if m:
-                  case_id, name = m.group(1), m.group(2)
-                  if case_id not in index:
-                    index[case_id] = len(rows)
-                    rows.append([case_id, name, 'RUNNING'])
-                  continue
-                m = done_re.match(line)
-                if m:
-                  status, case_id = m.group(1), m.group(2)
-                  if case_id in index:
-                    rows[index[case_id]][2] = status
-                  else:
-                    rows.append([case_id, case_id, status])
-                    index[case_id] = len(rows) - 1
-          except FileNotFoundError:
-            rows = []
-
-          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
-          for _, _, status in rows:
-            counts[status] = counts.get(status, 0) + 1
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-            out.write('## Case Summary\n\n')
-            out.write(f"- Total: {len(rows)}\\n")
-            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-            out.write('\\n')
-            out.write('| Case | Name | Status |\\n')
-            out.write('| --- | --- | --- |\\n')
-            for case_id, name, status in rows:
-              out.write(f'| {case_id} | {name} | {status} |\\n')
-          PY
-          {
-            echo "# RustFS upgrade compatibility report"
-            echo ""
-            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Trigger: ${{ github.event_name }}"
-            echo "- From: ${FROM_SOURCE}"
-            echo "- To: ${TO_SOURCE}"
-            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
-            echo ""
-            cat "${CASE_TABLE}" || true
-            echo ""
-            echo "## Log tail"
-            echo '```text'
-            tail -n 200 "${LOG_FILE}" || true
-            echo '```'
-          } | tee "${REPORT_FILE}"
-          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-
-      - name: Upload functional report to dashboard
-        if: always()
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
-          REPORT_FILE: /tmp/rustfs-upgrade-report.md
-          SUITE: upgrade
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
-            exit 0
-          fi
-          DATE="$(date -u +%Y-%m-%d)"
-          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
-          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
-          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
-              '{message:$msg, content:$content, sha:$sha}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
-              '{message:$msg, content:$content}' \
-              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
-          fi
-
-      - name: File failure issue in rustfs/backlog
-        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          SUITE: 'upgrade'
-          SUITE_LABEL: 'Upgrade compatibility'
-          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
-          LOG_FILE: '/tmp/rustfs-upgrade.log'
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
-            exit 0
-          fi
-          TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
-          EXISTING="$(gh issue list -R rustfs/backlog --state all \
-            --search "in:title \"run ${GITHUB_RUN_ID}\"" \
-            --json number --jq '.[].number' || true)"
-          if [ -n "${EXISTING}" ]; then
-            echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
-            exit 0
-          fi
-          redact() {
-            sed -E \
-              -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
-              -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
-              -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
-          }
-          BODY_FILE="$(mktemp)"
-          {
-            echo "The **${SUITE_LABEL}** functional suite failed."
-            echo ""
-            echo "- Suite: \`${SUITE}\`"
-            echo "- Run: ${RUN_URL}"
-            echo "- Trigger: ${GITHUB_EVENT_NAME}"
-            echo "- Date: $(date -u +%Y-%m-%d)"
-            echo ""
-            echo "## Report (errors and symptoms)"
-            echo ""
-            if [ -s "${REPORT_FILE}" ]; then
-              redact < "${REPORT_FILE}"
-            elif [ -s "${LOG_FILE:-}" ]; then
-              echo "(report file missing; log tail below)"
-              echo ""
-              tail -n 200 "${LOG_FILE}" | redact
-            else
-              echo "(no report or log file was produced)"
-            fi
-          } | head -c 55000 > "${BODY_FILE}"
-          gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
-          if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test; then
-            gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
-          fi
-          echo "filed backlog issue for suite ${SUITE}"
-
-      - name: Upload report and logs
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: rustfs-upgrade-test-${{ github.run_id }}
-          path: |
-            /tmp/rustfs-upgrade-report.md
-            /tmp/rustfs-upgrade.*/*
-          if-no-files-found: ignore
-          retention-days: 3
-
-      - name: Cleanup environment (after)
-        if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
-        run: |
-          set -euo pipefail
-          read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
-          SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
-          for node in "${NODES[@]}"; do
-            ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
-              set -euo pipefail
-              SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
-              ${SUDO} systemctl stop rustfs 2>/dev/null || true
-              if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
-                ${SUDO} dpkg -P rustfs
-              fi
-              for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
-              ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
-            '
-          done
-
-      - name: "Continue functional chain (next: S3 compatibility)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-s3' \
-              -F 'client_payload[from_suite]=upgrade'; then
-              echo "dispatched next suite S3 compatibility (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch S3 compatibility after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after upgrade (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **upgrade** to **S3 compatibility** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-s3'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-s3'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
-
-      - name: Notify on failure
-        if: failure()
-        run: |
-          echo "RustFS upgrade compatibility test failed"
-          echo "From: ${{ inputs.from_url || inputs.from_version || 'release (default)' }}"
-          echo "To: ${{ inputs.to_url || inputs.to_version || 'nightly (R2 latest)' }}"
-          echo "See the uploaded report and logs for details."

.github/workflows/star-history.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: overtrue/repo-visuals-action@fd79cba437ecfac933d00a69add17eb95d3939c3 # v1.3.1
+      - uses: overtrue/repo-visuals-action@72f34d24769ff5d341956da2f23952594ef2f1e2 # v1.3.0
         with:
           github-token: ${{ github.token }}
           output-branch: star-history

.gitignore

@@ -57,6 +57,9 @@ docs/*
 !docs/operations/**
 !docs/testing/
 !docs/testing/**
+docs/heal-scanner-logging-governance.md
+docs/benchmark/rustfs-target-bench/
+docs/benchmark/*.md
 .codegraph/*
 .docker/test/compat/data/*
 .docker/test/compat/kms/*
@@ -80,8 +83,6 @@ worktrees/*
 
 # Local AI-agent review artifacts (omo evidence dumps)
 .omo/
-# Legacy per-tool skill dir; skills live in .agents/skills (shared by all agents)
-.mimocode/
 
 # insta scratch files; the accepted .snap files ARE the assertions and are committed
 *.snap.new

AGENTS.md

@@ -86,7 +86,6 @@ This file contains repository-wide rules. Use the nearest subdirectory
 - CI gates: `.github/workflows/ci.yml`.
 - PR format: `.github/pull_request_template.md`.
 - Architecture routing: `ARCHITECTURE.md` and `docs/architecture/README.md`.
-- Knowledge-base index and documentation rules: `docs/architecture/README.md`.
 - Agent skills: `.agents/skills/*/SKILL.md`.
 
 Do not commit one-shot plans, trackers, migration ledgers, benchmark snapshots,
@@ -124,13 +123,12 @@ runtime/build output:
 - Use `make pre-commit` only when its repository-wide fast checks add confidence
   beyond the focused checks.
 
-### Broad Cross-Module Changes
+### Broad or High-Risk Changes
 
-Do not run `make pre-pr` by default before opening a PR. Consider it only when
-the final diff is broad, spans multiple modules, and targeted checks cannot
-bound the impact. Decide dynamically from the affected boundaries and risks;
-otherwise use the scoped formatting, linting, compilation, and test checks
-above.
+After the required adversarial review, run `make pre-pr` when targeted coverage
+cannot bound the impact, including dependency/toolchain/build-matrix changes,
+unbounded cross-crate APIs, or locking, durability, erasure coding, replication,
+RPC, IAM/KMS/auth, cryptography, on-disk/on-wire, and S3-visible behavior.
 
 `make pre-pr` includes `make pre-commit`; never run both for the same unchanged
 diff. Do not repeat a check already covered by a successful umbrella gate.

ARCHITECTURE.md

@@ -62,7 +62,7 @@ rustfs/                      # Workspace root (virtual manifest)
 │   ├── utils/               # Pure utility functions
 │   ├── ...                  # (see "Crate Reference" below)
 │   └── e2e_test/            # End-to-end integration tests
-└── docs/                    # Agent knowledge base: contracts, runbooks, testing rules (index: docs/architecture/README.md)
+└── docs/                    # Design documents and analysis
 ```
 
 ### Main Crate Layers (`rustfs/src/`)

CLAUDE.md

@@ -15,7 +15,7 @@ cargo check -p <crate>               # fast type-check one crate
 cargo test -p <crate>                # test one crate
 cargo fmt --all                      # format (required before PR)
 make pre-commit                      # fast gate: fmt + arch checks + quick-check (NO clippy/tests)
-make pre-pr                          # optional full gate for broad cross-module changes
+make pre-pr                          # full pre-PR gate: fmt + arch checks + clippy + tests
 make build-docker BUILD_OS=ubuntu22.04
 ```
 
@@ -27,7 +27,6 @@ make build-docker BUILD_OS=ubuntu22.04
 
 ## Where to look (do not duplicate here)
 
-- Agent knowledge base index and doc-writing rules: [docs/architecture/README.md](docs/architecture/README.md)
 - Crate membership: `Cargo.toml` `[workspace].members`
 - Architecture, layering, crate map: [ARCHITECTURE.md](ARCHITECTURE.md)
 - Migration guardrails & readiness contracts: [docs/architecture/](docs/architecture/README.md)

CONTRIBUTING.md

@@ -62,20 +62,12 @@ make test
 # Fast pre-commit gate — see below for exactly what it runs
 make pre-commit
 
-# Optional full gate for broad cross-module changes (pre-commit + clippy + tests)
+# Full pre-PR gate (pre-commit gates + clippy + tests)
 make pre-pr
 ```
 
 > `make test` requires [cargo-nextest](https://nexte.st) (CI runs it and only nextest honours `.config/nextest.toml` test-groups). Install it with `cargo install cargo-nextest --locked` or a prebuilt binary (see https://nexte.st/docs/installation/). To run the plain `cargo test` fallback anyway (results not authoritative — serialization semantics differ from CI), set `RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1`.
 
-> Some guard checks are Python (`test-wiring-check` in `make pre-commit`, plus the
-> security-coverage and scheduled-validation self-tests in `make test`) and import
-> `tomllib`, so they need **Python 3.11+**. Make resolves the interpreter through
-> `scripts/python_bin.sh`, which prefers a `python3.11`+ on `PATH` and otherwise falls
-> back to `uv run --python 3.12`. macOS ships `/usr/bin/python3` at 3.9, so install a
-> newer one (`brew install python@3.12`) or [uv](https://docs.astral.sh/uv/); pin a
-> specific interpreter with `RUSTFS_PYTHON=/path/to/python3.12`.
-
 > For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md).
 
 > For the event, timeout, required-status, and local reproduction matrix, see [docs/testing/ci-gates.md](docs/testing/ci-gates.md).
@@ -96,16 +88,14 @@ make pre-pr
 8. `quick-check` — `cargo check --workspace --exclude e2e_test`
 
 **`make pre-commit` does NOT run clippy and does NOT run any tests.**
-It does not replace the scoped Clippy and test checks applicable to a change.
+A green `make pre-commit` is not enough to open a pull request.
 
 `make pre-pr` is the **full** gate: it runs all of the guard checks above,
 then `clippy-check` (`cargo clippy --all-targets --all-features -- -D warnings`)
 and `test` (shell script tests, workspace tests excluding `e2e_test`, and doc
 tests). Complete the applicable multi-role adversarial review described in
-`AGENTS.md` first. Do not run `make pre-pr` locally by default before opening or
-updating a pull request. Consider it only for a broad change that spans multiple
-modules and whose impact cannot be bounded by targeted checks; decide from the
-affected boundaries and risks. CI still runs its configured repository gates.
+`AGENTS.md` before running `make pre-pr`; then run the gate before opening or
+updating a pull request. This is what CI enforces.
 
 ### 🔒 Git Pre-commit Hooks (optional)
 
@@ -124,9 +114,8 @@ Or manually:
 chmod +x .git/hooks/pre-commit
 ```
 
-With or without a hook, follow the verification tiers in `AGENTS.md`. Run the
-applicable scoped checks, and reserve `make pre-pr` for broad cross-module
-changes whose impact cannot be bounded by those checks.
+With or without a hook, the expectation is the same: run `make pre-commit`
+before committing and `make pre-pr` before opening a pull request.
 
 ### 📝 Formatting Configuration
 
@@ -165,9 +154,7 @@ Example output when formatting fails:
 3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
 4. **Commit your changes**: `git commit -m "your message"`
 5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
-6. **Run applicable scoped checks before opening/updating a PR**; consider
-   `make pre-pr` only for broad cross-module changes whose impact cannot be
-   bounded by targeted checks
+6. **Run the full gate before opening/updating a PR**: `make pre-pr` (clippy + tests)
 7. **Push to your branch**: `git push`
 
 ### 🛠️ IDE Integration

Cargo.lock

@@ -347,9 +347,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
 
 [[package]]
 name = "arrow"
-version = "59.3.0"
+version = "59.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c14b3d39f306bc28fd639d59f06e17a0f377d0021e1b7e9054e4d6fedc98774"
+checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -368,9 +368,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-arith"
-version = "59.3.0"
+version = "59.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce2961626677665b2195eb59242af4c7befe7b8737ca2050295389362380104e"
+checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -382,9 +382,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-array"
-version = "59.3.0"
+version = "59.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584"
+checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97"
 dependencies = [
  "ahash",
  "arrow-buffer",
@@ -401,9 +401,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-buffer"
-version = "59.3.0"
+version = "59.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182"
+checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90"
 dependencies = [
  "bytes",
  "half",
@@ -413,9 +413,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-cast"
-version = "59.3.0"
+version = "59.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "635c9c635668ad26adf76cce8fb276c4be7cf06e63bd516de7da514f9680ee53"
+checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -435,9 +435,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-csv"
-version = "59.3.0"
+version = "59.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c2ebf8d631e79b02c16cf5ae860561272c26024ec88fce389a56aaddd558e86"
+checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36"
 dependencies = [
  "arrow-array",
  "arrow-cast",
@@ -450,9 +450,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "59.3.0"
+version = "59.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358"
+checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
@@ -463,9 +463,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ipc"
-version = "59.3.0"
+version = "59.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599"
+checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -479,9 +479,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-json"
-version = "59.3.0"
+version = "59.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2f57d7a81969f24ccf80809587b76c09897e6f829d2d65a5976bfb3218851f1"
+checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -504,9 +504,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ord"
-version = "59.3.0"
+version = "59.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c900759f3bd8354fd4196bc4403eee846894dc2adf66b4225472006a0bf18c5"
+checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -517,9 +517,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-row"
-version = "59.3.0"
+version = "59.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4c6425032e28266e3fc4ff680805e57e670d6ea92473043f3e65b7ed6ac79f2"
+checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -530,9 +530,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-schema"
-version = "59.3.0"
+version = "59.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd"
+checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e"
 dependencies = [
  "serde_core",
  "serde_json",
@@ -540,9 +540,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-select"
-version = "59.3.0"
+version = "59.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8"
+checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -554,9 +554,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-string"
-version = "59.3.0"
+version = "59.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e0813f3c35c1cfea65e14c20a953440f7783c088b7ad2d0db162ccdeefcec14"
+checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -627,7 +627,8 @@ dependencies = [
 [[package]]
 name = "astral-tokio-tar"
 version = "0.7.0"
-source = "git+https://github.com/cxymds/tokio-tar.git?rev=603756478b7668436e464519c77ccac22a99ba96#603756478b7668436e464519c77ccac22a99ba96"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f2e989b33246fe9240d39accf4dd9a01e0b6c1f3ce9dd095e0a47fa02505523"
 dependencies = [
  "futures-core",
  "libc",
@@ -919,9 +920,9 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-rs"
-version = "1.18.1"
+version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e"
+checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e"
 dependencies = [
  "aws-lc-sys",
  "untrusted 0.7.1",
@@ -930,9 +931,9 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-sys"
-version = "0.45.0"
+version = "0.44.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27"
+checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483"
 dependencies = [
  "cc",
  "cmake",
@@ -2162,7 +2163,6 @@ dependencies = [
  "compression-core",
  "flate2",
  "liblzma",
- "lz4",
  "memchr",
  "zstd",
  "zstd-safe",
@@ -3916,7 +3916,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
 
 [[package]]
 name = "e2e_test"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "anyhow",
  "astral-tokio-tar",
@@ -3926,7 +3926,6 @@ dependencies = [
  "aws-sdk-s3",
  "aws-sdk-sts",
  "aws-smithy-http-client",
- "aws-smithy-types",
  "base64-simd",
  "bytes",
  "chrono",
@@ -3944,7 +3943,6 @@ dependencies = [
  "hyper-util",
  "local-ip-address",
  "md-5 0.11.0",
- "minlz",
  "opentelemetry-proto",
  "prost 0.14.4",
  "rand 0.10.2",
@@ -4210,7 +4208,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -5039,9 +5037,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
 [[package]]
 name = "hermit-abi"
-version = "0.5.3"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284"
+checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
 
 [[package]]
 name = "hex"
@@ -5668,7 +5666,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
 dependencies = [
  "hermit-abi",
  "libc",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -6106,9 +6104,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
 
 [[package]]
 name = "libredox"
-version = "0.1.23"
+version = "0.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed"
+checksum = "d7955dfc218a8afb29dfeffd540e3a6e96baeb94fe7138228dd7cc6937fbbf96"
 dependencies = [
  "libc",
 ]
@@ -6650,11 +6648,10 @@ dependencies = [
 
 [[package]]
 name = "mysql_async"
-version = "0.37.1"
+version = "0.37.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40d11da0e2d9fad4640c9f9198ee431c6d68444568f83ef1f10f3367270071e4"
+checksum = "3519e91b0d254ac1ffa495bc42053286cb2172ad7241d5b3b1b9f8a891f21ee2"
 dependencies = [
- "arc-swap",
  "bytes",
  "crossbeam-queue",
  "crossbeam-utils",
@@ -6987,9 +6984,9 @@ checksum = "a3c00a0c9600379bd32f8972de90676a7672cba3bf4886986bc05902afc1e093"
 
 [[package]]
 name = "nvml-wrapper"
-version = "0.13.0"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d164abbde0b3c03edb9edb9cb8d31a7f5b79015c692b7c771f6e0840e9106b9f"
+checksum = "f049ae562349fefb8e837eb15443da1e7c6dcbd8a11f52a228f92220c2e5c85e"
 dependencies = [
  "bitflags 2.13.1",
  "libloading",
@@ -7001,9 +6998,9 @@ dependencies = [
 
 [[package]]
 name = "nvml-wrapper-sys"
-version = "0.10.0"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d2079f4c9b6d2170bfb71c6355734ead6c47da75c179847395c31f9f2f66ede"
+checksum = "6b4d594420fcda43b1c2c4bd44d48974aa3c7a9ab2cbf10dc18e35265767bf0b"
 dependencies = [
  "libloading",
 ]
@@ -7528,9 +7525,9 @@ dependencies = [
 
 [[package]]
 name = "parquet"
-version = "59.3.0"
+version = "59.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff322f54b1a0f9288e614ed1f2d329b380af5476420db19f46ffb865e1163d73"
+checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -8037,9 +8034,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
 
 [[package]]
 name = "ppmd-rust"
-version = "1.4.1"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e9219bcb9d7aca6b2f63c83cf100cf78bcd619ac46e6ecbd0dd90869a39345d"
+checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24"
 
 [[package]]
 name = "ppv-lite86"
@@ -8611,7 +8608,7 @@ dependencies = [
  "once_cell",
  "socket2",
  "tracing",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -9387,7 +9384,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "aes-gcm",
  "anyhow",
@@ -9462,7 +9459,6 @@ dependencies = [
  "rustfs-io-metrics",
  "rustfs-keystone",
  "rustfs-kms",
- "rustfs-license",
  "rustfs-lock",
  "rustfs-log-analyzer",
  "rustfs-madmin",
@@ -9514,7 +9510,7 @@ dependencies = [
  "tokio-util",
  "tonic",
  "tower",
- "tower-http 0.7.1",
+ "tower-http 0.7.0",
  "tracing",
  "tracing-opentelemetry",
  "tracing-subscriber",
@@ -9529,7 +9525,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-audit"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "const-str",
  "futures",
@@ -9551,7 +9547,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-checksums"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "base64-simd",
  "bytes",
@@ -9567,7 +9563,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-common"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "hotpath",
  "metrics",
@@ -9580,7 +9576,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-concurrency"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "hotpath",
  "insta",
@@ -9593,7 +9589,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-config"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "const-str",
  "hotpath",
@@ -9603,7 +9599,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-credentials"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "base64-simd",
  "hmac 0.13.0",
@@ -9617,7 +9613,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-crypto"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "aes-gcm",
  "argon2",
@@ -9638,7 +9634,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-data-usage"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "hotpath",
  "rmp-serde",
@@ -9648,7 +9644,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-ecstore"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "arc-swap",
  "async-channel",
@@ -9783,7 +9779,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-extension-schema"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "hotpath",
  "serde",
@@ -9793,7 +9789,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-filemeta"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "arc-swap",
  "byteorder",
@@ -9820,7 +9816,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-heal"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "async-trait",
  "base64-simd",
@@ -9856,7 +9852,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-heal-contracts"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "serde",
  "serde_json",
@@ -9866,7 +9862,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-iam"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -9914,7 +9910,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-io-core"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "bytes",
  "hotpath",
@@ -9926,7 +9922,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-io-metrics"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "criterion",
  "hotpath",
@@ -9990,7 +9986,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-keystone"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "bytes",
  "futures",
@@ -10017,7 +10013,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-kms"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "aes-gcm",
  "anyhow",
@@ -10065,16 +10061,9 @@ dependencies = [
  "zeroize",
 ]
 
-[[package]]
-name = "rustfs-license"
-version = "1.0.0-rc.5"
-dependencies = [
- "thiserror 2.0.20",
-]
-
 [[package]]
 name = "rustfs-lifecycle"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "async-trait",
  "hotpath",
@@ -10091,12 +10080,13 @@ dependencies = [
  "time",
  "tokio",
  "tracing",
+ "url",
  "uuid",
 ]
 
 [[package]]
 name = "rustfs-lock"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "async-trait",
  "compact_str",
@@ -10119,7 +10109,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-log-analyzer"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "chrono",
  "flate2",
@@ -10138,7 +10128,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-madmin"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "hotpath",
  "http 1.5.0",
@@ -10176,7 +10166,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-notify"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -10211,7 +10201,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-object-capacity"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "criterion",
  "futures",
@@ -10230,7 +10220,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-object-data-cache"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "bytes",
  "criterion",
@@ -10247,7 +10237,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-obs"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "chrono",
  "crossbeam-channel",
@@ -10305,7 +10295,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-policy"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "async-trait",
  "base64-simd",
@@ -10336,7 +10326,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-protocols"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "astral-tokio-tar",
  "async-compression",
@@ -10398,7 +10388,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-protos"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "flatbuffers",
  "hotpath",
@@ -10423,7 +10413,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-replication"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "byteorder",
  "bytes",
@@ -10441,7 +10431,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-rio"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "aes-gcm",
  "arc-swap",
@@ -10458,7 +10448,6 @@ dependencies = [
  "hyper",
  "hyper-util",
  "md-5 0.11.0",
- "minlz",
  "pin-project-lite",
  "rand 0.10.2",
  "reqwest",
@@ -10482,7 +10471,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-rio-v2"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "aes-gcm",
  "bytes",
@@ -10505,7 +10494,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-s3-client"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "base64-simd",
  "bytes",
@@ -10549,7 +10538,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-s3-ops"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "hotpath",
  "rustfs-s3-types",
@@ -10557,7 +10546,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-s3-types"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "hotpath",
  "serde",
@@ -10566,16 +10555,12 @@ dependencies = [
 
 [[package]]
 name = "rustfs-s3select-api"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
- "arc-swap",
- "async-compression",
  "async-trait",
  "bytes",
  "chrono",
- "crc-fast",
  "datafusion",
- "flate2",
  "futures",
  "futures-core",
  "hotpath",
@@ -10591,7 +10576,6 @@ dependencies = [
  "serial_test",
  "thiserror 2.0.20",
  "tokio",
- "tokio-stream",
  "tokio-util",
  "tracing",
  "transform-stream",
@@ -10601,7 +10585,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-s3select-query"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "async-recursion",
  "async-trait",
@@ -10614,15 +10598,13 @@ dependencies = [
  "rustfs-s3select-api",
  "rustfs-test-utils",
  "s3s",
- "serde",
- "serde_json",
  "tokio",
  "tracing",
 ]
 
 [[package]]
 name = "rustfs-scanner"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "async-trait",
  "bytes",
@@ -10665,7 +10647,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-scanner-contracts"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "chrono",
  "jiff",
@@ -10680,7 +10662,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-security-governance"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "hotpath",
  "thiserror 2.0.20",
@@ -10688,7 +10670,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-signer"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "base64-simd",
  "bytes",
@@ -10706,7 +10688,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-storage-api"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "async-trait",
  "hotpath",
@@ -10721,7 +10703,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-targets"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "arc-swap",
  "async-nats",
@@ -10775,7 +10757,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-test-utils"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "hotpath",
  "rustfs-data-usage",
@@ -10791,7 +10773,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-tls-runtime"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "arc-swap",
  "hotpath",
@@ -10812,7 +10794,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-trusted-proxies"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "async-trait",
  "axum",
@@ -10849,7 +10831,7 @@ dependencies = [
 
 [[package]]
 name = "rustfs-utils"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "base64-simd",
  "blake2",
@@ -10891,11 +10873,10 @@ dependencies = [
 
 [[package]]
 name = "rustfs-zip"
-version = "1.0.0-rc.5"
+version = "1.0.0-rc.4"
 dependencies = [
  "async-compression",
  "hotpath",
- "rustfs-rio",
  "thiserror 2.0.20",
  "tokio",
 ]
@@ -10953,7 +10934,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -11026,7 +11007,7 @@ dependencies = [
  "security-framework",
  "security-framework-sys",
  "webpki-root-certs",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -11083,7 +11064,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 [[package]]
 name = "s3s"
 version = "0.15.0"
-source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
+source = "git+https://github.com/rustfs/s3s.git?rev=9c4690d8e73fc8d184031a19b2c4539ebc77d180#9c4690d8e73fc8d184031a19b2c4539ebc77d180"
 dependencies = [
  "arc-swap",
  "arrayvec",
@@ -11114,7 +11095,6 @@ dependencies = [
  "pin-project-lite",
  "quick-xml",
  "regex",
- "s3s-rfc2047",
  "s3s-sigv2",
  "s3s-sigv4",
  "serde",
@@ -11138,45 +11118,28 @@ dependencies = [
  "zeroize",
 ]
 
-[[package]]
-name = "s3s-rfc2047"
-version = "0.16.0-alpha.1"
-source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
-dependencies = [
- "base64-simd",
- "thiserror 2.0.20",
-]
-
 [[package]]
 name = "s3s-sigv2"
 version = "0.16.0-alpha.1"
-source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
+source = "git+https://github.com/rustfs/s3s.git?rev=9c4690d8e73fc8d184031a19b2c4539ebc77d180#9c4690d8e73fc8d184031a19b2c4539ebc77d180"
 dependencies = [
- "base64-simd",
- "hmac 0.13.0",
  "jiff",
- "sha1 0.11.0",
- "smallvec",
  "thiserror 2.0.20",
 ]
 
 [[package]]
 name = "s3s-sigv4"
 version = "0.16.0-alpha.1"
-source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
+source = "git+https://github.com/rustfs/s3s.git?rev=9c4690d8e73fc8d184031a19b2c4539ebc77d180#9c4690d8e73fc8d184031a19b2c4539ebc77d180"
 dependencies = [
  "arrayvec",
  "base64-simd",
  "hex-simd",
- "hmac 0.13.0",
  "jiff",
  "nom 8.0.0",
  "serde",
- "sha2 0.11.0",
  "smallvec",
- "std-next",
  "thiserror 2.0.20",
- "zeroize",
 ]
 
 [[package]]
@@ -11786,9 +11749,9 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.16.0"
+version = "1.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f"
+checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
 dependencies = [
  "serde",
 ]
@@ -12068,9 +12031,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "suppaftp"
-version = "11.0.0"
+version = "10.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46c5095831abc0d7944a2d50d6ec6abcd75b9d165d9377deb3e45798cae2343a"
+checksum = "821001051ea3d12a60fb790b8c7cb9a6f5f8698dcfdca4cd533a025fefb0b5b8"
 dependencies = [
  "async-trait",
  "chrono",
@@ -12261,10 +12224,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.4.3",
+ "getrandom 0.3.4",
  "once_cell",
  "rustix",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -12757,9 +12720,9 @@ dependencies = [
 
 [[package]]
 name = "tower-http"
-version = "0.7.1"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c"
+checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233"
 dependencies = [
  "async-compression",
  "bitflags 2.13.1",
@@ -13383,7 +13346,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]

Cargo.toml

@@ -29,7 +29,6 @@ members = [
     "crates/heal-contracts", # Heal request/response channel contracts
     "crates/iam", # Identity and Access Management
     "crates/keystone", # OpenStack Keystone integration
-    "crates/license", # License and entitlement provider contracts
     "crates/lifecycle", # Lifecycle rule evaluation contracts
     "crates/kms", # Key Management Service
     "crates/lock", # Distributed locking implementation
@@ -72,8 +71,8 @@ resolver = "3"
 edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
-rust-version = "1.98.0"
-version = "1.0.0-rc.5"
+rust-version = "1.97.1"
+version = "1.0.0-rc.4"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
 keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -90,61 +89,60 @@ redundant_clone = "warn"
 
 [workspace.dependencies]
 # RustFS Internal Crates
-rustfs = { path = "./rustfs", version = "1.0.0-rc.5" }
-rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.5" }
-rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.5" }
-rustfs-scanner-contracts = { path = "crates/scanner-contracts", version = "1.0.0-rc.5" }
-rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.5" }
-rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.5" }
-rustfs-common = { path = "crates/common", version = "1.0.0-rc.5" }
-rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.5" }
-rustfs-config = { path = "./crates/config", version = "1.0.0-rc.5" }
-rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.5" }
-rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.5" }
-rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.5" }
-rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.5" }
-rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.5" }
-rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.5" }
-rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.5" }
-rustfs-license = { path = "crates/license", version = "1.0.0-rc.5" }
-rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.5" }
-rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.5" }
-rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.5" }
-rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.5" }
-rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.5" }
-rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.5" }
-rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.5" }
-rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.5" }
-rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.5", default-features = false }
-rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.5" }
-rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.5" }
-rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.5" }
-rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.5" }
-rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.5" }
-rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.5" }
-rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.5" }
-rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.5" }
-rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.5" }
-rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.5" }
-rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.5" }
-rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.5" }
-rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.5" }
-rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.5" }
-rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.5" }
-rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.5" }
-rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.5" }
-rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.5" }
-rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.5" }
-rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.5" }
-rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.5" }
-rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.5" }
-rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.5" }
-rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" }
+rustfs = { path = "./rustfs", version = "1.0.0-rc.4" }
+rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.4" }
+rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.4" }
+rustfs-scanner-contracts = { path = "crates/scanner-contracts", version = "1.0.0-rc.4" }
+rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.4" }
+rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.4" }
+rustfs-common = { path = "crates/common", version = "1.0.0-rc.4" }
+rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.4" }
+rustfs-config = { path = "./crates/config", version = "1.0.0-rc.4" }
+rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.4" }
+rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.4" }
+rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.4" }
+rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.4" }
+rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.4" }
+rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.4" }
+rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.4" }
+rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.4" }
+rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.4" }
+rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.4" }
+rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.4" }
+rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.4" }
+rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.4" }
+rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.4" }
+rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.4" }
+rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.4", default-features = false }
+rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.4" }
+rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.4" }
+rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.4" }
+rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.4" }
+rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.4" }
+rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.4" }
+rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.4" }
+rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.4" }
+rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.4" }
+rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.4" }
+rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.4" }
+rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.4" }
+rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.4" }
+rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.4" }
+rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.4" }
+rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.4" }
+rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.4" }
+rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.4" }
+rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.4" }
+rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.4" }
+rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.4" }
+rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.4" }
+rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.4" }
+rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.4" }
 
 # Async Runtime and Networking
 async-channel = "2.5.0"
 async_zip = { default-features = false, version = "0.0.19" }
-mysql_async = { default-features = false, version = "0.37.1" }
+mysql_async = { default-features = false, version = "0.37" }
 async-compression = { version = "0.4.43" }
 async-recursion = "1.1.1"
 async-trait = "0.1.92"
@@ -176,7 +174,7 @@ tonic = { version = "0.14.6" }
 tonic-prost = { version = "0.14.6" }
 tonic-prost-build = { version = "0.14.6" }
 tower = { version = "0.5.3" }
-tower-http = { version = "0.7.1" }
+tower-http = { version = "0.7.0" }
 
 # Serialization and Data Formats
 apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
@@ -234,8 +232,7 @@ tokio-postgres-rustls = "0.14.0"
 # Utilities and Tools
 anyhow = "1.0.104"
 arc-swap = "1.9.2"
-# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until every parser hardening used by Snowball is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published release includes extension, physical-entry, and sparse limits, cancellation-safe sparse parsing, and error-fused entry streams.
-astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
+astral-tokio-tar = "0.7.0"
 atoi = "3.1.0"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.11.0" }
@@ -285,7 +282,7 @@ mime_guess = "2.0.5"
 moka = { version = "0.12.16" }
 netif = "0.1.6"
 num_cpus = { version = "1.17.0" }
-nvml-wrapper = "0.13.0"
+nvml-wrapper = "0.12.1"
 parking_lot = "0.12.5"
 path-absolutize = "4.0.1"
 percent-encoding = "2.3.2"
@@ -307,11 +304,11 @@ rustify = { version = "0.7", default-features = false }
 rustix = { version = "1.1.4" }
 rust-embed = { version = "8.12.0" }
 rustc-hash = { version = "2.1.3" }
-s3s = { git = "https://github.com/rustfs/s3s.git", rev = "bdcb6259339c41369f9f1c60e3a42b5ab8da607b", version = "0.15.0", features = ["minio"] }
+s3s = { git = "https://github.com/rustfs/s3s.git", rev = "9c4690d8e73fc8d184031a19b2c4539ebc77d180", version = "0.15.0", features = ["minio"] }
 serial_test = "4.0.1"
 shadow-rs = { default-features = false, version = "2.0.0" }
 siphasher = "1.0.3"
-smallvec = { version = "1.16.0" }
+smallvec = { version = "1.15.2" }
 compact_str = "0.10.0"
 snap = "1.1.2"
 starshard = { version = "2.3.0" }
@@ -357,7 +354,7 @@ pyroscope = { version = "2.1.1" }
 # FTP and SFTP
 libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
-suppaftp = { version = "11.0.0" }
+suppaftp = { version = "10.0.2" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
 russh = { version = "0.63.1" }
 russh-sftp = "2.4.0"

Makefile

@@ -23,12 +23,6 @@ SHELL := $(shell which bash)
 .SHELLFLAGS = -eu -o pipefail -c
 
 DOCKER_CLI ?= docker
-# Python interpreter for the repository's helper scripts. They import tomllib
-# (Python 3.11+), while macOS still ships /usr/bin/python3 at 3.9, so calls go
-# through a resolver that picks a new-enough interpreter (or falls back to uv).
-# Override with RUSTFS_PYTHON=/path/to/python3.12, or replace the resolver via
-# RUSTFS_PYTHON_BIN=<command>.
-RUSTFS_PYTHON_BIN ?= ./scripts/python_bin.sh
 IMAGE_NAME ?= rustfs:v1.0.0
 CONTAINER_NAME ?= rustfs-dev
 # Docker build configurations

README.md

@@ -48,33 +48,16 @@ Unlike other storage systems, RustFS is released under the permissible Apache 2.
 - **Open Source**: Licensed under Apache 2.0, encouraging unrestricted community contributions and commercial usage.
 - **User-Friendly**: Designed with simplicity in mind for easy deployment and management.
 
-Status legend: ✅ Available — shipped and covered by CI gates; 🧪 Preview — shipped behind an opt-in flag or with a bounded compatibility claim.
-
-| Feature                          | Status       | Feature                            | Status       |
-| :------------------------------- | :----------- | :--------------------------------- | :----------- |
-| **S3 Core Features**             | ✅ Available | **Distributed Mode**               | ✅ Available |
-| **Upload / Download**            | ✅ Available | **Single Node Mode**               | ✅ Available |
-| **Versioning**                   | ✅ Available | **Bitrot Protection**              | ✅ Available |
-| **Object Lock (WORM)**           | ✅ Available | **Healing & Scanner**              | ✅ Available |
-| **Server-Side Encryption**       | ✅ Available | **Pool Expansion / Decommission**  | ✅ Available |
-| **RustFS KMS**                   | ✅ Available | **Bucket Replication**             | ✅ Available |
-| **Lifecycle Management (ILM)**   | ✅ Available | **Site Replication**               | ✅ Available |
-| **ILM Tiering (Remote S3)**      | ✅ Available | **Bucket Quota**                   | ✅ Available |
-| **S3 Select**                    | ✅ Available | **Event Notifications**            | ✅ Available |
-| **S3 Tables (Iceberg REST)**     | 🧪 Preview   | **Audit Logging**                  | ✅ Available |
-| **IAM / Policies**               | ✅ Available | **Logging & Observability**        | ✅ Available |
-| **OIDC / SSO**                   | ✅ Available | **Web Console**                    | ✅ Available |
-| **Keystone Auth**                | ✅ Available | **K8s Helm Charts**                | ✅ Available |
-| **Swift API**                    | ✅ Available | **FTPS / WebDAV**                  | ✅ Available |
-| **Multi-Tenancy**                | ✅ Available | **SFTP**                           | ✅ Available |
-| **MinIO On-Disk Compatibility**  | 🧪 Preview   |                                    |              |
-
-Notes:
-
-- **RustFS KMS**: Vault (KV2 / Transit) and AWS KMS backends are supported for production. The `Local` and `Static` backends are for development and testing only. See [KMS backend security properties](docs/operations/kms-backend-security.md).
-- **Swift API / SFTP**: opt-in cargo features (`--features swift`, `--features sftp`, or `full`). FTPS and WebDAV are enabled in the default build.
-- **S3 Tables**: ships as an Iceberg REST Catalog with automated PyIceberg and DuckDB coverage; other engines and vendor profiles carry bounded claims listed in the [S3 Tables support matrix](docs/architecture/s3-tables-support-matrix.md).
-- **MinIO On-Disk Compatibility**: gated behind the `rio-v2` feature and not part of the default build. Objects MinIO encrypted are not readable by RustFS. See [MinIO file-format interoperability](docs/architecture/minio-file-format-compat.md).
+| Feature                 | Status       | Feature                  | Status           |
+| :---------------------- | :----------- | :----------------------- | :--------------- |
+| **S3 Core Features**    | ✅ Available | **Bitrot Protection**    | ✅ Available     |
+| **Upload / Download**   | ✅ Available | **Single Node Mode**     | ✅ Available     |
+| **Versioning**          | ✅ Available | **Bucket Replication**   | ✅ Available     |
+| **Logging**             | ✅ Available | **Lifecycle Management** | 🚧 Under Testing |
+| **Event Notifications** | ✅ Available | **Distributed Mode**     | 🚧 Under Testing |
+| **K8s Helm Charts**     | ✅ Available | **RustFS KMS**           | 🚧 Under Testing |
+| **Keystone Auth**       | ✅ Available | **Multi-Tenancy**        | ✅ Available     |
+| **Swift API**           | ✅ Available | **Swift Metadata Ops**   | 🚧 Partial       |
 
 ## RustFS vs MinIO Performance
 
@@ -132,7 +115,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
 # Using specific version
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.4
 ```
 
 If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command

README_ZH.md

@@ -112,7 +112,7 @@ chown -R 10001:10001 data logs
 docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
 
 # 使用指定版本运行
-docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
+docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.4
 ```
 
 如果您通过绑定挂载启用 TLS 证书目录，也请用同样方式准备该目录：

crates/config/src/constants/capacity.rs

@@ -59,20 +59,20 @@ pub const ENV_CAPACITY_MAX_TIMEOUT: &str = "RUSTFS_CAPACITY_MAX_TIMEOUT";
 // ============================================================================
 
 /// Scheduled update interval in seconds
-/// Default: 600 seconds (10 minutes)
-pub const DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS: u64 = 600;
+/// Default: 120 seconds (2 minutes)
+pub const DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS: u64 = 120;
 
 /// Write trigger delay in seconds
-/// Default: 30 seconds
-pub const DEFAULT_WRITE_TRIGGER_DELAY_SECS: u64 = 30;
+/// Default: 5 seconds
+pub const DEFAULT_WRITE_TRIGGER_DELAY_SECS: u64 = 5;
 
 /// Write frequency threshold (writes per minute)
-/// Default: 20 writes/minute
-pub const DEFAULT_WRITE_FREQUENCY_THRESHOLD: usize = 20;
+/// Default: 5 writes/minute
+pub const DEFAULT_WRITE_FREQUENCY_THRESHOLD: usize = 5;
 
 /// Fast update threshold in seconds
-/// Default: 120 seconds
-pub const DEFAULT_FAST_UPDATE_THRESHOLD_SECS: u64 = 120;
+/// Default: 30 seconds
+pub const DEFAULT_FAST_UPDATE_THRESHOLD_SECS: u64 = 30;
 
 /// Maximum files threshold for sampling
 /// Default: 200,000 files
@@ -129,16 +129,4 @@ mod tests {
         assert_eq!(ENV_CAPACITY_MIN_TIMEOUT, "RUSTFS_CAPACITY_MIN_TIMEOUT");
         assert_eq!(ENV_CAPACITY_MAX_TIMEOUT, "RUSTFS_CAPACITY_MAX_TIMEOUT");
     }
-
-    #[test]
-    fn test_capacity_default_values() {
-        assert_eq!(DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS, 600);
-        assert_eq!(DEFAULT_WRITE_TRIGGER_DELAY_SECS, 30);
-        assert_eq!(DEFAULT_WRITE_FREQUENCY_THRESHOLD, 20);
-        assert_eq!(DEFAULT_FAST_UPDATE_THRESHOLD_SECS, 120);
-        assert_eq!(DEFAULT_MAX_FILES_THRESHOLD, 200_000);
-        assert_eq!(DEFAULT_STAT_TIMEOUT_SECS, 3);
-        assert_eq!(DEFAULT_SAMPLE_RATE, 200);
-        assert_eq!(DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, 600);
-    }
 }

crates/config/src/constants/scanner.rs

@@ -198,11 +198,11 @@ pub const ENV_SCANNER_IDLE_MODE: &str = "RUSTFS_SCANNER_IDLE_MODE";
 /// Environment variable that controls scanner cache save timeout in seconds.
 /// The scanner enforces a minimum value of `1`.
 /// - Unit: seconds (u64).
-/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=14`
+/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=30`
 pub const ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS: &str = "RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS";
 
 /// Default scanner cache save timeout in seconds.
-pub const DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS: u64 = 14;
+pub const DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS: u64 = 30;
 
 /// Environment variable that caps concurrent scanner set tasks.
 /// A value of `0` keeps the existing topology-based concurrency.

crates/e2e_test/Cargo.toml

@@ -100,8 +100,7 @@ aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a",
 aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
 aws-config = { workspace = true }
 aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
-aws-smithy-types.workspace = true
-async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
+async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] }
 async-trait = { workspace = true }
 flate2.workspace = true
 http.workspace = true
@@ -115,7 +114,6 @@ rustfs-signer.workspace = true
 # server's implementation: a shared helper could agree with a bug on both sides.
 data-encoding = { workspace = true }
 hmac = { workspace = true }
-minlz.workspace = true
 sha1 = { workspace = true }
 serde_urlencoded = { workspace = true }
 tracing = { workspace = true }

crates/e2e_test/README.md

@@ -169,7 +169,7 @@ the same profile for membership and execution with one nightly worker.
 | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
 | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
-| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
+| Direct upgrade from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
@@ -233,8 +233,8 @@ spawn error. Install the pinned CI version before running their profiles.
   [`src/policy/README.md`](src/policy/README.md),
   [`src/protocols/README.md`](src/protocols/README.md),
   [`src/reliant/README.md`](src/reliant/README.md)
-- Per-module counts: `cargo nextest list -p e2e_test --profile <profile>`
-  (one-liner in [`docs/testing/README.md`](../../docs/testing/README.md))
+- Authoritative per-module counts:
+  [`docs/testing/e2e-suite-inventory.md`](../../docs/testing/e2e-suite-inventory.md)
 - Test pyramid & flake policy: [`docs/testing/README.md`](../../docs/testing/README.md)
 
 ## CI smoke subset (`--profile e2e-smoke`)
@@ -271,12 +271,12 @@ Note on `#[serial]`: nextest runs each test in its own process, so
 parallel-safe by construction (random port + isolated temp dir), which the
 current subset is.
 
-### Test inventory
+### Authoritative test inventory
 
-Per-module counts are not committed; list them with
-`cargo nextest list -p e2e_test --profile <profile>` (the result is
-platform-dependent because some modules are linux-only; the `jq` one-liner is
-in `docs/testing/README.md`). When a profile membership change is
+`docs/testing/e2e-suite-inventory.md` records the per-module test counts as
+listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or
+moving e2e tests so acceptance numbers in the test-strategy issues
+(backlog#1147–#1155) stay auditable. When a profile membership change is
 intentional, review its JSON listing before updating the matching
 `.config/e2e-*-selection.txt` test-ID digest. Update only the platform that
 produced the listing:

crates/e2e_test/src/checksum_upload_test.rs

@@ -22,7 +22,7 @@ mod tests {
     use aws_sdk_s3::config::{Credentials, Region, RequestChecksumCalculation};
     use aws_sdk_s3::error::ProvideErrorMetadata;
     use aws_sdk_s3::primitives::ByteStream;
-    use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart, ServerSideEncryption};
+    use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
     use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
     use md5::{Digest as Md5Digest, Md5};
     use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
@@ -260,117 +260,6 @@ mod tests {
         info!("PASSED: HeadObject returns stored SHA256 digest");
     }
 
-    #[tokio::test]
-    async fn test_head_object_returns_sse_s3_checksum() {
-        init_logging();
-
-        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
-        env.start_rustfs_server_with_env(
-            vec![],
-            &[
-                ("RUSTFS_SSE_S3_MASTER_KEY", "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI="),
-                ("RUSTFS_CONSOLE_ENABLE", "false"),
-            ],
-        )
-        .await
-        .expect("Failed to start RustFS");
-
-        let client = create_s3_client(&env);
-        let bucket = "test-sse-s3-checksum-head";
-        create_bucket(&client, bucket).await.expect("Failed to create bucket");
-
-        let put = client
-            .put_object()
-            .bucket(bucket)
-            .key("encrypted.txt")
-            .body(ByteStream::from_static(b"encrypted checksum"))
-            .server_side_encryption(ServerSideEncryption::Aes256)
-            .checksum_algorithm(ChecksumAlgorithm::Crc32)
-            .send()
-            .await
-            .expect("SSE-S3 PutObject with CRC32 failed");
-        let expected = put.checksum_crc32().expect("PutObject must return CRC32");
-
-        let head = client
-            .head_object()
-            .bucket(bucket)
-            .key("encrypted.txt")
-            .checksum_mode(ChecksumMode::Enabled)
-            .send()
-            .await
-            .expect("SSE-S3 HeadObject failed");
-
-        assert_eq!(head.checksum_crc32(), Some(expected));
-
-        client
-            .copy_object()
-            .bucket(bucket)
-            .key("encrypted-copy.txt")
-            .copy_source(format!("{bucket}/encrypted.txt"))
-            .server_side_encryption(ServerSideEncryption::Aes256)
-            .send()
-            .await
-            .expect("SSE-S3 CopyObject failed");
-        let copy_head = client
-            .head_object()
-            .bucket(bucket)
-            .key("encrypted-copy.txt")
-            .checksum_mode(ChecksumMode::Enabled)
-            .send()
-            .await
-            .expect("SSE-S3 copied HeadObject failed");
-
-        assert_eq!(copy_head.checksum_crc32(), Some(expected));
-
-        let multipart_key = "encrypted-multipart.txt";
-        let create = client
-            .create_multipart_upload()
-            .bucket(bucket)
-            .key(multipart_key)
-            .server_side_encryption(ServerSideEncryption::Aes256)
-            .checksum_algorithm(ChecksumAlgorithm::Crc32)
-            .send()
-            .await
-            .expect("SSE-S3 CreateMultipartUpload with CRC32 failed");
-        let upload_id = create.upload_id().expect("CreateMultipartUpload must return an upload ID");
-        let part = client
-            .upload_part()
-            .bucket(bucket)
-            .key(multipart_key)
-            .upload_id(upload_id)
-            .part_number(1)
-            .body(ByteStream::from_static(b"encrypted multipart checksum"))
-            .checksum_algorithm(ChecksumAlgorithm::Crc32)
-            .send()
-            .await
-            .expect("SSE-S3 UploadPart with CRC32 failed");
-        let completed_part = CompletedPart::builder()
-            .part_number(1)
-            .e_tag(part.e_tag().expect("UploadPart must return an ETag"))
-            .checksum_crc32(part.checksum_crc32().expect("UploadPart must return CRC32"))
-            .build();
-        let complete = client
-            .complete_multipart_upload()
-            .bucket(bucket)
-            .key(multipart_key)
-            .upload_id(upload_id)
-            .multipart_upload(CompletedMultipartUpload::builder().parts(completed_part).build())
-            .send()
-            .await
-            .expect("SSE-S3 CompleteMultipartUpload with CRC32 failed");
-        let expected_multipart = complete.checksum_crc32().expect("CompleteMultipartUpload must return CRC32");
-        let multipart_head = client
-            .head_object()
-            .bucket(bucket)
-            .key(multipart_key)
-            .checksum_mode(ChecksumMode::Enabled)
-            .send()
-            .await
-            .expect("SSE-S3 multipart HeadObject failed");
-
-        assert_eq!(multipart_head.checksum_crc32(), Some(expected_multipart));
-    }
-
     /// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content.
     /// Uses part size >= 5MB (server minimum) for two parts.
     #[tokio::test]

crates/e2e_test/src/cluster_multidrive_pool_test.rs

@@ -27,10 +27,8 @@
 //! Readiness is established by the harness's `start()` handshake (TCP reachability
 //! plus an S3 `ListBuckets` poll) — there are no fixed sleeps.
 //!
-//! The volume-proxy smoke below also proves that the socket-level fault proxy
-//! can be installed before startup without changing the client-facing node URL.
-//! A full lock-plane partition matrix and 5GiB large-object budget remain
-//! tracked separately.
+//! Out of scope for this block (tracked separately): network fault injection
+//! (toxiproxy / socket proxy) and 5GiB large-object budgets.
 
 use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
 
@@ -78,28 +76,6 @@ async fn cluster_multidrive_single_pool_smoke() -> TestResult {
     Ok(())
 }
 
-/// 4 nodes x 4 drives, single pool: exercise the maximum local erasure layout
-/// supported by the cluster harness. This remains in the nightly lane because
-/// it starts four real server processes and sixteen data directories.
-#[tokio::test]
-async fn cluster_four_node_four_drive_single_pool_smoke() -> TestResult {
-    crate::common::init_logging();
-
-    let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, 4)).await?;
-
-    let volumes = cluster.rustfs_volumes_arg();
-    assert_eq!(volumes.split(' ').count(), 16, "expected 16 explicit endpoints, got: {volumes}");
-    assert!(!volumes.contains('{'), "single-pool layout must not use ellipses: {volumes}");
-    assert!(cluster.nodes.iter().all(|node| node.data_dirs.len() == 4));
-
-    cluster.start().await?;
-    cluster.create_test_bucket(BUCKET).await?;
-
-    let payload = vec![0x3Cu8; 1024 * 1024];
-    put_get_roundtrip(&cluster, "multidrive-4/object", &payload).await?;
-    Ok(())
-}
-
 /// Two single-node pools, 2 drives each: the multi-pool layout boots and
 /// round-trips. Every pool is a distinct erasure pool (`pool_idx` 0 and 1).
 #[tokio::test]
@@ -127,27 +103,3 @@ async fn cluster_two_pool_smoke() -> TestResult {
     put_get_roundtrip(&cluster, "twopool/object", &payload).await?;
     Ok(())
 }
-
-/// A real cluster smoke for the volume FaultProxy wiring. The proxy target is
-/// not listening yet when it is created; cluster startup must still converge
-/// once the target node starts, and peer disk/RPC traffic must traverse it.
-#[tokio::test]
-async fn cluster_volume_fault_proxy_pass_smoke() -> TestResult {
-    crate::common::init_logging();
-
-    let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(2, 2)).await?;
-    let proxy = cluster.start_volume_proxy_for_node(0).await?;
-    let proxied = proxy.local_addr().to_string();
-    assert!(cluster.rustfs_volumes_arg().contains(&proxied));
-
-    let result: TestResult = async {
-        cluster.start().await?;
-        cluster.create_test_bucket(BUCKET).await?;
-        let payload = vec![0x6Du8; 256 * 1024];
-        put_get_roundtrip(&cluster, "volume-proxy/object", &payload).await
-    }
-    .await;
-
-    proxy.shutdown().await;
-    result
-}

crates/e2e_test/src/common.rs

@@ -34,7 +34,6 @@ use serde_json;
 use std::ffi::OsStr;
 use std::fs as stdfs;
 use std::io::ErrorKind;
-use std::net::SocketAddr;
 use std::path::{Path, PathBuf};
 use std::process::{Child, Command, Stdio};
 use std::sync::Once;
@@ -1215,9 +1214,6 @@ pub struct RustFSTestClusterEnvironment {
     pub node_extra_env: Vec<Vec<(String, String)>>,
     pub node_capture_log_paths: Vec<Option<String>>,
     pub topology: ClusterTopology,
-    /// Optional socket proxies used for the corresponding node's volume
-    /// endpoints. Proxies must be installed before [`Self::start`].
-    volume_proxy_addresses: Vec<Option<SocketAddr>>,
 }
 
 impl RustFSTestClusterEnvironment {
@@ -1309,7 +1305,6 @@ impl RustFSTestClusterEnvironment {
             extra_env.push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string()));
         }
 
-        let node_count = topology.node_count;
         Ok(Self {
             nodes,
             temp_dir,
@@ -1319,7 +1314,6 @@ impl RustFSTestClusterEnvironment {
             node_extra_env: vec![Vec::new(); topology.node_count],
             node_capture_log_paths: vec![None; topology.node_count],
             topology,
-            volume_proxy_addresses: vec![None; node_count],
         })
     }
 
@@ -1387,34 +1381,6 @@ impl RustFSTestClusterEnvironment {
         self.build_volumes_arg()
     }
 
-    /// Start a socket proxy for one node's volume endpoints and route all
-    /// subsequent `RUSTFS_VOLUMES` references for that node through it.
-    ///
-    /// Call this before [`Self::start`], then use the returned proxy's
-    /// [`crate::fault_proxy::FaultProxy::set_mode`] to inject latency,
-    /// blackhole, or one-way partition faults. The node's own listen address
-    /// remains direct, so S3 clients can still reach it while peer disk/RPC
-    /// traffic is steered through the proxy.
-    pub async fn start_volume_proxy_for_node(
-        &mut self,
-        node_idx: usize,
-    ) -> Result<crate::fault_proxy::FaultProxy, Box<dyn std::error::Error + Send + Sync>> {
-        self.ensure_node_index(node_idx)?;
-        if self.volume_proxy_addresses[node_idx].is_some() {
-            return Err(format!("a volume proxy is already configured for node {node_idx}").into());
-        }
-        let target = self.nodes[node_idx].address.parse::<SocketAddr>()?;
-        let proxy = crate::fault_proxy::FaultProxy::start(target).await?;
-        self.volume_proxy_addresses[node_idx] = Some(proxy.local_addr());
-        Ok(proxy)
-    }
-
-    fn volume_address(&self, node_idx: usize) -> String {
-        self.volume_proxy_addresses[node_idx]
-            .map(|address| address.to_string())
-            .unwrap_or_else(|| self.nodes[node_idx].address.clone())
-    }
-
     fn build_volumes_arg(&self) -> String {
         let pools = self.topology.normalized_pools();
 
@@ -1423,11 +1389,7 @@ impl RustFSTestClusterEnvironment {
             return self
                 .nodes
                 .iter()
-                .enumerate()
-                .flat_map(|(node_idx, n)| {
-                    let address = self.volume_address(node_idx);
-                    n.data_dirs.iter().map(move |dir| format!("http://{}{}", address, dir))
-                })
+                .flat_map(|n| n.data_dirs.iter().map(move |dir| format!("http://{}{}", n.address, dir)))
                 .collect::<Vec<_>>()
                 .join(" ");
         }
@@ -1438,19 +1400,13 @@ impl RustFSTestClusterEnvironment {
         pools
             .iter()
             .map(|nodes| {
-                let node_idx = nodes[0];
-                let node = &self.nodes[node_idx];
+                let node = &self.nodes[nodes[0]];
                 let base = node
                     .data_dirs
                     .first()
                     .and_then(|d| d.rsplit_once('/').map(|(parent, _)| parent))
                     .unwrap_or(&node.data_dir);
-                format!(
-                    "http://{}{}/drive{{0...{}}}",
-                    self.volume_address(node_idx),
-                    base,
-                    self.topology.drives_per_node - 1
-                )
+                format!("http://{}{}/drive{{0...{}}}", node.address, base, self.topology.drives_per_node - 1)
             })
             .collect::<Vec<_>>()
             .join(" ")
@@ -1469,18 +1425,31 @@ impl RustFSTestClusterEnvironment {
     ///   times out, or cluster service readiness times out.
     pub async fn start(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let binary_path = rustfs_binary_path();
-        self.start_with_binary(&binary_path).await
-    }
-
-    /// Start every cluster node with a specific RustFS binary.
-    ///
-    /// Upgrade compatibility tests use this to initialize a cluster with a
-    /// pinned previous release before replacing nodes with the workspace build.
-    pub async fn start_with_binary(&mut self, binary_path: &Path) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let volumes_arg = self.build_volumes_arg();
 
-        for node_idx in 0..self.nodes.len() {
-            self.spawn_node(node_idx, binary_path, &volumes_arg)?;
+        for (i, node) in self.nodes.iter_mut().enumerate() {
+            info!("Starting cluster node {} on {}", i, node.address);
+
+            let mut command = Command::new(&binary_path);
+            command
+                .env("RUSTFS_VOLUMES", &volumes_arg)
+                .env("RUSTFS_ADDRESS", &node.address)
+                .env("RUSTFS_ACCESS_KEY", &self.access_key)
+                .env("RUSTFS_SECRET_KEY", &self.secret_key)
+                .env("RUSTFS_CONSOLE_ENABLE", "false")
+                .env("RUST_LOG", "rustfs=info,rustfs_notify=debug");
+
+            for (key, value) in &self.extra_env {
+                command.env(key, value);
+            }
+            for (key, value) in &self.node_extra_env[i] {
+                command.env(key, value);
+            }
+            capture_command_logs(&mut command, self.node_capture_log_paths[i].as_deref())?;
+
+            let process = command.current_dir(&node.data_dir).spawn()?;
+
+            node.process = Some(process);
         }
 
         for (i, node) in self.nodes.iter().enumerate() {
@@ -1496,46 +1465,20 @@ impl RustFSTestClusterEnvironment {
 
     /// Start one node process using the cluster's existing volume layout.
     pub async fn start_node(&mut self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        let binary_path = rustfs_binary_path();
-        self.start_node_from_binary(node_idx, &binary_path).await
-    }
-
-    /// Start one stopped cluster node with a specific RustFS binary while
-    /// preserving the cluster's volume layout and that node's data directory.
-    pub async fn start_node_from_binary(
-        &mut self,
-        node_idx: usize,
-        binary_path: &Path,
-    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        let volumes_arg = self.build_volumes_arg();
-        self.spawn_node(node_idx, binary_path, &volumes_arg)?;
-
-        self.wait_for_node_ready(&self.nodes[node_idx].address, node_idx).await?;
-        self.wait_for_node_service_ready(node_idx).await?;
-        Ok(())
-    }
-
-    fn spawn_node(
-        &mut self,
-        node_idx: usize,
-        binary_path: &Path,
-        volumes_arg: &str,
-    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         self.ensure_node_index(node_idx)?;
         if self.nodes[node_idx].process.is_some() {
             return Err(format!("cluster node {node_idx} is already running").into());
         }
-        if !binary_path.is_file() {
-            return Err(format!("RustFS binary does not exist: {}", binary_path.display()).into());
-        }
 
+        let binary_path = rustfs_binary_path();
+        let volumes_arg = self.build_volumes_arg();
         let log_path = self.node_capture_log_paths[node_idx].clone();
         let node = &mut self.nodes[node_idx];
-        info!("Starting cluster node {} on {} with {}", node_idx, node.address, binary_path.display());
+        info!("Starting cluster node {} on {}", node_idx, node.address);
 
-        let mut command = Command::new(binary_path);
+        let mut command = Command::new(&binary_path);
         command
-            .env("RUSTFS_VOLUMES", volumes_arg)
+            .env("RUSTFS_VOLUMES", &volumes_arg)
             .env("RUSTFS_ADDRESS", &node.address)
             .env("RUSTFS_ACCESS_KEY", &self.access_key)
             .env("RUSTFS_SECRET_KEY", &self.secret_key)
@@ -1552,6 +1495,9 @@ impl RustFSTestClusterEnvironment {
 
         let process = command.current_dir(&node.data_dir).spawn()?;
         node.process = Some(process);
+
+        self.wait_for_node_ready(&self.nodes[node_idx].address, node_idx).await?;
+        self.wait_for_node_service_ready(node_idx).await?;
         Ok(())
     }
 
@@ -1699,51 +1645,6 @@ impl RustFSTestClusterEnvironment {
         process.wait()?;
         Ok(())
     }
-
-    /// Gracefully stop one cluster node and wait for its process to exit.
-    ///
-    /// This is intentionally separate from [`Self::stop_node`]: the latter is
-    /// a hard kill used by crash-recovery tests, while this path lets RustFS
-    /// complete its normal shutdown hooks before a test restarts the node.
-    pub async fn stop_node_gracefully(&mut self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        self.ensure_node_index(node_idx)?;
-
-        #[cfg(unix)]
-        {
-            let Some(process) = self.nodes[node_idx].process.as_ref() else {
-                return Ok(());
-            };
-            let pid = process.id().to_string();
-            let signal_status = Command::new("kill").args(["-TERM", &pid]).status()?;
-            if !signal_status.success() {
-                return Err(format!("failed to send SIGTERM to cluster node {node_idx} (pid {pid})").into());
-            }
-
-            let mut process = self.nodes[node_idx]
-                .process
-                .take()
-                .ok_or_else(|| format!("cluster node {node_idx} process disappeared while stopping"))?;
-            let deadline = std::time::Instant::now() + Duration::from_secs(45);
-            loop {
-                if let Some(status) = process.try_wait()? {
-                    info!("Cluster node {} stopped gracefully with {}", node_idx, status);
-                    return Ok(());
-                }
-                if std::time::Instant::now() >= deadline {
-                    let _ = process.kill();
-                    let _ = process.wait();
-                    return Err(format!("cluster node {node_idx} did not stop gracefully within 45 seconds").into());
-                }
-                sleep(Duration::from_millis(100)).await;
-            }
-        }
-
-        #[cfg(not(unix))]
-        {
-            let _ = node_idx;
-            Err("graceful cluster-node stop is only supported on Unix E2E hosts".into())
-        }
-    }
 }
 
 impl Drop for RustFSTestClusterEnvironment {
@@ -2099,7 +2000,7 @@ mod tests {
         }
         let multidrive = topology.drives_per_node > 1;
 
-        let nodes: Vec<ClusterNode> = (0..topology.node_count)
+        let nodes = (0..topology.node_count)
             .map(|i| {
                 let address = format!("127.0.0.1:{}", 9000 + i);
                 let data_dirs: Vec<String> = if multidrive {
@@ -2120,7 +2021,6 @@ mod tests {
             })
             .collect();
 
-        let node_count = nodes.len();
         RustFSTestClusterEnvironment {
             nodes,
             temp_dir,
@@ -2130,7 +2030,6 @@ mod tests {
             node_extra_env: vec![Vec::new(); topology.node_count],
             node_capture_log_paths: vec![None; topology.node_count],
             topology,
-            volume_proxy_addresses: vec![None; node_count],
         }
     }
 
@@ -2215,25 +2114,6 @@ mod tests {
         assert!(ClusterTopology::single_pool_multidrive(1, 1).validate().is_ok());
     }
 
-    #[tokio::test]
-    async fn volume_proxy_rewrites_cluster_volume_endpoint() {
-        let mut env = RustFSTestClusterEnvironment::new(1)
-            .await
-            .expect("cluster environment should allocate a node");
-        let direct = env.nodes[0].address.clone();
-        let proxy = env
-            .start_volume_proxy_for_node(0)
-            .await
-            .expect("volume proxy should bind before the target server starts");
-        let proxied = proxy.local_addr().to_string();
-        let volumes = env.rustfs_volumes_arg();
-
-        assert!(volumes.contains(&proxied), "volumes must use the proxy address: {volumes}");
-        assert!(!volumes.contains(&direct), "volumes must not retain the direct address: {volumes}");
-
-        proxy.shutdown().await;
-    }
-
     #[test]
     fn cluster_node_env_supports_per_node_overrides() {
         let mut env = fake_cluster(ClusterTopology::single_pool(4));

crates/e2e_test/src/degraded_listing_availability_test.rs

@@ -1,174 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Regression: an object legally committed at degraded write quorum must stay
-//! listable while a *different* drive is offline.
-//!
-//! On a 4-drive EC 2+2 set, a PUT made while one drive is down persists
-//! `xl.meta` on 3 of 4 drives (write quorum). If a different drive later goes
-//! offline before heal converges, a strict latest-listing quorum of 3 can only
-//! ever observe 2 copies, so ListObjectsV2 silently dropped the object even
-//! though GetObject (read quorum 2) still succeeded. Exposed by the flaky
-//! "Mixed-version rolling upgrade from rc.2" CI lane (run 33478999853); the
-//! product fix relaxes the listing's required object quorum by the number of
-//! set drives the listing could not consult (see
-//! `latest_listing_required_object_quorum` in
-//! `crates/ecstore/src/store/list_objects.rs`).
-
-#[cfg(test)]
-mod tests {
-    use crate::common::{RustFSTestClusterEnvironment, init_logging};
-    use aws_sdk_s3::Client;
-    use bytes::Bytes;
-    use std::collections::HashSet;
-    use std::error::Error;
-    use std::time::{Duration, Instant};
-    use tracing::info;
-
-    type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
-
-    const BUCKET: &str = "degraded-listing-availability";
-    const OBJECT_COUNT: usize = 8;
-    /// Well under the observed heal-convergence gap (~50s in the CI incident),
-    /// so a listing that only completes after heal restores the missing copy
-    /// still fails this deadline on a regressed build.
-    const LISTING_DEADLINE: Duration = Duration::from_secs(25);
-    const GET_RETRY_DEADLINE: Duration = Duration::from_secs(15);
-    const PUT_RETRY_DEADLINE: Duration = Duration::from_secs(15);
-
-    fn object_key(idx: usize) -> String {
-        format!("degraded-object-{idx:02}")
-    }
-
-    async fn list_all_keys(client: &Client) -> Result<HashSet<String>, Box<dyn Error + Send + Sync>> {
-        let mut keys = HashSet::new();
-        let mut continuation_token: Option<String> = None;
-        loop {
-            let response = client
-                .list_objects_v2()
-                .bucket(BUCKET)
-                .set_continuation_token(continuation_token.clone())
-                .send()
-                .await?;
-            keys.extend(
-                response
-                    .contents()
-                    .iter()
-                    .filter_map(|object| object.key().map(str::to_owned)),
-            );
-            match response.next_continuation_token() {
-                Some(token) => continuation_token = Some(token.to_owned()),
-                None => break,
-            }
-        }
-        Ok(keys)
-    }
-
-    /// 4-node single-drive cluster (EC 2+2, write quorum 3):
-    /// 1. Stop node 1 and PUT objects — each commits on nodes {0, 2, 3} only.
-    /// 2. Stop node 3 (a holder drive), then bring node 1 back before heal can
-    ///    recreate the missing copies there.
-    /// 3. Every object still satisfies read quorum (nodes 0 and 2), so GET
-    ///    must succeed AND ListObjectsV2 must report every key well before
-    ///    heal converges.
-    #[tokio::test]
-    async fn degraded_write_remains_listable_while_a_different_drive_is_offline() -> TestResult {
-        init_logging();
-
-        let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
-        // Listing availability must not depend on heal convergence: disable
-        // the background healers so the degraded objects keep their metadata
-        // on exactly 3 of 4 drives for the whole test.
-        cluster.set_env("RUSTFS_HEAL_ENABLED", "false");
-        cluster.set_env("RUSTFS_SCANNER_ENABLED", "false");
-        cluster.start().await?;
-        cluster.create_test_bucket(BUCKET).await?;
-        let client = cluster.create_s3_client(0)?;
-
-        info!("stopping node 1 so the uploads commit at degraded write quorum (3 of 4)");
-        cluster.stop_node(1)?;
-        // The first writes after a node drops can see transient 503s while the
-        // survivors notice the dead peer; retry briefly (overwrites of the same
-        // unversioned key are idempotent).
-        for idx in 0..OBJECT_COUNT {
-            let key = object_key(idx);
-            let body = format!("degraded listing payload {idx}");
-            let deadline = Instant::now() + PUT_RETRY_DEADLINE;
-            loop {
-                let request = client
-                    .put_object()
-                    .bucket(BUCKET)
-                    .key(&key)
-                    .body(Bytes::from(body.clone()).into());
-                match request.send().await {
-                    Ok(_) => break,
-                    Err(error) if Instant::now() < deadline => {
-                        info!("retrying degraded PUT for {key}: {error}");
-                        tokio::time::sleep(Duration::from_millis(500)).await;
-                    }
-                    Err(error) => return Err(format!("degraded PUT for {key} failed: {error}").into()),
-                }
-            }
-        }
-
-        info!("stopping node 3 (holds a copy) and restoring node 1 (holds none)");
-        cluster.stop_node(3)?;
-        cluster.start_node(1).await?;
-
-        // The first requests after a node drops can see transient 503s while
-        // the survivors notice the dead peer; retry briefly before asserting.
-        for idx in 0..OBJECT_COUNT {
-            let key = object_key(idx);
-            let deadline = Instant::now() + GET_RETRY_DEADLINE;
-            let body = loop {
-                match client.get_object().bucket(BUCKET).key(&key).send().await {
-                    Ok(response) => break response.body.collect().await?.into_bytes(),
-                    Err(error) if Instant::now() < deadline => {
-                        info!("retrying degraded GET for {key}: {error}");
-                        tokio::time::sleep(Duration::from_millis(500)).await;
-                    }
-                    Err(error) => return Err(format!("degraded object {key} failed read quorum GET: {error}").into()),
-                }
-            };
-            assert!(!body.is_empty(), "degraded object {key} should read back at read quorum");
-        }
-
-        let expected: HashSet<String> = (0..OBJECT_COUNT).map(object_key).collect();
-        let deadline = Instant::now() + LISTING_DEADLINE;
-        let listed = loop {
-            let listed = match list_all_keys(&client).await {
-                Ok(keys) => keys,
-                Err(error) if Instant::now() < deadline => {
-                    info!("retrying degraded listing: {error}");
-                    tokio::time::sleep(Duration::from_millis(500)).await;
-                    continue;
-                }
-                Err(error) => return Err(error),
-            };
-            if expected.is_subset(&listed) {
-                break listed;
-            }
-            assert!(
-                Instant::now() < deadline,
-                "objects readable at read quorum stayed missing from ListObjectsV2 for {LISTING_DEADLINE:?}: \
-                 missing={:?} listed={listed:?}",
-                expected.difference(&listed).collect::<Vec<_>>(),
-            );
-            tokio::time::sleep(Duration::from_millis(500)).await;
-        };
-        info!(listed = listed.len(), "degraded objects are listable while node 3 is offline");
-
-        Ok(())
-    }
-}

crates/e2e_test/src/kms/multipart_encryption_test.rs

@@ -560,12 +560,18 @@ async fn test_multipart_encryption_type(
         .set_parts(Some(completed_parts))
         .build();
 
-    let complete_request = s3_client
+    let mut complete_request = s3_client
         .complete_multipart_upload()
         .bucket(bucket)
         .key(object_key)
         .upload_id(upload_id)
         .multipart_upload(completed_multipart_upload);
+    if matches!(encryption_type, EncryptionType::SSEC) {
+        complete_request = complete_request
+            .sse_customer_algorithm("AES256")
+            .sse_customer_key(sse_c_key.as_ref().unwrap())
+            .sse_customer_key_md5(sse_c_md5.as_ref().unwrap());
+    }
     let _complete_output = complete_request.send().await?;
 
     // Download and verify

crates/e2e_test/src/lib.rs

@@ -348,11 +348,6 @@ mod delete_regression_test;
 #[cfg(test)]
 mod listing_regression_test;
 
-// Cluster regression: objects committed at degraded write quorum must stay
-// listable while a different drive is offline (CI run 33478999853).
-#[cfg(test)]
-mod degraded_listing_availability_test;
-
 // P1 regression: bucket statistics accuracy (rustfs#5615, #5008, #5116, #5055, #3898, #1012)
 #[cfg(test)]
 mod bucket_stats_regression_test;

crates/e2e_test/src/multipart_auth_test.rs

@@ -15,7 +15,7 @@
 //! Regression coverage for anonymous access on multipart control APIs.
 
 use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
-use async_compression::tokio::write::{BzEncoder, Lz4Encoder, XzEncoder};
+use async_compression::tokio::write::{BzEncoder, XzEncoder};
 use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
 use aws_sdk_s3::operation::head_object::HeadObjectOutput;
 use aws_sdk_s3::primitives::ByteStream;
@@ -23,10 +23,7 @@ use aws_sdk_s3::types::{
     ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
 };
 use chrono::{Duration as ChronoDuration, Utc};
-use flate2::{
-    Compression,
-    write::{GzEncoder, ZlibEncoder},
-};
+use flate2::{Compression, write::GzEncoder};
 use http::HeaderValue;
 use http::header::{CONTENT_TYPE, HOST};
 use md5::{Digest as Md5Digest, Md5};
@@ -190,12 +187,6 @@ fn gzip_bytes(data: &[u8]) -> Vec<u8> {
     encoder.finish().expect("gzip encoder should finish")
 }
 
-fn zlib_bytes(data: &[u8]) -> Vec<u8> {
-    let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
-    encoder.write_all(data).expect("zlib encoder should accept input");
-    encoder.finish().expect("zlib encoder should finish")
-}
-
 fn zstd_bytes(data: &[u8]) -> Vec<u8> {
     let mut encoder = zstd::Encoder::new(Vec::new(), 0).expect("zstd encoder should initialize");
     encoder.write_all(data).expect("zstd encoder should accept input");
@@ -218,45 +209,6 @@ async fn xz_bytes(data: &[u8]) -> Vec<u8> {
     encoder.into_inner().into_inner()
 }
 
-async fn lz4_bytes(data: &[u8]) -> Vec<u8> {
-    let cursor = Cursor::new(Vec::new());
-    let mut encoder = Lz4Encoder::new(cursor);
-    encoder.write_all(data).await.expect("LZ4 encoder should accept input");
-    encoder.shutdown().await.expect("LZ4 encoder should finish");
-    encoder.into_inner().into_inner()
-}
-
-/// Encode the S2 framed stream shape emitted by minio-go PutObjectsSnowball
-/// with `Compress: true`: 1 MiB independent blocks, better compression,
-/// masked CRC-32C, and the `S2sTwO` stream identifier.
-fn minio_go_snowball_s2_bytes(data: &[u8]) -> Vec<u8> {
-    const BLOCK_SIZE: usize = 1 << 20;
-    const CHECKSUM_SIZE: usize = 4;
-
-    let mut output = b"\xff\x06\x00\x00S2sTwO".to_vec();
-    let mut encoder = minlz::Encoder::new();
-    for block in data.chunks(BLOCK_SIZE) {
-        let compressed = encoder.encode_better(block);
-        let compressed_limit = block.len().saturating_sub(block.len() / 32).saturating_sub(5);
-        let (chunk_type, payload) = if compressed.len() <= compressed_limit {
-            (0x00, compressed.as_slice())
-        } else {
-            (0x01, block)
-        };
-        let chunk_len = payload.len() + CHECKSUM_SIZE;
-        assert!(chunk_len < 1 << 24, "S2 fixture chunk must fit the 24-bit frame length");
-        output.extend_from_slice(&[
-            chunk_type,
-            (chunk_len & 0xff) as u8,
-            ((chunk_len >> 8) & 0xff) as u8,
-            ((chunk_len >> 16) & 0xff) as u8,
-        ]);
-        output.extend_from_slice(&minlz::crc::crc(block).to_le_bytes());
-        output.extend_from_slice(payload);
-    }
-    output
-}
-
 fn assert_s3_error_code<T, E>(result: Result<T, SdkError<E>>, code: &str)
 where
     T: std::fmt::Debug,
@@ -3504,62 +3456,6 @@ async fn test_signed_put_object_extract_expands_tar_entries_with_prefix_headers(
     Ok(())
 }
 
-#[tokio::test]
-async fn test_signed_put_object_extract_ignore_dirs_skips_unauthorized_directory()
--> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    init_logging();
-
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-
-    let bucket = "signed-extract-ignore-dirs-auth";
-    let archive_key = "bundle.tar";
-    let allowed_member = "allowed/member.txt";
-    let denied_directory = "denied/";
-    let username = "snowball-ignore-dirs";
-    let secret_key = "snowball-ignore-dirs-secret";
-    let expected_body = b"allowed-body";
-
-    let admin_client = env.create_s3_client();
-    admin_client.create_bucket().bucket(bucket).send().await?;
-    create_restricted_user(&env, username, secret_key).await?;
-
-    let policy = serde_json::json!({
-        "Version": "2012-10-17",
-        "Statement": [{
-            "Effect": "Allow",
-            "Principal": { "AWS": [username] },
-            "Action": ["s3:PutObject"],
-            "Resource": [
-                format!("arn:aws:s3:::{bucket}/{archive_key}"),
-                format!("arn:aws:s3:::{bucket}/{allowed_member}")
-            ]
-        }]
-    })
-    .to_string();
-    admin_client.put_bucket_policy().bucket(bucket).policy(policy).send().await?;
-
-    let restricted_client = restricted_user_client(&env, username, secret_key);
-    let tar_bytes = make_tar(&[(allowed_member, expected_body)], &[denied_directory]).await;
-    restricted_client
-        .put_object()
-        .bucket(bucket)
-        .key(archive_key)
-        .body(ByteStream::from(tar_bytes))
-        .customize()
-        .mutate_request(|req| {
-            req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
-            req.headers_mut().insert("x-amz-meta-snowball-ignore-dirs", "true");
-        })
-        .send()
-        .await?;
-
-    let stored = admin_client.get_object().bucket(bucket).key(allowed_member).send().await?;
-    assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), expected_body);
-
-    Ok(())
-}
-
 #[tokio::test]
 async fn test_signed_put_object_extract_preserves_request_metadata_on_extracted_objects()
 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -4289,60 +4185,6 @@ async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box
     Ok(())
 }
 
-#[tokio::test]
-async fn test_signed_put_object_extract_expands_s2_and_lz4_by_magic_with_raw_etags()
--> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    init_logging();
-
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-
-    let bucket = "signed-extract-magic-codecs";
-    let client = env.create_s3_client();
-    client.create_bucket().bucket(bucket).send().await?;
-
-    let s2_tar = make_tar(&[("s2/object.txt", b"s2-body")], &[]).await;
-    let s2_archive = minio_go_snowball_s2_bytes(&s2_tar);
-    let expected_s2_etag = format!("\"{}\"", md5_hex(&s2_archive));
-    let s2_response = client
-        .put_object()
-        .bucket(bucket)
-        // minio-go intentionally uploads a compressed S2 stream with a .tar key.
-        .key("snowball-upload-0123456789abcdef.tar")
-        .body(ByteStream::from(s2_archive))
-        .customize()
-        .mutate_request(|req| {
-            req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
-        })
-        .send()
-        .await?;
-    assert_eq!(s2_response.e_tag(), Some(expected_s2_etag.as_str()));
-
-    let s2_object = client.get_object().bucket(bucket).key("s2/object.txt").send().await?;
-    assert_eq!(s2_object.body.collect().await?.into_bytes().as_ref(), b"s2-body");
-
-    let lz4_tar = make_tar(&[("lz4/object.txt", b"lz4-body")], &[]).await;
-    let lz4_archive = lz4_bytes(&lz4_tar).await;
-    let expected_lz4_etag = format!("\"{}\"", md5_hex(&lz4_archive));
-    let lz4_response = client
-        .put_object()
-        .bucket(bucket)
-        .key("also-looks-like-a-plain.tar")
-        .body(ByteStream::from(lz4_archive))
-        .customize()
-        .mutate_request(|req| {
-            req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
-        })
-        .send()
-        .await?;
-    assert_eq!(lz4_response.e_tag(), Some(expected_lz4_etag.as_str()));
-
-    let lz4_object = client.get_object().bucket(bucket).key("lz4/object.txt").send().await?;
-    assert_eq!(lz4_object.body.collect().await?.into_bytes().as_ref(), b"lz4-body");
-
-    Ok(())
-}
-
 #[tokio::test]
 async fn test_signed_put_object_extract_preserves_entry_mtime() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
@@ -4467,15 +4309,9 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
     let context_archive_resources = [
         format!("arn:aws:s3:::{bucket}/tag-context.tar"),
         format!("arn:aws:s3:::{bucket}/lock-context.tar"),
-        format!("arn:aws:s3:::{bucket}/legal-hold-context.tar"),
-        format!("arn:aws:s3:::{bucket}/user-agent-bypass.tar"),
-        format!("arn:aws:s3:::{bucket}/sse-bypass.tar"),
     ];
     let tag_entry_resource = format!("arn:aws:s3:::{bucket}/tag-context-entry.txt");
     let lock_entry_resource = format!("arn:aws:s3:::{bucket}/lock-context-entry.txt");
-    let legal_hold_entry_resource = format!("arn:aws:s3:::{bucket}/legal-hold-context-entry.txt");
-    let user_agent_entry_resource = format!("arn:aws:s3:::{bucket}/user-agent-bypass-entry.txt");
-    let sse_entry_resource = format!("arn:aws:s3:::{bucket}/sse-bypass-entry.txt");
     let policy = serde_json::json!({
         "Version": "2012-10-17",
         "Statement": [
@@ -4535,7 +4371,7 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
                 "Sid": "PaxContextArchives",
                 "Effect": "Allow",
                 "Principal": { "AWS": [pax_context_user] },
-                "Action": ["s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectLegalHold", "s3:PutObjectTagging"],
+                "Action": ["s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectTagging"],
                 "Resource": context_archive_resources
             },
             {
@@ -4575,49 +4411,6 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
                 "Principal": { "AWS": [pax_context_user] },
                 "Action": ["s3:PutObjectRetention"],
                 "Resource": [lock_entry_resource]
-            },
-            {
-                "Sid": "PaxLegalHoldContextPut",
-                "Effect": "Allow",
-                "Principal": { "AWS": [pax_context_user] },
-                "Action": ["s3:PutObject"],
-                "Resource": [legal_hold_entry_resource.clone()]
-            },
-            {
-                "Sid": "PaxLegalHoldContextAction",
-                "Effect": "Allow",
-                "Principal": { "AWS": [pax_context_user] },
-                "Action": ["s3:PutObjectLegalHold"],
-                "Resource": [legal_hold_entry_resource],
-                "Condition": {
-                    "StringEquals": {
-                        "s3:object-lock-legal-hold": "OFF"
-                    }
-                }
-            },
-            {
-                "Sid": "MemberUserAgentCondition",
-                "Effect": "Allow",
-                "Principal": { "AWS": [pax_context_user] },
-                "Action": ["s3:PutObject"],
-                "Resource": [user_agent_entry_resource],
-                "Condition": {
-                    "StringEquals": {
-                        "aws:UserAgent": "trusted"
-                    }
-                }
-            },
-            {
-                "Sid": "MemberSseCondition",
-                "Effect": "Allow",
-                "Principal": { "AWS": [pax_context_user] },
-                "Action": ["s3:PutObject"],
-                "Resource": [sse_entry_resource],
-                "Condition": {
-                    "StringEquals": {
-                        "s3:x-amz-server-side-encryption": "AES256"
-                    }
-                }
             }
         ]
     })
@@ -4630,13 +4423,8 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
     let cases = [
         (
             "legal-hold.tar",
-            put_only_client.clone(),
-            HashMap::from([("minio.metadata.x-amz-object-lock-legal-hold", "ON".to_string())]),
-        ),
-        (
-            "tagging.tar",
             put_only_client,
-            HashMap::from([("minio.metadata.x-amz-tagging", "classification=restricted".to_string())]),
+            HashMap::from([("minio.metadata.x-amz-object-lock-legal-hold", "ON".to_string())]),
         ),
         (
             "retention-condition.tar",
@@ -4724,57 +4512,6 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
     assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), b"condition-body");
 
     let pax_context_client = restricted_user_client(&env, pax_context_user, pax_context_secret);
-    for (archive_key, entry_key, pax_key, injected_value, outer_user_agent) in [
-        (
-            "user-agent-bypass.tar",
-            "user-agent-bypass-entry.txt",
-            "minio.metadata.user-agent",
-            "trusted",
-            Some("untrusted"),
-        ),
-        (
-            "sse-bypass.tar",
-            "sse-bypass-entry.txt",
-            "minio.metadata.x-amz-server-side-encryption",
-            "AES256",
-            None,
-        ),
-    ] {
-        let pax = HashMap::from([(pax_key, injected_value.to_string())]);
-        let archive = make_tar_with_pax_entry(entry_key, b"must-not-write", None, &pax).await;
-        let err = pax_context_client
-            .put_object()
-            .bucket(bucket)
-            .key(archive_key)
-            .body(ByteStream::from(archive))
-            .customize()
-            .mutate_request(move |req| {
-                req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
-                if let Some(user_agent) = outer_user_agent {
-                    req.headers_mut().insert("user-agent", user_agent);
-                }
-            })
-            .send()
-            .await
-            .expect_err("PAX metadata must not satisfy unrelated IAM request conditions");
-        assert_eq!(
-            err.as_service_error().and_then(|error| error.meta().code()),
-            Some("AccessDenied"),
-            "{archive_key}"
-        );
-        let err = admin_client
-            .head_object()
-            .bucket(bucket)
-            .key(entry_key)
-            .send()
-            .await
-            .expect_err("a denied PAX member must not be written");
-        assert!(matches!(
-            err.as_service_error().and_then(|error| error.meta().code()),
-            Some("NoSuchKey" | "NotFound")
-        ));
-    }
-
     let tag_pax = HashMap::from([("minio.metadata.x-amz-tagging", "classification=public".to_string())]);
     let archive = make_tar_with_pax_entry("tag-context-entry.txt", b"tag-context-body", None, &tag_pax).await;
     pax_context_client
@@ -4838,34 +4575,6 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
         pax_retain_until
     );
 
-    let legal_hold_pax = HashMap::from([("minio.metadata.x-amz-object-lock-legal-hold", "ON".to_string())]);
-    let archive = make_tar_with_pax_entry("legal-hold-context-entry.txt", b"must-not-write", None, &legal_hold_pax).await;
-    let err = pax_context_client
-        .put_object()
-        .bucket(bucket)
-        .key("legal-hold-context.tar")
-        .object_lock_legal_hold_status(aws_sdk_s3::types::ObjectLockLegalHoldStatus::Off)
-        .body(ByteStream::from(archive))
-        .customize()
-        .mutate_request(|req| {
-            req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
-        })
-        .send()
-        .await
-        .expect_err("PAX legal hold must replace the outer value in the member IAM condition context");
-    assert_eq!(err.as_service_error().and_then(|error| error.meta().code()), Some("AccessDenied"));
-    let err = admin_client
-        .head_object()
-        .bucket(bucket)
-        .key("legal-hold-context-entry.txt")
-        .send()
-        .await
-        .expect_err("a denied PAX legal-hold member must not be written");
-    assert!(matches!(
-        err.as_service_error().and_then(|error| error.meta().code()),
-        Some("NoSuchKey" | "NotFound")
-    ));
-
     Ok(())
 }
 
@@ -5341,8 +5050,8 @@ async fn test_signed_put_object_extract_expands_tzst_archive() -> Result<(), Box
 }
 
 #[tokio::test]
-async fn test_signed_put_object_extract_uses_magic_without_requiring_or_trusting_extension()
--> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+async fn test_signed_put_object_extract_rejects_missing_archive_extension() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
+{
     init_logging();
 
     let mut env = RustFSTestEnvironment::new().await?;
@@ -5355,7 +5064,8 @@ async fn test_signed_put_object_extract_uses_magic_without_requiring_or_trusting
     admin_client.create_bucket().bucket(bucket).send().await?;
 
     let tar_bytes = make_tar(&[("plain.txt", b"plain-body")], &[]).await;
-    admin_client
+
+    let result = admin_client
         .put_object()
         .bucket(bucket)
         .key(archive_key)
@@ -5365,80 +5075,15 @@ async fn test_signed_put_object_extract_uses_magic_without_requiring_or_trusting
             req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
         })
         .send()
-        .await?;
+        .await;
 
-    let plain = admin_client.get_object().bucket(bucket).key("plain.txt").send().await?;
-    assert_eq!(plain.body.collect().await?.into_bytes().as_ref(), b"plain-body");
-
-    let raw_with_gzip_suffix = make_tar(&[("raw-with-wrong-suffix.txt", b"raw-body")], &[]).await;
-    admin_client
-        .put_object()
-        .bucket(bucket)
-        .key("raw-but-named.tar.gz")
-        .body(ByteStream::from(raw_with_gzip_suffix))
-        .customize()
-        .mutate_request(|req| {
-            req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
-        })
-        .send()
-        .await?;
-
-    let raw = admin_client
-        .get_object()
-        .bucket(bucket)
-        .key("raw-with-wrong-suffix.txt")
-        .send()
-        .await?;
-    assert_eq!(raw.body.collect().await?.into_bytes().as_ref(), b"raw-body");
-
-    let gzip_with_tar_suffix = gzip_bytes(&make_tar(&[("gzip-with-wrong-suffix.txt", b"gzip-body")], &[]).await);
-    admin_client
-        .put_object()
-        .bucket(bucket)
-        .key("gzip-but-named.tar")
-        .body(ByteStream::from(gzip_with_tar_suffix))
-        .customize()
-        .mutate_request(|req| {
-            req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
-        })
-        .send()
-        .await?;
-
-    let gzip = admin_client
-        .get_object()
-        .bucket(bucket)
-        .key("gzip-with-wrong-suffix.txt")
-        .send()
-        .await?;
-    assert_eq!(gzip.body.collect().await?.into_bytes().as_ref(), b"gzip-body");
-
-    let zlib_archive = zlib_bytes(&make_tar(&[("zlib-extension.txt", b"zlib-body")], &[]).await);
-    admin_client
-        .put_object()
-        .bucket(bucket)
-        .key("bundle.zlib")
-        .body(ByteStream::from(zlib_archive))
-        .customize()
-        .mutate_request(|req| {
-            req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
-        })
-        .send()
-        .await?;
-
-    let zlib = admin_client
-        .get_object()
-        .bucket(bucket)
-        .key("zlib-extension.txt")
-        .send()
-        .await?;
-    assert_eq!(zlib.body.collect().await?.into_bytes().as_ref(), b"zlib-body");
+    assert_s3_error_code(result, "InvalidArgument");
 
     Ok(())
 }
 
 #[tokio::test]
-async fn test_signed_put_object_extract_rejects_invalid_archive_payload() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
-{
+async fn test_signed_put_object_extract_rejects_invalid_tar_gz_payload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
 
     let mut env = RustFSTestEnvironment::new().await?;

crates/e2e_test/src/negative_sigv4_test.rs

@@ -36,10 +36,9 @@
 use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
-use rustfs_signer::constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER};
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
 use rustfs_signer::request_signature_v4::{SIGN_V4_ALGORITHM, get_scope, get_signature, get_signing_key};
 use std::fmt::Write as _;
-use std::io::Cursor;
 use time::macros::format_description;
 use time::{Duration, OffsetDateTime};
 use tracing::info;
@@ -99,37 +98,15 @@ impl SigV4 {
     /// header AND folded into the canonical request — pass the hash of the
     /// body you *claim* to send, which may differ from what you actually send.
     fn sign(&self, method: &str, path: &str, canonical_query: &str, content_sha256: &str) -> SignedHeaders {
-        self.sign_with_extra_headers(method, path, canonical_query, content_sha256, &[])
-    }
-
-    /// Sign additional request headers while preserving SigV4's lowercase,
-    /// lexicographically sorted canonical-header representation.
-    fn sign_with_extra_headers(
-        &self,
-        method: &str,
-        path: &str,
-        canonical_query: &str,
-        content_sha256: &str,
-        extra_signed_headers: &[(&str, &str)],
-    ) -> SignedHeaders {
         let amz_date = amz_datetime(self.time);
-        let mut canonical_header_values = vec![
-            ("host", self.host.as_str()),
-            ("x-amz-content-sha256", content_sha256),
-            ("x-amz-date", amz_date.as_str()),
-        ];
-        canonical_header_values.extend(extra_signed_headers.iter().copied());
-        canonical_header_values.sort_unstable_by(|left, right| left.0.cmp(right.0));
+        let signed_headers = "host;x-amz-content-sha256;x-amz-date";
 
-        let signed_headers = canonical_header_values
-            .iter()
-            .map(|(name, _)| *name)
-            .collect::<Vec<_>>()
-            .join(";");
-        let mut canonical_headers = String::new();
-        for (name, value) in canonical_header_values {
-            let _ = writeln!(canonical_headers, "{name}:{value}");
-        }
+        let canonical_headers = format!(
+            "host:{host}\nx-amz-content-sha256:{sha}\nx-amz-date:{date}\n",
+            host = self.host,
+            sha = content_sha256,
+            date = amz_date,
+        );
         let canonical_request =
             format!("{method}\n{path}\n{canonical_query}\n{canonical_headers}\n{signed_headers}\n{content_sha256}");
 
@@ -202,34 +179,6 @@ async fn setup(env: &mut RustFSTestEnvironment) -> Result<(), Box<dyn std::error
     Ok(())
 }
 
-async fn build_single_member_archive(
-    member_key: &str,
-    member_body: &[u8],
-) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
-    let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
-    let mut header = tokio_tar::Header::new_gnu();
-    header.set_size(member_body.len() as u64);
-    header.set_mode(0o644);
-    header.set_cksum();
-    builder.append_data(&mut header, member_key, Cursor::new(member_body)).await?;
-    Ok(builder.into_inner().await?.into_inner())
-}
-
-fn sha256_base64(data: &[u8]) -> String {
-    use sha2::{Digest, Sha256};
-
-    base64_simd::STANDARD.encode_to_string(Sha256::digest(data))
-}
-
-fn encode_unsigned_aws_chunked_with_sha256_trailer(decoded: &[u8]) -> Vec<u8> {
-    let checksum = sha256_base64(decoded);
-    let mut encoded = format!("{:x}\r\n", decoded.len()).into_bytes();
-    encoded.extend_from_slice(decoded);
-    encoded.extend_from_slice(b"\r\n0\r\n");
-    encoded.extend_from_slice(format!("x-amz-checksum-sha256:{checksum}\r\n\r\n").as_bytes());
-    encoded
-}
-
 /// Positive control: a correctly hand-signed request must succeed. Without
 /// this, every negative assertion below could pass for the wrong reason (a
 /// broken signer that never produces a valid signature).
@@ -300,128 +249,6 @@ async fn tampered_signature_returns_signature_does_not_match() -> Result<(), Box
     Ok(())
 }
 
-/// `STREAMING-UNSIGNED-PAYLOAD-TRAILER` disables per-chunk signatures, not the
-/// seed/header SigV4 signature. A forged request must be rejected before the
-/// Snowball handler can publish any archive member.
-#[tokio::test]
-async fn snowball_streaming_unsigned_trailer_rejects_forged_signature() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    setup(&mut env).await?;
-
-    let archive_key = "forged-streaming-snowball.tar";
-    let member_key = "must-not-be-published.txt";
-    let archive = build_single_member_archive(member_key, b"forged request payload").await?;
-    let decoded_content_length = archive.len().to_string();
-    let encoded_body = encode_unsigned_aws_chunked_with_sha256_trailer(&archive);
-    let path = format!("/{BUCKET}/{archive_key}");
-
-    let mut signer = SigV4::new(&env);
-    signer.secret_key = "wrong-secret-for-forged-streaming-request".to_string();
-    let extra_signed_headers = [
-        ("content-encoding", "aws-chunked"),
-        ("x-amz-decoded-content-length", decoded_content_length.as_str()),
-        ("x-amz-meta-snowball-auto-extract", "true"),
-        ("x-amz-trailer", "x-amz-checksum-sha256"),
-    ];
-    let headers = signer.sign_with_extra_headers("PUT", &path, "", UNSIGNED_PAYLOAD_TRAILER, &extra_signed_headers);
-
-    let response = local_http_client()
-        .put(format!("{}{}", env.url, path))
-        .header("authorization", &headers.authorization)
-        .header("content-encoding", "aws-chunked")
-        .header("x-amz-content-sha256", &headers.content_sha256)
-        .header("x-amz-date", &headers.amz_date)
-        .header("x-amz-decoded-content-length", &decoded_content_length)
-        .header("x-amz-meta-snowball-auto-extract", "true")
-        .header("x-amz-trailer", "x-amz-checksum-sha256")
-        .body(encoded_body)
-        .send()
-        .await?;
-    let status = response.status();
-    let body = response.text().await?;
-    assert_eq!(status.as_u16(), 403, "forged streaming signature must be 403, body:\n{body}");
-    assert_error_code(&body, "SignatureDoesNotMatch");
-
-    let absent = env
-        .create_s3_client()
-        .get_object()
-        .bucket(BUCKET)
-        .key(member_key)
-        .send()
-        .await
-        .expect_err("a forged streaming request must not publish a Snowball member");
-    assert_eq!(absent.raw_response().map(|response| response.status().as_u16()), Some(404));
-    assert_eq!(absent.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey"));
-
-    env.stop_server();
-    Ok(())
-}
-
-/// Snowball must consume the complete aws-chunked body before reading the
-/// trailing checksum exported by s3s into the PutObject response.
-#[tokio::test]
-async fn snowball_streaming_unsigned_trailer_returns_sha256_checksum() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    setup(&mut env).await?;
-
-    let archive_key = "valid-streaming-snowball.tar";
-    let member_key = "streaming-checksum-member.txt";
-    let member_body = b"valid streaming Snowball payload";
-    let archive = build_single_member_archive(member_key, member_body).await?;
-    let expected_checksum = sha256_base64(&archive);
-    let decoded_content_length = archive.len().to_string();
-    let encoded_body = encode_unsigned_aws_chunked_with_sha256_trailer(&archive);
-    let path = format!("/{BUCKET}/{archive_key}");
-
-    let signer = SigV4::new(&env);
-    let extra_signed_headers = [
-        ("content-encoding", "aws-chunked"),
-        ("x-amz-decoded-content-length", decoded_content_length.as_str()),
-        ("x-amz-meta-snowball-auto-extract", "true"),
-        ("x-amz-sdk-checksum-algorithm", "SHA256"),
-        ("x-amz-trailer", "x-amz-checksum-sha256"),
-    ];
-    let headers = signer.sign_with_extra_headers("PUT", &path, "", UNSIGNED_PAYLOAD_TRAILER, &extra_signed_headers);
-
-    let response = local_http_client()
-        .put(format!("{}{}", env.url, path))
-        .header("authorization", &headers.authorization)
-        .header("content-encoding", "aws-chunked")
-        .header("x-amz-content-sha256", &headers.content_sha256)
-        .header("x-amz-date", &headers.amz_date)
-        .header("x-amz-decoded-content-length", &decoded_content_length)
-        .header("x-amz-meta-snowball-auto-extract", "true")
-        .header("x-amz-sdk-checksum-algorithm", "SHA256")
-        .header("x-amz-trailer", "x-amz-checksum-sha256")
-        .body(encoded_body)
-        .send()
-        .await?;
-    let status = response.status();
-    let response_checksum = response
-        .headers()
-        .get("x-amz-checksum-sha256")
-        .and_then(|value| value.to_str().ok())
-        .map(str::to_owned);
-    let response_body = response.text().await?;
-    assert_eq!(status.as_u16(), 200, "valid streaming Snowball PUT failed, body:\n{response_body}");
-    assert_eq!(response_checksum.as_deref(), Some(expected_checksum.as_str()));
-
-    let member = env
-        .create_s3_client()
-        .get_object()
-        .bucket(BUCKET)
-        .key(member_key)
-        .send()
-        .await?;
-    let stored = member.body.collect().await?.into_bytes();
-    assert_eq!(stored.as_ref(), member_body);
-
-    env.stop_server();
-    Ok(())
-}
-
 /// (b) A valid AccessKeyId paired with the wrong secret key must be rejected
 /// with SignatureDoesNotMatch / 403.
 #[tokio::test]
@@ -549,68 +376,6 @@ async fn tampered_upload_part_payload_is_rejected() -> Result<(), Box<dyn std::e
     Ok(())
 }
 
-/// s3s v0.16 validates the aws-chunked decoded length while RustFS consumes the
-/// body stream. Mismatches are client body errors and must not leak as 500s.
-#[tokio::test]
-async fn aws_chunked_decoded_length_mismatch_returns_incomplete_body() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    setup(&mut env).await?;
-
-    for (key, declared_len) in [
-        ("decoded-length-overrun.bin", 3_usize),
-        ("decoded-length-shortfall.bin", 9_usize),
-    ] {
-        let decoded = b"decoded";
-        assert_ne!(declared_len, decoded.len(), "test case must exercise a mismatch");
-        let encoded_body = encode_unsigned_aws_chunked_with_sha256_trailer(decoded);
-        let decoded_content_length = declared_len.to_string();
-        let path = format!("/{BUCKET}/{key}");
-        let signer = SigV4::new(&env);
-        let extra_signed_headers = [
-            ("content-encoding", "aws-chunked"),
-            ("x-amz-decoded-content-length", decoded_content_length.as_str()),
-            ("x-amz-trailer", "x-amz-checksum-sha256"),
-        ];
-        let headers = signer.sign_with_extra_headers("PUT", &path, "", UNSIGNED_PAYLOAD_TRAILER, &extra_signed_headers);
-
-        let response = local_http_client()
-            .put(format!("{}{}", env.url, path))
-            .header("authorization", &headers.authorization)
-            .header("content-encoding", "aws-chunked")
-            .header("x-amz-content-sha256", &headers.content_sha256)
-            .header("x-amz-date", &headers.amz_date)
-            .header("x-amz-decoded-content-length", &decoded_content_length)
-            .header("x-amz-trailer", "x-amz-checksum-sha256")
-            .body(encoded_body)
-            .timeout(std::time::Duration::from_secs(10))
-            .send()
-            .await?;
-        let status = response.status();
-        let body = response.text().await.unwrap_or_default();
-        assert_eq!(
-            status,
-            reqwest::StatusCode::BAD_REQUEST,
-            "decoded length mismatch must be a client error, body:\n{body}"
-        );
-        assert_error_code(&body, "IncompleteBody");
-
-        let absent = env
-            .create_s3_client()
-            .get_object()
-            .bucket(BUCKET)
-            .key(key)
-            .send()
-            .await
-            .expect_err("decoded length mismatch must not publish an object");
-        assert_eq!(absent.raw_response().map(|response| response.status().as_u16()), Some(404));
-        assert_eq!(absent.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey"));
-    }
-
-    env.stop_server();
-    Ok(())
-}
-
 /// (e) A request whose `x-amz-date` is skewed beyond the server's tolerance
 /// (s3s default 900s / 15 min) must be rejected with RequestTimeTooSkewed /
 /// 403. The signature is otherwise valid: the credential-scope date and

crates/e2e_test/src/reliant/mod.rs

@@ -21,6 +21,5 @@ mod head_tls_bodyless_test;
 mod lifecycle;
 mod lock;
 mod node_interact_test;
-mod s3_select_compression;
 mod sql;
 mod tiering;

crates/e2e_test/src/reliant/s3_select_compression.rs

@@ -1,351 +0,0 @@
-#![cfg(test)]
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use crate::common::{RustFSTestEnvironment, init_logging};
-use async_compression::tokio::write::BzEncoder;
-use aws_sdk_s3::{
-    Client,
-    error::ProvideErrorMetadata,
-    operation::select_object_content::{SelectObjectContentOutput, builders::SelectObjectContentFluentBuilder},
-    types::{
-        CompressionType, CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput,
-        JsonType, OutputSerialization, SelectObjectContentEventStream,
-    },
-};
-use aws_smithy_types::event_stream::RawMessage;
-use bytes::Bytes;
-use flate2::{Compression, write::GzEncoder};
-use std::{error::Error, io::Cursor, time::Duration};
-use tokio::io::AsyncWriteExt;
-
-const BUCKET: &str = "s3-select-compression";
-const SELECT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(30);
-
-type TestResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
-
-async fn create_test_environment(extra_env: &[(&str, &str)]) -> TestResult<(RustFSTestEnvironment, Client)> {
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server_with_env(vec![], extra_env).await?;
-    let client = env.create_s3_client();
-    client.create_bucket().bucket(BUCKET).send().await?;
-    Ok((env, client))
-}
-
-async fn put_object(client: &Client, key: &str, body: &[u8]) -> TestResult<()> {
-    client
-        .put_object()
-        .bucket(BUCKET)
-        .key(key)
-        .body(Bytes::copy_from_slice(body).into())
-        .send()
-        .await?;
-    Ok(())
-}
-
-fn gzip(input: &[u8]) -> TestResult<Vec<u8>> {
-    let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
-    std::io::Write::write_all(&mut encoder, input)?;
-    Ok(encoder.finish()?)
-}
-
-async fn bzip2(input: &[u8]) -> TestResult<Vec<u8>> {
-    let mut encoder = BzEncoder::new(Cursor::new(Vec::new()));
-    encoder.write_all(input).await?;
-    encoder.shutdown().await?;
-    Ok(encoder.into_inner().into_inner())
-}
-
-fn csv_select_request(
-    client: &Client,
-    key: &str,
-    compression: CompressionType,
-    expression: &str,
-) -> SelectObjectContentFluentBuilder {
-    client
-        .select_object_content()
-        .bucket(BUCKET)
-        .key(key)
-        .expression(expression)
-        .expression_type(ExpressionType::Sql)
-        .input_serialization(
-            InputSerialization::builder()
-                .compression_type(compression)
-                .csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
-                .build(),
-        )
-        .output_serialization(OutputSerialization::builder().csv(CsvOutput::builder().build()).build())
-}
-
-fn json_select_request(
-    client: &Client,
-    key: &str,
-    compression: CompressionType,
-    json_type: JsonType,
-) -> SelectObjectContentFluentBuilder {
-    client
-        .select_object_content()
-        .bucket(BUCKET)
-        .key(key)
-        .expression("SELECT name FROM S3Object")
-        .expression_type(ExpressionType::Sql)
-        .input_serialization(
-            InputSerialization::builder()
-                .compression_type(compression)
-                .json(JsonInput::builder().set_type(Some(json_type)).build())
-                .build(),
-        )
-        .output_serialization(OutputSerialization::builder().json(JsonOutput::builder().build()).build())
-}
-
-async fn collect_success(
-    mut response: SelectObjectContentOutput,
-    compressed_bytes: usize,
-    processed_bytes: usize,
-) -> TestResult<Vec<u8>> {
-    tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async move {
-        let mut records = Vec::new();
-        let mut stats = None;
-        let mut saw_end = false;
-
-        while let Some(event) = response.payload.recv().await? {
-            assert!(!saw_end, "Select emitted an event after End");
-            match event {
-                SelectObjectContentEventStream::Records(event) => {
-                    assert!(stats.is_none(), "Select emitted Records after Stats");
-                    if let Some(payload) = event.payload {
-                        records.extend_from_slice(payload.as_ref());
-                    }
-                }
-                SelectObjectContentEventStream::Stats(event) => {
-                    assert!(stats.is_none(), "Select emitted more than one Stats event");
-                    stats = event.details;
-                }
-                SelectObjectContentEventStream::End(_) => {
-                    assert!(stats.is_some(), "Select emitted End before Stats");
-                    saw_end = true;
-                }
-                _ => assert!(stats.is_none(), "Select emitted a non-terminal event after Stats"),
-            }
-        }
-
-        let stats = stats.ok_or("Select response ended without a Stats event")?;
-        assert_eq!(stats.bytes_scanned(), Some(i64::try_from(compressed_bytes)?));
-        assert_eq!(stats.bytes_processed(), Some(i64::try_from(processed_bytes)?));
-        assert_eq!(stats.bytes_returned(), Some(i64::try_from(records.len())?));
-        assert!(saw_end, "Select response ended without an End event");
-        Ok::<_, Box<dyn Error + Send + Sync>>(records)
-    })
-    .await
-    .map_err(|_| -> Box<dyn Error + Send + Sync> { "Select response timed out".into() })?
-}
-
-async fn assert_truncated_stream_failure(mut response: SelectObjectContentOutput) -> TestResult<()> {
-    tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async move {
-        loop {
-            match response.payload.recv().await {
-                Err(error) => {
-                    // S3 Select request-level errors use `error` frames, which this SDK version exposes as raw response errors.
-                    if let Some(code) = error.code() {
-                        assert_eq!(code, "TruncatedInput", "unexpected modeled event-stream error: {error:?}");
-                    } else if let aws_sdk_s3::error::SdkError::ResponseError(context) = &error
-                        && let RawMessage::Decoded(message) = context.raw()
-                    {
-                        let header = |name: &str| {
-                            message
-                                .headers()
-                                .iter()
-                                .find(|header| header.name().as_str() == name)
-                                .and_then(|header| header.value().as_string().ok())
-                                .map(|value| value.as_str())
-                        };
-                        assert_eq!(header(":message-type"), Some("error"));
-                        assert_eq!(header(":error-code"), Some("TruncatedInput"));
-                    } else {
-                        panic!("unexpected event-stream error: {error:?}");
-                    }
-                    return Ok(());
-                }
-                Ok(Some(SelectObjectContentEventStream::Stats(_))) | Ok(Some(SelectObjectContentEventStream::End(_))) => {
-                    return Err("truncated compressed input reached a success terminal event".into());
-                }
-                Ok(Some(_)) => {}
-                Ok(None) => return Err("truncated compressed input ended without an error event".into()),
-            }
-        }
-    })
-    .await
-    .map_err(|_| -> Box<dyn Error + Send + Sync> { "truncated Select response timed out".into() })?
-}
-
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-async fn test_select_object_content_compressed_csv_and_json() -> TestResult<()> {
-    const CSV: &[u8] = b"name,age\nAlice,30\nBob,25\n";
-    const JSON_LINES: &[u8] = b"{\"name\":\"Alice\"}\n{\"name\":\"Bob\"}\n";
-    const JSON_DOCUMENT: &[u8] = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
-
-    let (_env, client) = create_test_environment(&[]).await?;
-
-    let gzip_csv = gzip(CSV)?;
-    put_object(&client, "records.csv.gz", &gzip_csv).await?;
-    let gzip_csv_records = collect_success(
-        csv_select_request(&client, "records.csv.gz", CompressionType::Gzip, "SELECT * FROM S3Object")
-            .send()
-            .await?,
-        gzip_csv.len(),
-        CSV.len(),
-    )
-    .await?;
-    assert_eq!(gzip_csv_records, b"Alice,30\nBob,25\n");
-
-    let bzip_csv = bzip2(CSV).await?;
-    put_object(&client, "records.csv.bz2", &bzip_csv).await?;
-    let bzip_csv_records = collect_success(
-        csv_select_request(&client, "records.csv.bz2", CompressionType::Bzip2, "SELECT * FROM S3Object")
-            .send()
-            .await?,
-        bzip_csv.len(),
-        CSV.len(),
-    )
-    .await?;
-    assert_eq!(bzip_csv_records, gzip_csv_records);
-
-    let gzip_json_lines = gzip(JSON_LINES)?;
-    put_object(&client, "json-lines", &gzip_json_lines).await?;
-    let gzip_json_records = collect_success(
-        json_select_request(&client, "json-lines", CompressionType::Gzip, JsonType::Lines)
-            .send()
-            .await?,
-        gzip_json_lines.len(),
-        JSON_LINES.len(),
-    )
-    .await?;
-    assert_eq!(gzip_json_records, JSON_LINES);
-
-    let bzip_json_lines = bzip2(JSON_LINES).await?;
-    put_object(&client, "records.jsonl.bz2", &bzip_json_lines).await?;
-    let bzip_json_records = collect_success(
-        json_select_request(&client, "records.jsonl.bz2", CompressionType::Bzip2, JsonType::Lines)
-            .send()
-            .await?,
-        bzip_json_lines.len(),
-        JSON_LINES.len(),
-    )
-    .await?;
-    assert_eq!(bzip_json_records, gzip_json_records);
-
-    let gzip_json_document = gzip(JSON_DOCUMENT)?;
-    put_object(&client, "document.json.gz", &gzip_json_document).await?;
-    let document_records = collect_success(
-        json_select_request(&client, "document.json.gz", CompressionType::Gzip, JsonType::Document)
-            .send()
-            .await?,
-        gzip_json_document.len(),
-        JSON_DOCUMENT.len(),
-    )
-    .await?;
-    assert_eq!(document_records, JSON_LINES);
-
-    Ok(())
-}
-
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-async fn test_select_object_content_invalid_compressed_stream_fails() -> TestResult<()> {
-    const CSV: &[u8] = b"name\nAlice\n";
-
-    let (_env, client) = create_test_environment(&[]).await?;
-
-    put_object(&client, "invalid.csv.gz", CSV).await?;
-    let invalid = csv_select_request(&client, "invalid.csv.gz", CompressionType::Gzip, "SELECT * FROM S3Object")
-        .send()
-        .await
-        .expect_err("invalid GZIP header must fail before streaming");
-    assert_eq!(
-        invalid.as_service_error().and_then(ProvideErrorMetadata::code),
-        Some("InvalidCompressionFormat")
-    );
-
-    put_object(&client, "empty.csv.gz", b"").await?;
-    let empty = csv_select_request(&client, "empty.csv.gz", CompressionType::Gzip, "SELECT * FROM S3Object")
-        .send()
-        .await
-        .expect_err("empty GZIP input must fail as truncated");
-    assert_eq!(empty.as_service_error().and_then(ProvideErrorMetadata::code), Some("TruncatedInput"));
-
-    let mut truncated = bzip2(CSV).await?;
-    truncated.pop();
-    put_object(&client, "truncated.csv.bz2", &truncated).await?;
-    let truncated = csv_select_request(&client, "truncated.csv.bz2", CompressionType::Bzip2, "SELECT * FROM S3Object")
-        .send()
-        .await?;
-    assert_truncated_stream_failure(truncated).await?;
-
-    Ok(())
-}
-
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-async fn test_select_object_content_compressed_disconnect_releases_query() -> TestResult<()> {
-    const OBJECT: &str = "disconnect.csv.gz";
-    const ROWS: usize = 16 * 1024;
-    const RELEASE_ATTEMPTS: usize = 20;
-    const RELEASE_BACKOFF: Duration = Duration::from_millis(25);
-
-    let (_env, client) = create_test_environment(&[("RUSTFS_S3SELECT_MAX_CONCURRENT_QUERIES", "1")]).await?;
-    let row = format!("{}\n", "x".repeat(1023));
-    let mut body = Vec::with_capacity("value\n".len() + ROWS * row.len());
-    body.extend_from_slice(b"value\n");
-    for _ in 0..ROWS {
-        body.extend_from_slice(row.as_bytes());
-    }
-    let compressed = gzip(&body)?;
-    put_object(&client, OBJECT, &compressed).await?;
-
-    let first = csv_select_request(&client, OBJECT, CompressionType::Gzip, "SELECT * FROM S3Object")
-        .send()
-        .await?;
-    let saturated = csv_select_request(&client, OBJECT, CompressionType::Gzip, "SELECT * FROM S3Object")
-        .send()
-        .await
-        .expect_err("the unread compressed response should retain the only query permit");
-    assert_eq!(saturated.as_service_error().and_then(ProvideErrorMetadata::code), Some("SlowDown"));
-
-    drop(first);
-    let second = tokio::time::timeout(Duration::from_secs(5), async {
-        for attempt in 0..RELEASE_ATTEMPTS {
-            match csv_select_request(&client, OBJECT, CompressionType::Gzip, "SELECT * FROM S3Object")
-                .send()
-                .await
-            {
-                Ok(response) => return Ok::<_, Box<dyn Error + Send + Sync>>(response),
-                Err(error)
-                    if error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("SlowDown")
-                        && attempt + 1 < RELEASE_ATTEMPTS =>
-                {
-                    tokio::time::sleep(RELEASE_BACKOFF).await;
-                }
-                Err(error) if error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("SlowDown") => {
-                    return Err("disconnected compressed Select retained its query permit".into());
-                }
-                Err(error) => return Err(format!("unexpected Select error after disconnect: {error}").into()),
-            }
-        }
-        Err("query permit release retry loop ended unexpectedly".into())
-    })
-    .await
-    .map_err(|_| -> Box<dyn Error + Send + Sync> { "compressed Select did not release its query permit".into() })??;
-    drop(second);
-
-    Ok(())
-}

crates/e2e_test/src/reliant/sql.rs

@@ -17,8 +17,7 @@ use crate::common::{RustFSTestEnvironment, init_logging};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::types::{
-    CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput, JsonType,
-    OutputSerialization, RequestProgress,
+    CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput, JsonType, OutputSerialization,
 };
 use bytes::Bytes;
 use std::error::Error;
@@ -27,9 +26,6 @@ use std::time::Duration;
 const BUCKET: &str = "test-sql-bucket";
 const CSV_OBJECT: &str = "test-data.csv";
 const JSON_OBJECT: &str = "test-data.json";
-const JSON_DOCUMENT_OBJECT: &str = "nested-data.json";
-const JSON_ROOT_ARRAY_OBJECT: &str = "root-array.json";
-const JSON_ROOT_SCALAR_ARRAY_OBJECT: &str = "root-scalars.json";
 const SELECT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(30);
 
 type TestResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
@@ -77,69 +73,6 @@ async fn upload_test_json(client: &Client) -> TestResult<()> {
     Ok(())
 }
 
-async fn upload_nested_json_document(client: &Client) -> TestResult<()> {
-    let json_data = r#"{"departments":[{"employees":[{"name":"Alice","active":true},{"name":"Bob","active":false}]},{"employees":[{"name":"Charlie","active":true}]}]}"#;
-
-    client
-        .put_object()
-        .bucket(BUCKET)
-        .key(JSON_DOCUMENT_OBJECT)
-        .body(Bytes::from_static(json_data.as_bytes()).into())
-        .send()
-        .await?;
-    client
-        .put_object()
-        .bucket(BUCKET)
-        .key(JSON_ROOT_ARRAY_OBJECT)
-        .body(Bytes::from_static(br#"[{"name":"Alice"},{"name":"Bob"}]"#).into())
-        .send()
-        .await?;
-    client
-        .put_object()
-        .bucket(BUCKET)
-        .key(JSON_ROOT_SCALAR_ARRAY_OBJECT)
-        .body(Bytes::from_static(b"[1,2]").into())
-        .send()
-        .await?;
-    Ok(())
-}
-
-async fn select_json_document(client: &Client, key: &str, expression: &str) -> TestResult<String> {
-    let response = client
-        .select_object_content()
-        .bucket(BUCKET)
-        .key(key)
-        .expression(expression)
-        .expression_type(ExpressionType::Sql)
-        .input_serialization(
-            InputSerialization::builder()
-                .json(JsonInput::builder().set_type(Some(JsonType::Document)).build())
-                .build(),
-        )
-        .output_serialization(OutputSerialization::builder().json(JsonOutput::builder().build()).build())
-        .send()
-        .await?;
-    process_select_response(response).await
-}
-
-fn csv_select_request(
-    client: &Client,
-    key: &str,
-) -> aws_sdk_s3::operation::select_object_content::builders::SelectObjectContentFluentBuilder {
-    client
-        .select_object_content()
-        .bucket(BUCKET)
-        .key(key)
-        .expression("SELECT * FROM S3Object")
-        .expression_type(ExpressionType::Sql)
-        .input_serialization(
-            InputSerialization::builder()
-                .csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
-                .build(),
-        )
-        .output_serialization(OutputSerialization::builder().csv(CsvOutput::builder().build()).build())
-}
-
 async fn process_select_response(
     mut event_stream: aws_sdk_s3::operation::select_object_content::SelectObjectContentOutput,
 ) -> TestResult<String> {
@@ -171,209 +104,6 @@ async fn process_select_response(
     .map_err(|_| -> Box<dyn Error + Send + Sync> { "Select response timed out".into() })?
 }
 
-async fn assert_input_byte_stats(
-    client: &Client,
-    object: &str,
-    body: &[u8],
-    expression: &str,
-    input_serialization: InputSerialization,
-    output_serialization: OutputSerialization,
-    progress_enabled: bool,
-) -> TestResult<()> {
-    client
-        .put_object()
-        .bucket(BUCKET)
-        .key(object)
-        .body(Bytes::copy_from_slice(body).into())
-        .send()
-        .await?;
-
-    let mut request = client
-        .select_object_content()
-        .bucket(BUCKET)
-        .key(object)
-        .expression(expression)
-        .expression_type(ExpressionType::Sql)
-        .input_serialization(input_serialization)
-        .output_serialization(output_serialization);
-    if progress_enabled {
-        request = request.request_progress(RequestProgress::builder().enabled(true).build());
-    }
-    let response = request.send().await?;
-
-    let mut payload = response.payload;
-    let mut records_len = 0_u64;
-    let mut last_progress: Option<aws_sdk_s3::types::Progress> = None;
-    let mut stats = None;
-    let mut saw_end = false;
-    tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async {
-        // The AWS SDK validates both event-stream CRCs before yielding an event.
-        while let Some(event) = payload.recv().await? {
-            assert!(!saw_end, "Select emitted an event after End");
-            match event {
-                aws_sdk_s3::types::SelectObjectContentEventStream::Records(records) => {
-                    assert!(stats.is_none(), "Select emitted Records after Stats");
-                    if let Some(bytes) = records.payload {
-                        records_len = records_len.saturating_add(u64::try_from(bytes.as_ref().len())?);
-                    }
-                }
-                aws_sdk_s3::types::SelectObjectContentEventStream::Progress(event) => {
-                    assert!(stats.is_none(), "Select emitted Progress after Stats");
-                    let details = event.details.ok_or("Progress event did not contain details")?;
-                    if let Some(previous) = last_progress.as_ref() {
-                        assert!(details.bytes_scanned() >= previous.bytes_scanned());
-                        assert!(details.bytes_processed() >= previous.bytes_processed());
-                        assert!(details.bytes_returned() >= previous.bytes_returned());
-                    }
-                    last_progress = Some(details);
-                }
-                aws_sdk_s3::types::SelectObjectContentEventStream::Stats(event) => {
-                    assert!(stats.is_none(), "Select emitted more than one Stats event");
-                    stats = event.details;
-                }
-                aws_sdk_s3::types::SelectObjectContentEventStream::End(_) => {
-                    assert!(stats.is_some(), "Select emitted End before Stats");
-                    saw_end = true;
-                }
-                _ => assert!(stats.is_none(), "Select emitted a non-terminal event after Stats"),
-            }
-        }
-        Ok::<(), Box<dyn Error + Send + Sync>>(())
-    })
-    .await
-    .map_err(|_| -> Box<dyn Error + Send + Sync> { "Select response timed out".into() })??;
-
-    let stats = stats.ok_or("Select response ended without a Stats event")?;
-    let input_len = i64::try_from(body.len())?;
-    assert_eq!(stats.bytes_scanned(), Some(input_len));
-    assert_eq!(stats.bytes_processed(), Some(input_len));
-    assert_eq!(stats.bytes_returned(), Some(i64::try_from(records_len)?));
-    if progress_enabled {
-        if let Some(progress) = last_progress {
-            assert!(stats.bytes_scanned() >= progress.bytes_scanned());
-            assert!(stats.bytes_processed() >= progress.bytes_processed());
-            assert!(stats.bytes_returned() >= progress.bytes_returned());
-        }
-    } else {
-        assert!(last_progress.is_none(), "disabled request progress emitted a Progress event");
-    }
-    assert!(saw_end, "Select response ended without an End event");
-    Ok(())
-}
-
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-async fn test_select_object_content_http_event_order_crc_and_input_byte_stats() -> TestResult<()> {
-    const CSV_BODY: &[u8] = b"name,age\nAlice,30\nBob,25\n";
-    const JSON_LINES_BODY: &[u8] = b"{\"name\":\"Alice\"}\n{\"name\":\"Bob\"}\n";
-    const JSON_DOCUMENT_BODY: &[u8] = b"[{\"name\":\"Alice\"},{\"name\":\"Bob\"}]";
-
-    let (_env, client) = create_test_environment().await?;
-    setup_test_bucket(&client).await?;
-    assert_input_byte_stats(
-        &client,
-        "input-metrics.csv",
-        CSV_BODY,
-        "SELECT name FROM S3Object",
-        InputSerialization::builder()
-            .csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
-            .build(),
-        OutputSerialization::builder().csv(CsvOutput::builder().build()).build(),
-        true,
-    )
-    .await?;
-    assert_input_byte_stats(
-        &client,
-        "input-metrics.jsonl",
-        JSON_LINES_BODY,
-        "SELECT name FROM S3Object",
-        InputSerialization::builder()
-            .json(JsonInput::builder().set_type(Some(JsonType::Lines)).build())
-            .build(),
-        OutputSerialization::builder().json(JsonOutput::builder().build()).build(),
-        true,
-    )
-    .await?;
-    assert_input_byte_stats(
-        &client,
-        "input-metrics.json",
-        JSON_DOCUMENT_BODY,
-        "SELECT name FROM S3Object",
-        InputSerialization::builder()
-            .json(JsonInput::builder().set_type(Some(JsonType::Document)).build())
-            .build(),
-        OutputSerialization::builder().json(JsonOutput::builder().build()).build(),
-        true,
-    )
-    .await?;
-    assert_input_byte_stats(
-        &client,
-        "input-metrics-without-progress.csv",
-        CSV_BODY,
-        "SELECT name FROM S3Object",
-        InputSerialization::builder()
-            .csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
-            .build(),
-        OutputSerialization::builder().csv(CsvOutput::builder().build()).build(),
-        false,
-    )
-    .await?;
-    Ok(())
-}
-
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-async fn test_select_object_content_http_disconnect_releases_query() -> TestResult<()> {
-    const OBJECT: &str = "disconnect.csv";
-    const ROWS: usize = 16 * 1024;
-    const RELEASE_BACKOFF: Duration = Duration::from_millis(25);
-
-    init_logging();
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server_with_env(vec![], &[("RUSTFS_S3SELECT_MAX_CONCURRENT_QUERIES", "1")])
-        .await?;
-    let client = env.create_s3_client();
-    setup_test_bucket(&client).await?;
-
-    let row = format!("{}\n", "x".repeat(1023));
-    let mut body = Vec::with_capacity("value\n".len() + ROWS * row.len());
-    body.extend_from_slice(b"value\n");
-    for _ in 0..ROWS {
-        body.extend_from_slice(row.as_bytes());
-    }
-    client
-        .put_object()
-        .bucket(BUCKET)
-        .key(OBJECT)
-        .body(Bytes::from(body).into())
-        .send()
-        .await?;
-
-    // Leaving this response body unread fills the bounded HTTP/event channels before the query can finish.
-    let first = csv_select_request(&client, OBJECT).send().await?;
-    let saturated = csv_select_request(&client, OBJECT)
-        .send()
-        .await
-        .expect_err("the first HTTP stream should retain the only query permit");
-    assert_eq!(saturated.as_service_error().and_then(ProvideErrorMetadata::code), Some("SlowDown"));
-
-    drop(first);
-    let second = tokio::time::timeout(Duration::from_secs(5), async {
-        loop {
-            match csv_select_request(&client, OBJECT).send().await {
-                Ok(response) => return Ok::<_, Box<dyn Error + Send + Sync>>(response),
-                Err(error) if error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("SlowDown") => {
-                    tokio::time::sleep(RELEASE_BACKOFF).await;
-                }
-                Err(error) => return Err(format!("unexpected Select error after disconnect: {error}").into()),
-            }
-        }
-    })
-    .await
-    .map_err(|_| -> Box<dyn Error + Send + Sync> { "disconnected Select did not release its query permit".into() })??;
-    drop(second);
-
-    Ok(())
-}
-
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
 async fn test_select_object_content_csv_basic() -> TestResult<()> {
     let (_env, client) = create_test_environment().await?;
@@ -498,107 +228,6 @@ async fn test_select_object_content_json_basic() -> TestResult<()> {
     Ok(())
 }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-async fn test_select_object_content_nested_json_source_path() -> TestResult<()> {
-    let (_env, client) = create_test_environment().await?;
-    setup_test_bucket(&client).await?;
-    upload_nested_json_document(&client).await?;
-
-    let result = select_json_document(
-        &client,
-        JSON_DOCUMENT_OBJECT,
-        "SELECT e.name FROM S3Object[*].departments[*].employees[*] AS e WHERE e.active = true",
-    )
-    .await?;
-    let names: Vec<String> = result
-        .lines()
-        .filter(|line| !line.trim().is_empty())
-        .map(|line| -> TestResult<String> {
-            let value: serde_json::Value = serde_json::from_str(line)?;
-            Ok(value["name"].as_str().ok_or("missing name field")?.to_string())
-        })
-        .collect::<TestResult<_>>()?;
-
-    assert_eq!(names, vec!["Alice", "Charlie"]);
-
-    let terminal_scalars = select_json_document(
-        &client,
-        JSON_DOCUMENT_OBJECT,
-        "SELECT NAME FROM S3Object[*].DEPARTMENTS[*].employees[*].NAME",
-    )
-    .await?;
-    let scalar_names: Vec<String> = terminal_scalars
-        .lines()
-        .map(|line| -> TestResult<String> {
-            let value: serde_json::Value = serde_json::from_str(line)?;
-            Ok(value["name"].as_str().ok_or("missing scalar name field")?.to_string())
-        })
-        .collect::<TestResult<_>>()?;
-    assert_eq!(scalar_names, vec!["Alice", "Bob", "Charlie"]);
-
-    let aliased_scalars = select_json_document(
-        &client,
-        JSON_DOCUMENT_OBJECT,
-        "SELECT v FROM S3Object[*].departments[*].employees[*].name AS v",
-    )
-    .await?;
-    let aliased_names: Vec<String> = aliased_scalars
-        .lines()
-        .map(|line| -> TestResult<String> {
-            let value: serde_json::Value = serde_json::from_str(line)?;
-            Ok(value["v"].as_str().ok_or("missing aliased scalar field")?.to_string())
-        })
-        .collect::<TestResult<_>>()?;
-    assert_eq!(aliased_names, vec!["Alice", "Bob", "Charlie"]);
-
-    let root_array = select_json_document(&client, JSON_ROOT_ARRAY_OBJECT, "SELECT c.name FROM S3Object[*][*] AS c").await?;
-    let root_names: Vec<String> = root_array
-        .lines()
-        .map(|line| -> TestResult<String> {
-            let value: serde_json::Value = serde_json::from_str(line)?;
-            Ok(value["name"].as_str().ok_or("missing root-array name field")?.to_string())
-        })
-        .collect::<TestResult<_>>()?;
-    assert_eq!(root_names, vec!["Alice", "Bob"]);
-
-    let root_index = select_json_document(&client, JSON_ROOT_ARRAY_OBJECT, "SELECT c.name FROM S3Object[*][0] AS c").await?;
-    let root_index_value: serde_json::Value = serde_json::from_str(root_index.trim())?;
-    assert_eq!(root_index_value["name"], "Alice");
-
-    let root_scalars = select_json_document(&client, JSON_ROOT_SCALAR_ARRAY_OBJECT, "SELECT V FROM S3Object AS V").await?;
-    let scalar_values: Vec<i64> = root_scalars
-        .lines()
-        .map(|line| -> TestResult<i64> {
-            let value: serde_json::Value = serde_json::from_str(line)?;
-            Ok(value["v"].as_i64().ok_or("missing root scalar value")?)
-        })
-        .collect::<TestResult<_>>()?;
-    assert_eq!(scalar_values, vec![1, 2]);
-
-    let implicit_root_scalars =
-        select_json_document(&client, JSON_ROOT_SCALAR_ARRAY_OBJECT, "SELECT S3Object FROM S3Object").await?;
-    let implicit_scalar_values: Vec<i64> = implicit_root_scalars
-        .lines()
-        .map(|line| -> TestResult<i64> {
-            let value: serde_json::Value = serde_json::from_str(line)?;
-            Ok(value["s3object"].as_i64().ok_or("missing implicit root scalar value")?)
-        })
-        .collect::<TestResult<_>>()?;
-    assert_eq!(implicit_scalar_values, vec![1, 2]);
-
-    let quoted_root_scalars =
-        select_json_document(&client, JSON_ROOT_SCALAR_ARRAY_OBJECT, "SELECT \"S3Object\" FROM \"S3Object\"").await?;
-    let quoted_scalar_values: Vec<i64> = quoted_root_scalars
-        .lines()
-        .map(|line| -> TestResult<i64> {
-            let value: serde_json::Value = serde_json::from_str(line)?;
-            Ok(value["S3Object"].as_i64().ok_or("missing quoted root scalar value")?)
-        })
-        .collect::<TestResult<_>>()?;
-    assert_eq!(quoted_scalar_values, vec![1, 2]);
-    Ok(())
-}
-
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
 async fn test_select_object_content_csv_limit() -> TestResult<()> {
     let (_env, client) = create_test_environment().await?;

crates/e2e_test/src/replacement_privileged_e2e_test.rs

@@ -40,10 +40,10 @@ mod tests {
 
     const ENABLE_ENV: &str = "RUSTFS_PRIVILEGED_REPLACEMENT_E2E";
     const NAMESPACE_ENV: &str = "RUSTFS_PRIVILEGED_REPLACEMENT_E2E_IN_NAMESPACE";
-    const LOG_DIR_ENV: &str = "RUSTFS_PRIVILEGED_REPLACEMENT_LOG_DIR";
     const TARGET_NODE: usize = 1;
     const TARGET_DRIVE: usize = 0;
     const MOUNT_SIZE: &str = "size=128m,mode=0700";
+    const ABSENT_SCANNER_OBSERVATION_TIMEOUT_SECS: u64 = 180;
     const REPLACEMENT_RECOVERY_DIR: &str = ".rustfs.sys/buckets/ahm-replacement";
     const REPLACEMENT_INTENT_SUFFIX: &str = "_ahm_replacement_intent.json";
     const REPLACEMENT_COMPLETION_PROOF_SUFFIX: &str = "_ahm_replacement_completion_proof.json";
@@ -142,23 +142,6 @@ mod tests {
             run_command("dmsetup", &["resume", &self.dm_name])
         }
 
-        fn verify_raw_io_is_unavailable(&self) -> Result<(), Box<dyn Error + Send + Sync>> {
-            let mapper = format!("/dev/mapper/{}", self.dm_name);
-            let output = Command::new("dd")
-                .env("LC_ALL", "C")
-                .arg(format!("if={mapper}"))
-                .args(["of=/dev/null", "bs=4096", "count=1", "iflag=direct", "status=none"])
-                .output()?;
-            if output.status.success() {
-                return Err(format!("dm-error target unexpectedly allowed a raw read from {mapper}").into());
-            }
-            let stderr = String::from_utf8_lossy(&output.stderr);
-            if !stderr.contains("Input/output error") {
-                return Err(format!("raw read from dm-error target failed unexpectedly: {stderr}").into());
-            }
-            Ok(())
-        }
-
         fn restore_available(&self) -> Result<(), Box<dyn Error + Send + Sync>> {
             let sectors = run_command_stdout("blockdev", &["--getsz", &self.loop_device])?;
             let linear_table = format!("0 {sectors} linear {} 0", self.loop_device);
@@ -211,72 +194,6 @@ mod tests {
         }
     }
 
-    struct ZramBlockMount {
-        target: PathBuf,
-        device: String,
-        mounted: bool,
-    }
-
-    impl ZramBlockMount {
-        fn reserve(target: &Path) -> Result<Self, Box<dyn Error + Send + Sync>> {
-            if !Path::new("/dev/zram-control").exists() {
-                run_command("modprobe", &["zram"])?;
-            }
-            let device = run_command_stdout("zramctl", &["--find", "--size", "256M"])?;
-            if device.is_empty() {
-                return Err("zramctl --find --size returned an empty device".into());
-            }
-
-            Ok(Self {
-                target: target.to_path_buf(),
-                device,
-                mounted: false,
-            })
-        }
-
-        fn mount_target(&mut self) -> Result<(), Box<dyn Error + Send + Sync>> {
-            let result = (|| {
-                run_command("mkfs.ext4", &["-F", &self.device])?;
-                let target_arg = path_to_string(&self.target, "zram replacement mount target")?;
-                run_command("mount", &[&self.device, &target_arg])
-            })();
-            if let Err(error) = result {
-                let _ = self.cleanup();
-                return Err(error);
-            }
-            self.mounted = true;
-            Ok(())
-        }
-
-        fn cleanup(&mut self) -> Result<(), Box<dyn Error + Send + Sync>> {
-            let mut first_error: Option<Box<dyn Error + Send + Sync>> = None;
-            if self.mounted {
-                if let Err(error) = detach_mount(&self.target) {
-                    first_error.get_or_insert(error);
-                } else {
-                    self.mounted = false;
-                }
-            }
-            if !self.device.is_empty() {
-                if let Err(error) = run_command("zramctl", &["--reset", &self.device]) {
-                    first_error.get_or_insert(error);
-                } else {
-                    self.device.clear();
-                }
-            }
-            if let Some(error) = first_error {
-                return Err(error);
-            }
-            Ok(())
-        }
-    }
-
-    impl Drop for ZramBlockMount {
-        fn drop(&mut self) {
-            let _ = self.cleanup();
-        }
-    }
-
     fn checked_command_output(program: &str, args: &[&str]) -> Result<std::process::Output, Box<dyn Error + Send + Sync>> {
         let output = Command::new(program).args(args).output()?;
         if output.status.success() {
@@ -381,18 +298,6 @@ mod tests {
         Err(format!("{ENABLE_ENV}=1 requires root or CAP_SYS_ADMIN; unshare exited with status {status}").into())
     }
 
-    fn replacement_node_log_path(
-        cluster_temp_dir: &str,
-        parity: usize,
-        node_index: usize,
-    ) -> Result<PathBuf, Box<dyn Error + Send + Sync>> {
-        let log_dir = std::env::var_os(LOG_DIR_ENV)
-            .map(PathBuf::from)
-            .unwrap_or_else(|| PathBuf::from(cluster_temp_dir));
-        fs::create_dir_all(&log_dir)?;
-        Ok(log_dir.join(format!("replacement-ec{parity}-node{node_index}-{}.log", std::process::id())))
-    }
-
     fn payload(len: usize, seed: u8) -> Vec<u8> {
         let mut next = seed;
         (0..len)
@@ -567,20 +472,8 @@ mod tests {
             if let Some(version_id) = &version.version_id {
                 request = request.version_id(version_id);
             }
-            let response = request.send().await.map_err(|error| {
-                format!("body GET failed for {}/{}@{:?}: {error}", version.bucket, version.key, version.version_id)
-            })?;
-            let body = response
-                .body
-                .collect()
-                .await
-                .map_err(|error| {
-                    format!(
-                        "body stream failed for {}/{}@{:?}: {error}",
-                        version.bucket, version.key, version.version_id
-                    )
-                })?
-                .into_bytes();
+            let response = request.send().await?;
+            let body = response.body.collect().await?.into_bytes();
             assert_eq!(
                 sha256_hex(&body),
                 *expected_sha256,
@@ -689,6 +582,81 @@ mod tests {
         Ok(())
     }
 
+    fn log_tail(log: &str) -> String {
+        let mut lines = log.lines().rev().take(80).collect::<Vec<_>>();
+        lines.reverse();
+        lines.join("\n")
+    }
+
+    fn log_len(path: &Path) -> Result<u64, Box<dyn Error + Send + Sync>> {
+        match fs::metadata(path) {
+            Ok(metadata) => Ok(metadata.len()),
+            Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(0),
+            Err(error) => Err(format!("failed to stat target node log {path:?}: {error}").into()),
+        }
+    }
+
+    fn log_from_offset(path: &Path, offset: u64) -> Result<String, Box<dyn Error + Send + Sync>> {
+        let log = match fs::read(path) {
+            Ok(log) => log,
+            Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(),
+            Err(error) => return Err(format!("failed to read target node log {path:?}: {error}").into()),
+        };
+        let start = usize::try_from(offset).unwrap_or(usize::MAX).min(log.len());
+        Ok(String::from_utf8_lossy(&log[start..]).into_owned())
+    }
+
+    fn live_disk_loss_scan_completed(log: &str, target_disk: &Path) -> bool {
+        let target = target_disk.to_string_lossy();
+        let mut saw_live_loss = false;
+        for line in log.lines() {
+            if line.contains("Heal auto-scan disk inspection failed")
+                && line.contains("check_failed")
+                && line.contains(target.as_ref())
+            {
+                saw_live_loss = true;
+                continue;
+            }
+            if saw_live_loss && (line.contains("Heal auto disk scanner idle") || line.contains("Heal auto-scan cycle completed"))
+            {
+                return true;
+            }
+        }
+        false
+    }
+
+    fn live_disk_loss_scan_completed_from_path(
+        log_path: &Path,
+        start_offset: u64,
+        target_disk: &Path,
+    ) -> Result<bool, Box<dyn Error + Send + Sync>> {
+        Ok(live_disk_loss_scan_completed(&log_from_offset(log_path, start_offset)?, target_disk))
+    }
+
+    async fn wait_for_live_disk_loss_observation(
+        log_path: &Path,
+        target_disk: &Path,
+        start_offset: u64,
+        timeout_secs: u64,
+    ) -> Result<(), Box<dyn Error + Send + Sync>> {
+        let deadline = Instant::now() + Duration::from_secs(timeout_secs);
+        let mut tick = interval(Duration::from_secs(1));
+        loop {
+            if live_disk_loss_scan_completed_from_path(log_path, start_offset, target_disk)? {
+                return Ok(());
+            }
+            if Instant::now() >= deadline {
+                let log = log_from_offset(log_path, start_offset)?;
+                return Err(format!(
+                    "scanner did not finish a live target-loss scan for {target_disk:?} within {timeout_secs}s; log tail:\n{}",
+                    log_tail(&log)
+                )
+                .into());
+            }
+            tick.tick().await;
+        }
+    }
+
     fn cluster_status_is_definitive(status: &serde_json::Value) -> Result<bool, Box<dyn Error + Send + Sync>> {
         status["cluster"]["definitive"]
             .as_bool()
@@ -739,13 +707,6 @@ mod tests {
             .collect()
     }
 
-    fn is_transient_recovery_version_absence(error: &(dyn Error + 'static)) -> bool {
-        matches!(
-            error.downcast_ref::<rustfs_filemeta::Error>(),
-            Some(rustfs_filemeta::Error::FileVersionNotFound)
-        )
-    }
-
     fn incomplete_versions(
         target_disk: &Path,
         versions: &[BaselineVersion],
@@ -753,21 +714,7 @@ mod tests {
         let mut missing = BTreeSet::new();
         for version in versions {
             let actual =
-                match census_object_version_on_disk(target_disk, &version.bucket, &version.key, version.version_id.as_deref()) {
-                    Ok(actual) => actual,
-                    // During replacement recovery, xl.meta may arrive before this
-                    // particular historical version. The generic census helper
-                    // correctly reports that as an error; this progress poll must
-                    // instead wait for the version to be restored.
-                    Err(error) if is_transient_recovery_version_absence(error.as_ref()) => {
-                        missing.insert(format!(
-                            "{}/{}@{:?}: version metadata not yet present on replacement",
-                            version.bucket, version.key, version.version_id
-                        ));
-                        continue;
-                    }
-                    Err(error) => return Err(error),
-                };
+                census_object_version_on_disk(target_disk, &version.bucket, &version.key, version.version_id.as_deref())?;
             if !actual.matches_manifest(&version.expected) {
                 missing.insert(format!("{}/{}@{:?}: {actual:?}", version.bucket, version.key, version.version_id));
             }
@@ -875,15 +822,13 @@ mod tests {
 
         let mut mount_ns = MountNamespaceGuard::new()?;
         let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(3, 4)).await?;
-        for node_index in 0..cluster.nodes.len() {
-            let node_log_path = replacement_node_log_path(&cluster.temp_dir, parity, node_index)?;
-            cluster.set_node_capture_log_path(node_index, node_log_path.to_string_lossy())?;
-        }
+        let target_log_path = PathBuf::from(&cluster.temp_dir).join(format!("replacement-node{TARGET_NODE}.log"));
+        cluster.set_node_capture_log_path(TARGET_NODE, target_log_path.to_string_lossy())?;
         let target_disk = PathBuf::from(&cluster.nodes[TARGET_NODE].data_dirs[TARGET_DRIVE]);
-        // The blank target uses a temporary zram block device, so the
-        // replacement readiness fence sees no root or sibling alias.
+        // Each drive below is an independent tmpfs mount, so this privileged
+        // path must exercise the production distinct-device/readiness fences.
         cluster.extra_env.retain(|(key, _)| key != "RUSTFS_UNSAFE_BYPASS_DISK_CHECK");
-        let image_root = PathBuf::from(&cluster.temp_dir).join("replacement-block-images");
+        let image_root = PathBuf::from(&cluster.temp_dir).join("replacement-faultable-images");
         let mut target_mount = None;
         for (node_index, node) in cluster.nodes.iter().enumerate() {
             for (drive_index, drive) in node.data_dirs.iter().enumerate() {
@@ -900,7 +845,6 @@ mod tests {
             }
         }
         let mut target_mount = target_mount.ok_or("target drive was not mounted with the faultable block fixture")?;
-        let mut replacement_mount = ZramBlockMount::reserve(&target_disk)?;
 
         cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
         cluster.set_env("RUSTFS_SCANNER_ENABLED", "true");
@@ -908,34 +852,28 @@ mod tests {
         cluster.set_env("RUSTFS_SCANNER_CYCLE", "1");
         cluster.set_env("RUSTFS_SCANNER_START_DELAY_SECS", "0");
         cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{parity}"));
-        for node_index in 0..cluster.nodes.len() {
-            cluster.set_node_env(node_index, "RUST_LOG", "rustfs=info,rustfs::heal::manager=debug,rustfs_notify=debug")?;
-        }
+        cluster.set_node_env(TARGET_NODE, "RUST_LOG", "rustfs=info,rustfs::heal::manager=debug,rustfs_notify=debug")?;
         cluster.start().await?;
 
         let clients = cluster.create_all_clients()?;
-        let versions = seed_baseline(&clients[0], &target_disk)
-            .await
-            .map_err(|error| format!("pre-fault baseline seeding failed: {error}"))?;
-        verify_bodies(&clients[0], &versions)
-            .await
-            .map_err(|error| format!("pre-fault body verification failed: {error}"))?;
+        let versions = seed_baseline(&clients[0], &target_disk).await?;
+        verify_bodies(&clients[0], &versions).await?;
 
-        target_mount
-            .make_unavailable()
-            .map_err(|error| format!("failed to install the dm-error target: {error}"))?;
-        target_mount
-            .verify_raw_io_is_unavailable()
-            .map_err(|error| format!("dm-error target was not proven by a direct raw read: {error}"))?;
-        assert_no_replacement_status_records(&cluster, &target_disk)
-            .await
-            .map_err(|error| format!("live-fault replacement status check failed: {error}"))?;
-        assert_no_replacement_admission_artifacts(&cluster, &target_disk)
-            .map_err(|error| format!("live-fault replacement artifact check failed: {error}"))?;
+        let live_loss_log_offset = log_len(&target_log_path)?;
+        target_mount.make_unavailable()?;
+        wait_for_live_disk_loss_observation(
+            &target_log_path,
+            &target_disk,
+            live_loss_log_offset,
+            ABSENT_SCANNER_OBSERVATION_TIMEOUT_SECS,
+        )
+        .await?;
+        assert_no_replacement_status_records(&cluster, &target_disk).await?;
+        assert_no_replacement_admission_artifacts(&cluster, &target_disk)?;
 
-        cluster.stop_node_gracefully(TARGET_NODE).await?;
+        cluster.stop_node(TARGET_NODE)?;
         target_mount.cleanup()?;
-        replacement_mount.mount_target()?;
+        mount_ns.mount_tmpfs(&target_disk, &format!("rustfs-e2e-p{parity}-replacement"))?;
         let missing_before_restart = incomplete_versions(&target_disk, &versions)?;
         assert_eq!(
             missing_before_restart.len(),
@@ -944,26 +882,46 @@ mod tests {
         );
         cluster.start_node(TARGET_NODE).await?;
 
-        let recovery_result = async {
-            wait_for_completed_replacement_with_census(&cluster, &target_disk, &versions, 420).await?;
-            verify_bodies(&clients[0], &versions).await
-        }
-        .await;
-        let stop_result = cluster.stop_node_gracefully(TARGET_NODE).await;
-        let replacement_cleanup_result = replacement_mount.cleanup();
+        wait_for_completed_replacement_with_census(&cluster, &target_disk, &versions, 420).await?;
+        verify_bodies(&clients[0], &versions).await?;
 
-        if let Err(error) = recovery_result {
-            if let Err(stop_error) = stop_result {
-                info!(%stop_error, "replacement target stop failed while preserving recovery failure");
-            }
-            if let Err(cleanup_error) = replacement_cleanup_result {
-                info!(%cleanup_error, "replacement zram cleanup failed while preserving recovery failure");
-            }
-            return Err(error);
-        }
-        stop_result?;
-        replacement_cleanup_result?;
+        Ok(())
+    }
 
+    #[test]
+    fn live_loss_barrier_requires_scanner_failure_after_log_offset() -> Result<(), Box<dyn Error + Send + Sync>> {
+        let target = Path::new("/mnt/target");
+        assert!(live_disk_loss_scan_completed(
+            "Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto-scan cycle completed",
+            target
+        ));
+        assert!(live_disk_loss_scan_completed(
+            "Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle",
+            target
+        ));
+        assert!(!live_disk_loss_scan_completed(
+            "Heal auto disk scanner idle\nHeal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed",
+            target
+        ));
+        assert!(!live_disk_loss_scan_completed(
+            "event=disk_health_check_failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle",
+            target
+        ));
+        assert!(!live_disk_loss_scan_completed(
+            "Heal auto-scan disk inspection failed endpoint=/mnt/other disk_state=check_failed\nHeal auto disk scanner idle",
+            target
+        ));
+        let path = std::env::temp_dir().join(format!("rustfs-replacement-scan-{}.log", std::process::id()));
+        let stale =
+            "Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle\n";
+        fs::write(&path, stale)?;
+        let offset = log_len(&path)?;
+        assert!(!live_disk_loss_scan_completed_from_path(&path, offset, target)?);
+        let fresh =
+            "Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle\n";
+        fs::write(&path, format!("{stale}{fresh}"))?;
+        assert!(live_disk_loss_scan_completed_from_path(&path, offset, target)?);
+        fs::remove_file(path)?;
         Ok(())
     }
 
@@ -996,15 +954,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn recovery_census_only_treats_missing_version_as_transient() {
-        let missing_version: Box<dyn Error + Send + Sync> = Box::new(rustfs_filemeta::Error::FileVersionNotFound);
-        let missing_file: Box<dyn Error + Send + Sync> = Box::new(rustfs_filemeta::Error::FileNotFound);
-
-        assert!(is_transient_recovery_version_absence(missing_version.as_ref()));
-        assert!(!is_transient_recovery_version_absence(missing_file.as_ref()));
-    }
-
     #[tokio::test]
     async fn completion_poll_samples_census_before_status() {
         let order = std::rc::Rc::new(std::cell::RefCell::new(Vec::new()));

crates/e2e_test/src/snowball_auto_extract_test.rs

@@ -17,56 +17,8 @@ mod tests {
     use crate::common::{RustFSTestEnvironment, init_logging};
     use aws_sdk_s3::error::ProvideErrorMetadata;
     use aws_sdk_s3::primitives::ByteStream;
-    use flate2::{Compression, write::GzEncoder};
     use std::error::Error;
-    use std::io::{Cursor, Write};
-
-    fn pax_record(key: &str, value: &str) -> Vec<u8> {
-        let payload = format!("{key}={value}\n");
-        let mut len = payload.len() + 3;
-        loop {
-            let record = format!("{len} {payload}");
-            if record.len() == len {
-                return record.into_bytes();
-            }
-            len = record.len();
-        }
-    }
-
-    async fn append_pax_header(
-        builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>,
-        entry_type: tokio_tar::EntryType,
-        records: &[(&str, &str)],
-    ) -> Result<(), Box<dyn Error + Send + Sync>> {
-        let mut payload = Vec::new();
-        for (key, value) in records {
-            payload.extend(pax_record(key, value));
-        }
-        let mut header = tokio_tar::Header::new_ustar();
-        header.set_entry_type(entry_type);
-        header.set_size(u64::try_from(payload.len()).expect("PAX payload length should fit in u64"));
-        header.set_mode(0o644);
-        header.set_cksum();
-        builder
-            .append_data(&mut header, "PaxHeaders.X/snowball", Cursor::new(payload))
-            .await?;
-        Ok(())
-    }
-
-    async fn append_typed_entry(
-        builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>,
-        path: &str,
-        entry_type: tokio_tar::EntryType,
-        body: &[u8],
-    ) -> Result<(), Box<dyn Error + Send + Sync>> {
-        let mut header = tokio_tar::Header::new_gnu();
-        header.set_entry_type(entry_type);
-        header.set_size(u64::try_from(body.len()).expect("TAR member length should fit in u64"));
-        header.set_mode(0o644);
-        header.set_cksum();
-        builder.append_data(&mut header, path, Cursor::new(body)).await?;
-        Ok(())
-    }
+    use std::io::Cursor;
 
     async fn build_test_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
         let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
@@ -117,50 +69,12 @@ mod tests {
         Ok(builder.into_inner().await?.into_inner())
     }
 
-    async fn build_archive_with_invalid_checksum() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
-        let mut archive = build_test_archive().await?;
-        archive[0] ^= 1;
-        Ok(archive)
-    }
-
-    async fn build_archive_with_negative_gnu_mtime() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
-        let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
-        let mut header = tokio_tar::Header::new_gnu();
-        header.set_size(b"negative-mtime-body".len() as u64);
-        header.set_mode(0o644);
-        header.as_old_mut().mtime.fill(0xff);
-        builder
-            .append_data(&mut header, "negative-mtime.txt", Cursor::new(b"negative-mtime-body".as_slice()))
-            .await?;
-        Ok(builder.into_inner().await?.into_inner())
-    }
-
-    fn gzip_member(payload: &[u8]) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
-        let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
-        encoder.write_all(payload)?;
-        Ok(encoder.finish()?)
-    }
-
-    async fn build_concatenated_gzip_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
-        let archive = build_test_archive().await?;
-        let split_at = archive.len() / 2;
-        let mut encoded = gzip_member(&archive[..split_at])?;
-        encoded.extend(gzip_member(&archive[split_at..])?);
-        Ok(encoded)
-    }
-
-    async fn build_gzip_archive_with_invalid_crc() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
-        let mut encoded = gzip_member(&build_test_archive().await?)?;
-        let crc_offset = encoded.len().checked_sub(8).expect("gzip fixture must contain a trailer");
-        encoded[crc_offset] ^= 1;
-        Ok(encoded)
-    }
-
-    fn append_raw_tar_entry_with_type(archive: &mut Vec<u8>, path: &[u8], data: &[u8], entry_type: u8) {
-        assert!(path.len() <= 100, "raw TAR fixture path must fit in the name field");
+    fn build_archive_with_parent_dir_entry(victim_bucket: &str) -> Vec<u8> {
+        let path = format!("../{victim_bucket}/evil-injected.txt");
+        let data = b"injected-body";
         let mut header = [0u8; 512];
 
-        header[..path.len()].copy_from_slice(path);
+        header[..path.len()].copy_from_slice(path.as_bytes());
         header[100..108].copy_from_slice(b"0000644\0");
         header[108..116].copy_from_slice(b"0000000\0");
         header[116..124].copy_from_slice(b"0000000\0");
@@ -168,7 +82,7 @@ mod tests {
         header[124..136].copy_from_slice(size.as_bytes());
         header[136..148].copy_from_slice(b"00000000000\0");
         header[148..156].fill(b' ');
-        header[156] = entry_type;
+        header[156] = b'0';
         header[257..263].copy_from_slice(b"ustar\0");
         header[263..265].copy_from_slice(b"00");
 
@@ -176,87 +90,11 @@ mod tests {
         let checksum = format!("{:06o}\0 ", checksum);
         header[148..156].copy_from_slice(checksum.as_bytes());
 
+        let mut archive = Vec::new();
         archive.extend_from_slice(&header);
         archive.extend_from_slice(data);
         let padding = (512 - (data.len() % 512)) % 512;
         archive.extend(std::iter::repeat_n(0, padding));
-    }
-
-    fn append_raw_tar_entry(archive: &mut Vec<u8>, path: &[u8], data: &[u8]) {
-        append_raw_tar_entry_with_type(archive, path, data, b'0');
-    }
-
-    fn build_archive_with_parent_dir_entry(victim_bucket: &str) -> Vec<u8> {
-        let path = format!("../{victim_bucket}/evil-injected.txt");
-        let mut archive = Vec::new();
-        append_raw_tar_entry(&mut archive, path.as_bytes(), b"injected-body");
-        archive.extend_from_slice(&[0u8; 1024]);
-        archive
-    }
-
-    async fn build_member_semantics_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
-        let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
-        append_pax_header(
-            &mut builder,
-            tokio_tar::EntryType::XGlobalHeader,
-            &[
-                ("minio.metadata.x-amz-meta-owner", "global"),
-                ("minio.metadata.x-amz-meta-snowball-auto-extract", "true"),
-            ],
-        )
-        .await?;
-        append_pax_header(
-            &mut builder,
-            tokio_tar::EntryType::XHeader,
-            &[("minio.metadata.x-amz-meta-owner", "local")],
-        )
-        .await?;
-        append_typed_entry(&mut builder, "regular.txt", tokio_tar::EntryType::Regular, b"regular-body").await?;
-        for (path, entry_type) in [
-            ("char", tokio_tar::EntryType::Char),
-            ("block", tokio_tar::EntryType::Block),
-            ("fifo", tokio_tar::EntryType::Fifo),
-        ] {
-            append_typed_entry(&mut builder, path, entry_type, b"").await?;
-        }
-        let mut directory = tokio_tar::Header::new_gnu();
-        directory.set_entry_type(tokio_tar::EntryType::Directory);
-        directory.set_size(0);
-        directory.set_mode(0o755);
-        directory.set_cksum();
-        builder
-            .append_data(&mut directory, "directory/", Cursor::new(Vec::new()))
-            .await?;
-        for (path, entry_type) in [
-            ("hard-link", tokio_tar::EntryType::Link),
-            ("symlink", tokio_tar::EntryType::Symlink),
-            ("continuous", tokio_tar::EntryType::Continuous),
-            ("unknown", tokio_tar::EntryType::Other(b'9')),
-        ] {
-            append_typed_entry(&mut builder, path, entry_type, b"").await?;
-        }
-        Ok(builder.into_inner().await?.into_inner())
-    }
-
-    async fn build_versioned_member_archive(path: &str, version_id: &str) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
-        let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
-        append_pax_header(&mut builder, tokio_tar::EntryType::XHeader, &[("minio.versionId", version_id)]).await?;
-        append_typed_entry(&mut builder, path, tokio_tar::EntryType::Regular, b"versioned-body").await?;
-        Ok(builder.into_inner().await?.into_inner())
-    }
-
-    fn build_archive_with_invalid_utf8_entry() -> Vec<u8> {
-        let mut archive = Vec::new();
-        append_raw_tar_entry(&mut archive, b"invalid-\xff.txt", b"ignored-body");
-        append_raw_tar_entry(&mut archive, b"valid.txt", b"valid-body");
-        archive.extend_from_slice(&[0u8; 1024]);
-        archive
-    }
-
-    fn build_archive_with_invalid_utf8_symlink() -> Vec<u8> {
-        let mut archive = Vec::new();
-        append_raw_tar_entry_with_type(&mut archive, b"invalid-\xff-link", b"", b'2');
-        append_raw_tar_entry(&mut archive, b"valid.txt", b"valid-body");
         archive.extend_from_slice(&[0u8; 1024]);
         archive
     }
@@ -297,147 +135,6 @@ mod tests {
         Ok(())
     }
 
-    #[tokio::test]
-    async fn snowball_auto_extract_applies_member_semantics_and_metadata_precedence() -> Result<(), Box<dyn Error + Send + Sync>>
-    {
-        init_logging();
-
-        let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server(vec![]).await?;
-
-        let client = env.create_s3_client();
-        let bucket = "snowball-member-semantics";
-        client.create_bucket().bucket(bucket).send().await?;
-        client
-            .put_object()
-            .bucket(bucket)
-            .key("fixture.tar")
-            .metadata("Snowball-Auto-Extract", "true")
-            .metadata("Minio-Snowball-Prefix", "members")
-            .metadata("owner", "outer")
-            .body(ByteStream::from(build_member_semantics_archive().await?))
-            .send()
-            .await?;
-
-        let regular = client.head_object().bucket(bucket).key("members/regular.txt").send().await?;
-        let regular_metadata = regular.metadata().expect("regular member should expose metadata");
-        assert_eq!(regular_metadata.get("owner").map(String::as_str), Some("local"));
-        assert!(!regular_metadata.contains_key("snowball-auto-extract"));
-        assert!(!regular_metadata.contains_key("minio-snowball-prefix"));
-
-        for key in ["char", "block", "fifo"] {
-            let head = client
-                .head_object()
-                .bucket(bucket)
-                .key(format!("members/{key}"))
-                .send()
-                .await?;
-            assert_eq!(head.content_length(), Some(0), "{key} should be materialized as an empty object");
-            assert_eq!(
-                head.metadata().and_then(|metadata| metadata.get("owner")).map(String::as_str),
-                Some("outer"),
-                "{key} should not inherit global PAX metadata"
-            );
-        }
-        let directory = client.head_object().bucket(bucket).key("members/directory/").send().await?;
-        assert_eq!(directory.content_length(), Some(0));
-
-        for key in ["hard-link", "symlink", "continuous", "unknown"] {
-            let error = client
-                .head_object()
-                .bucket(bucket)
-                .key(format!("members/{key}"))
-                .send()
-                .await
-                .expect_err("unsupported TAR entry type must be skipped");
-            assert_eq!(error.into_service_error().code(), Some("NotFound"), "{key}");
-        }
-
-        env.stop_server();
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn snowball_auto_extract_validates_pax_version_id_against_bucket_state() -> Result<(), Box<dyn Error + Send + Sync>> {
-        init_logging();
-
-        let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server(vec![]).await?;
-
-        let client = env.create_s3_client();
-        let bucket = "snowball-version-semantics";
-        client.create_bucket().bucket(bucket).send().await?;
-
-        client
-            .put_object()
-            .bucket(bucket)
-            .key("null.tar")
-            .metadata("Snowball-Auto-Extract", "true")
-            .body(ByteStream::from(build_versioned_member_archive("null.txt", "null").await?))
-            .send()
-            .await?;
-        let null_member = client.get_object().bucket(bucket).key("null.txt").send().await?;
-        assert_eq!(null_member.body.collect().await?.into_bytes().as_ref(), b"versioned-body");
-
-        for (archive_key, member_key, version_id) in [
-            ("uuid.tar", "uuid.txt", uuid::Uuid::new_v4().to_string()),
-            ("uppercase-null.tar", "uppercase-null.txt", "NULL".to_string()),
-        ] {
-            let error = client
-                .put_object()
-                .bucket(bucket)
-                .key(archive_key)
-                .metadata("Snowball-Auto-Extract", "true")
-                .body(ByteStream::from(build_versioned_member_archive(member_key, &version_id).await?))
-                .send()
-                .await
-                .expect_err("invalid or unversioned UUID import must be rejected");
-            assert_eq!(error.into_service_error().code(), Some("InvalidArgument"), "{archive_key}");
-            let missing = client
-                .head_object()
-                .bucket(bucket)
-                .key(member_key)
-                .send()
-                .await
-                .expect_err("rejected version import must not create an object");
-            assert_eq!(missing.into_service_error().code(), Some("NotFound"), "{member_key}");
-        }
-
-        client
-            .put_bucket_versioning()
-            .bucket(bucket)
-            .versioning_configuration(
-                aws_sdk_s3::types::VersioningConfiguration::builder()
-                    .status(aws_sdk_s3::types::BucketVersioningStatus::Enabled)
-                    .build(),
-            )
-            .send()
-            .await?;
-        let imported_version_id = uuid::Uuid::new_v4().to_string();
-        client
-            .put_object()
-            .bucket(bucket)
-            .key("versioned-uuid.tar")
-            .metadata("Snowball-Auto-Extract", "true")
-            .body(ByteStream::from(
-                build_versioned_member_archive("versioned-uuid.txt", &imported_version_id).await?,
-            ))
-            .send()
-            .await?;
-        let imported = client
-            .get_object()
-            .bucket(bucket)
-            .key("versioned-uuid.txt")
-            .version_id(&imported_version_id)
-            .send()
-            .await?;
-        assert_eq!(imported.version_id(), Some(imported_version_id.as_str()));
-        assert_eq!(imported.body.collect().await?.into_bytes().as_ref(), b"versioned-body");
-
-        env.stop_server();
-        Ok(())
-    }
-
     #[tokio::test]
     async fn snowball_auto_extract_supports_standard_headers_with_combined_extract_options()
     -> Result<(), Box<dyn Error + Send + Sync>> {
@@ -566,113 +263,6 @@ mod tests {
         Ok(())
     }
 
-    #[tokio::test]
-    async fn snowball_auto_extract_accepts_negative_gnu_mtime() -> Result<(), Box<dyn Error + Send + Sync>> {
-        init_logging();
-
-        let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server(vec![]).await?;
-
-        let client = env.create_s3_client();
-        let bucket = "snowball-negative-mtime";
-        client.create_bucket().bucket(bucket).send().await?;
-        client
-            .put_object()
-            .bucket(bucket)
-            .key("fixture.tar")
-            .metadata("Snowball-Auto-Extract", "true")
-            .body(ByteStream::from(build_archive_with_negative_gnu_mtime().await?))
-            .send()
-            .await?;
-
-        let object = client.get_object().bucket(bucket).key("negative-mtime.txt").send().await?;
-        assert_eq!(object.body.collect().await?.into_bytes().as_ref(), b"negative-mtime-body");
-
-        env.stop_server();
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn snowball_auto_extract_consumes_concatenated_gzip_members() -> Result<(), Box<dyn Error + Send + Sync>> {
-        init_logging();
-
-        let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server(vec![]).await?;
-
-        let client = env.create_s3_client();
-        let bucket = "snowball-concatenated-gzip";
-        client.create_bucket().bucket(bucket).send().await?;
-        client
-            .put_object()
-            .bucket(bucket)
-            .key("fixture.tar.gz")
-            .metadata("Snowball-Auto-Extract", "true")
-            .body(ByteStream::from(build_concatenated_gzip_archive().await?))
-            .send()
-            .await?;
-
-        let object = client.get_object().bucket(bucket).key("root.txt").send().await?;
-        assert_eq!(object.body.collect().await?.into_bytes().as_ref(), b"root payload\n");
-
-        env.stop_server();
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn snowball_auto_extract_rejects_gzip_crc_error_when_ignore_errors_enabled() -> Result<(), Box<dyn Error + Send + Sync>>
-    {
-        init_logging();
-
-        let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server(vec![]).await?;
-
-        let client = env.create_s3_client();
-        let bucket = "snowball-gzip-crc-ignore-errors";
-        client.create_bucket().bucket(bucket).send().await?;
-        let err = client
-            .put_object()
-            .bucket(bucket)
-            .key("fixture.tar.gz")
-            .metadata("Snowball-Auto-Extract", "true")
-            .metadata("Minio-Snowball-Ignore-Errors", "true")
-            .body(ByteStream::from(build_gzip_archive_with_invalid_crc().await?))
-            .send()
-            .await
-            .expect_err("gzip integrity failures must remain fatal under ignore-errors");
-
-        assert_eq!(err.into_service_error().code(), Some("InvalidArgument"));
-
-        env.stop_server();
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn snowball_auto_extract_rejects_mismatched_content_md5() -> Result<(), Box<dyn Error + Send + Sync>> {
-        init_logging();
-
-        let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server(vec![]).await?;
-
-        let client = env.create_s3_client();
-        let bucket = "snowball-content-md5";
-        client.create_bucket().bucket(bucket).send().await?;
-        let err = client
-            .put_object()
-            .bucket(bucket)
-            .key("fixture.tar")
-            .metadata("Snowball-Auto-Extract", "true")
-            .content_md5("AAAAAAAAAAAAAAAAAAAAAA==")
-            .body(ByteStream::from(build_test_archive().await?))
-            .send()
-            .await
-            .expect_err("mismatched Content-MD5 must fail after the raw body reaches EOF");
-
-        assert_eq!(err.into_service_error().code(), Some("BadDigest"));
-
-        env.stop_server();
-        Ok(())
-    }
-
     #[tokio::test]
     async fn snowball_auto_extract_ignores_invalid_entries_when_requested() -> Result<(), Box<dyn Error + Send + Sync>> {
         init_logging();
@@ -709,100 +299,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn snowball_auto_extract_skips_non_utf8_symlink_without_ignore_errors() -> Result<(), Box<dyn Error + Send + Sync>> {
-        init_logging();
-
-        let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server(vec![]).await?;
-
-        let client = env.create_s3_client();
-        let bucket = "snowball-invalid-utf8-link";
-        client.create_bucket().bucket(bucket).send().await?;
-
-        client
-            .put_object()
-            .bucket(bucket)
-            .key("fixture.tar")
-            .metadata("Snowball-Auto-Extract", "true")
-            .body(ByteStream::from(build_archive_with_invalid_utf8_symlink()))
-            .send()
-            .await?;
-
-        let valid = client.get_object().bucket(bucket).key("valid.txt").send().await?;
-        assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"valid-body");
-        let listed = client.list_objects_v2().bucket(bucket).send().await?;
-        let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect();
-        assert_eq!(keys, vec!["valid.txt"]);
-
-        env.stop_server();
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn snowball_auto_extract_skips_non_utf8_member_without_lossy_key_collision() -> Result<(), Box<dyn Error + Send + Sync>>
-    {
-        init_logging();
-
-        let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server(vec![]).await?;
-
-        let client = env.create_s3_client();
-        let bucket = "snowball-invalid-utf8";
-        client.create_bucket().bucket(bucket).send().await?;
-
-        client
-            .put_object()
-            .bucket(bucket)
-            .key("fixture.tar")
-            .metadata("Snowball-Auto-Extract", "true")
-            .metadata("Minio-Snowball-Ignore-Errors", "true")
-            .body(ByteStream::from(build_archive_with_invalid_utf8_entry()))
-            .send()
-            .await?;
-
-        let valid = client.get_object().bucket(bucket).key("valid.txt").send().await?;
-        assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"valid-body");
-        let listed = client.list_objects_v2().bucket(bucket).send().await?;
-        let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect();
-        assert_eq!(keys, vec!["valid.txt"]);
-
-        env.stop_server();
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn snowball_auto_extract_rejects_corrupt_tar_when_ignore_errors_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
-        init_logging();
-
-        let mut env = RustFSTestEnvironment::new().await?;
-        env.start_rustfs_server(vec![]).await?;
-
-        let client = env.create_s3_client();
-        let bucket = "snowball-corrupt-ignore-errors";
-        let archive = build_archive_with_invalid_checksum().await?;
-        client.create_bucket().bucket(bucket).send().await?;
-
-        let err = client
-            .put_object()
-            .bucket(bucket)
-            .key("fixture.tar")
-            .metadata("Snowball-Auto-Extract", "true")
-            .metadata("Minio-Snowball-Ignore-Errors", "true")
-            .body(ByteStream::from(archive))
-            .send()
-            .await
-            .expect_err("corrupt TAR structure must remain fatal under ignore-errors");
-        assert_eq!(err.into_service_error().code(), Some("InvalidArgument"));
-
-        let listed = client.list_objects_v2().bucket(bucket).send().await?;
-        assert!(listed.contents().is_empty(), "corrupt archive must not produce objects");
-
-        env.stop_server();
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn snowball_auto_extract_rejects_parent_dir_entry_even_when_ignore_errors_enabled()
+    async fn snowball_auto_extract_rejects_parent_dir_entry_without_cross_bucket_write()
     -> Result<(), Box<dyn Error + Send + Sync>> {
         init_logging();
 
@@ -822,7 +319,6 @@ mod tests {
             .bucket(attacker_bucket)
             .key("fixture.tar")
             .metadata("Snowball-Auto-Extract", "true")
-            .metadata("Minio-Snowball-Ignore-Errors", "true")
             .body(ByteStream::from(archive))
             .send()
             .await

crates/e2e_test/src/storage_api.rs

@@ -15,7 +15,7 @@
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
 #[cfg(test)]
-pub(crate) use rustfs_ecstore::api::disk::{RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions};
+pub(crate) use rustfs_ecstore::api::disk::{VolumeInfo, WalkDirOptions};
 pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::rpc::{

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -12,17 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, rustfs_binary_path};
+use crate::common::{RustFSTestEnvironment, init_logging};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
     BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ServerSideEncryption, VersioningConfiguration,
 };
-use std::path::{Path, PathBuf};
-use std::time::Duration;
-use tokio::task::JoinSet;
-use tokio::time::{Instant, sleep};
+use std::path::PathBuf;
 
 type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
 
@@ -31,14 +28,6 @@ const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
 const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
 const PLAIN_BUCKET: &str = "upgrade-plain-data";
 const VERSIONED_BUCKET: &str = "upgrade-versioned-data";
-const MIXED_BUCKET: &str = "upgrade-mixed-version-data";
-const MIXED_NODE_COUNT: usize = 4;
-const MULTIPART_WORKERS: usize = 16;
-const MULTIPART_UPLOADS_PER_WORKER: usize = 16;
-// Peers keep a restarted node's drive in Suspect/Returning for roughly
-// probe_interval (2s) x success_threshold (3) after it comes back; 30s
-// comfortably covers that window plus CI scheduling jitter.
-const LISTING_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(30);
 
 fn source_binary() -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> {
     let path = std::env::var_os(SOURCE_BINARY_ENV)
@@ -114,132 +103,6 @@ async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec<
     Ok(())
 }
 
-fn configure_cluster_logs(cluster: &mut RustFSTestClusterEnvironment) -> TestResult {
-    let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else {
-        return Ok(());
-    };
-    std::fs::create_dir_all(&log_dir)?;
-    for node_idx in 0..cluster.nodes.len() {
-        let path = Path::new(&log_dir).join(format!("mixed-upgrade-node-{node_idx}.log"));
-        cluster.set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?;
-    }
-    Ok(())
-}
-
-async fn write_multipart_load(clients: &[Client], phase: &str) -> Result<Vec<String>, Box<dyn std::error::Error + Send + Sync>> {
-    let mut tasks = JoinSet::new();
-    for worker in 0..MULTIPART_WORKERS {
-        let client = clients[worker % clients.len()].clone();
-        let phase = phase.to_string();
-        tasks.spawn(async move {
-            let mut keys = Vec::with_capacity(MULTIPART_UPLOADS_PER_WORKER);
-            for upload in 0..MULTIPART_UPLOADS_PER_WORKER {
-                let key = format!("{phase}/multipart/{worker:02}/{upload:02}");
-                let part = vec![u8::try_from(worker)?; 64 * 1024];
-                write_multipart(&client, MIXED_BUCKET, &key, &[part]).await?;
-                keys.push(key);
-            }
-            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(keys)
-        });
-    }
-
-    let mut keys = Vec::with_capacity(MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER);
-    while let Some(result) = tasks.join_next().await {
-        keys.extend(result??);
-    }
-    Ok(keys)
-}
-
-/// Assert that `client` eventually lists exactly `expected` objects under
-/// `{phase}/`, polling until [`LISTING_CONVERGENCE_TIMEOUT`].
-///
-/// A single-snapshot assertion here is racy by construction: each phase both
-/// writes and lists within seconds of a node restart. While a peer still holds
-/// the restarted node's drive in Suspect/Returning, strict-quorum listing
-/// consults only the remaining three drives and drops any object that was
-/// itself legally written at write quorum (3/4 drives) during an earlier
-/// node's identical post-restart window — its xl.meta is then visible on only
-/// two of the three consulted drives, below the required object quorum of
-/// three. GET still succeeds for such objects; only the listing under-counts
-/// until drive health converges. A genuine upgrade data-loss regression still
-/// fails after the deadline.
-async fn wait_for_phase_listing(client: &Client, phase: &str, expected: usize, context: &str) -> TestResult {
-    let deadline = Instant::now() + LISTING_CONVERGENCE_TIMEOUT;
-    loop {
-        let listed = client
-            .list_objects_v2()
-            .bucket(MIXED_BUCKET)
-            .prefix(format!("{phase}/"))
-            .send()
-            .await?;
-        let count = listed.contents().len();
-        if count == expected {
-            return Ok(());
-        }
-        if Instant::now() >= deadline {
-            return Err(format!(
-                "{context}: listing under {phase}/ returned {count} of {expected} objects even after {}s of post-restart convergence",
-                LISTING_CONVERGENCE_TIMEOUT.as_secs()
-            )
-            .into());
-        }
-        sleep(Duration::from_millis(500)).await;
-    }
-}
-
-async fn exercise_mixed_cluster(
-    cluster: &RustFSTestClusterEnvironment,
-    phase: &str,
-    current_node: usize,
-    previous_node: usize,
-) -> TestResult {
-    let clients = cluster.create_all_clients()?;
-    let current_client = &clients[current_node];
-    let previous_client = &clients[previous_node];
-
-    let current_key = format!("{phase}/written-by-current");
-    let current_body = format!("{phase}: current RustFS build").into_bytes();
-    current_client
-        .put_object()
-        .bucket(MIXED_BUCKET)
-        .key(&current_key)
-        .body(ByteStream::from(current_body.clone()))
-        .send()
-        .await?;
-    assert_eq!(read_object(previous_client, MIXED_BUCKET, &current_key, None).await?.1, current_body);
-
-    let previous_key = format!("{phase}/written-by-previous");
-    let previous_body = format!("{phase}: previous RustFS release").into_bytes();
-    previous_client
-        .put_object()
-        .bucket(MIXED_BUCKET)
-        .key(&previous_key)
-        .body(ByteStream::from(previous_body.clone()))
-        .send()
-        .await?;
-    assert_eq!(read_object(current_client, MIXED_BUCKET, &previous_key, None).await?.1, previous_body);
-
-    let multipart_keys = write_multipart_load(&clients, phase).await?;
-    let expected_count = multipart_keys.len() + 2;
-    for (label, client) in [("current", current_client), ("previous", previous_client)] {
-        wait_for_phase_listing(
-            client,
-            phase,
-            expected_count,
-            &format!("the {label} RustFS version must stream the complete mixed-version listing"),
-        )
-        .await?;
-    }
-
-    let last_multipart_key = format!("{phase}/multipart/{:02}/{:02}", MULTIPART_WORKERS - 1, MULTIPART_UPLOADS_PER_WORKER - 1);
-    assert_eq!(
-        read_object(previous_client, MIXED_BUCKET, &last_multipart_key, None).await?.1,
-        vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
-    );
-
-    Ok(())
-}
-
 #[tokio::test]
 #[ignore = "requires a pinned previous RustFS release binary"]
 async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {
@@ -389,43 +252,3 @@ async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {
 
     Ok(())
 }
-
-#[tokio::test]
-#[ignore = "requires a pinned previous RustFS release binary"]
-async fn rolling_upgrade_from_rc2_preserves_mixed_version_contracts() -> TestResult {
-    init_logging();
-    let previous_binary = source_binary()?;
-    let current_binary = rustfs_binary_path();
-    let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
-    cluster.set_env("RUST_LOG", "rustfs=warn,rustfs_notify=warn");
-    configure_cluster_logs(&mut cluster)?;
-    cluster.start_with_binary(&previous_binary).await?;
-    cluster.create_test_bucket(MIXED_BUCKET).await?;
-
-    cluster.stop_node(0)?;
-    cluster.start_node_from_binary(0, &current_binary).await?;
-    exercise_mixed_cluster(&cluster, "one-current-node", 0, 1).await?;
-
-    for node_idx in [1, 2] {
-        cluster.stop_node(node_idx)?;
-        cluster.start_node_from_binary(node_idx, &current_binary).await?;
-    }
-    exercise_mixed_cluster(&cluster, "one-previous-node", 0, 3).await?;
-
-    cluster.stop_node(3)?;
-    cluster.start_node_from_binary(3, &current_binary).await?;
-
-    for (node_idx, client) in cluster.create_all_clients()?.iter().enumerate() {
-        for phase in ["one-current-node", "one-previous-node"] {
-            wait_for_phase_listing(
-                client,
-                phase,
-                MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER + 2,
-                &format!("node {node_idx}: the homogeneous current cluster must preserve every object"),
-            )
-            .await?;
-        }
-    }
-
-    Ok(())
-}

crates/ecstore/Cargo.toml

@@ -166,7 +166,7 @@ uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnos
 reed-solomon-erasure = { workspace = true, features = ["simd-accel"] }
 reed-solomon-simd = { workspace = true }
 lazy_static.workspace = true
-moka = { workspace = true, features = ["future", "sync"] }
+moka = { workspace = true, features = ["future"] }
 rustfs-lock.workspace = true
 rustfs-io-metrics.workspace = true
 regex = { workspace = true }
@@ -185,7 +185,7 @@ hyper-rustls = { workspace = true, default-features = false, features = ["native
 hostname.workspace = true
 rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
 rustls-pki-types.workspace = true
-tokio = { workspace = true, features = ["io-util", "sync", "signal", "fs", "rt-multi-thread", "time"] }
+tokio = { workspace = true, features = ["io-util", "sync", "signal", "fs", "rt-multi-thread"] }
 tonic = { workspace = true, features = ["gzip", "deflate"] }
 xxhash-rust = { workspace = true, features = ["xxh64", "xxh3"] }
 tower = { workspace = true, features = ["timeout"] }

crates/ecstore/src/api/mod.rs

@@ -75,10 +75,6 @@ pub mod bucket {
                 delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
                 inspect_transition_transaction_for_operator,
             };
-            #[cfg(feature = "test-util")]
-            pub use crate::bucket::lifecycle::transition_transaction::{
-                TransitionTransactionRecoveryStats, recover_transition_transaction_records,
-            };
         }
 
         pub mod evaluator {
@@ -103,17 +99,6 @@ pub mod bucket {
             pub use crate::bucket::lifecycle::tier_delete_journal::{
                 persist_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
             };
-
-            #[cfg(feature = "test-util")]
-            pub mod test_util {
-                /// Model a single-node, all-v6 fleet after its capability probe has completed.
-                ///
-                /// Call this only once while constructing an isolated test store, before any
-                /// tier-delete journal permit or background worker can be active.
-                pub fn install_all_v6_fleet_capability_proof() {
-                    crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
-                }
-            }
         }
 
         pub mod tier_last_day_stats {
@@ -128,6 +113,7 @@ pub mod bucket {
     }
 
     pub mod metadata {
+        pub use crate::bucket::metadata::BUCKET_DURABILITY_CONFIG;
         pub use crate::bucket::metadata::{
             BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG,
             BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_QUOTA_CONFIG_FILE,
@@ -136,7 +122,6 @@ pub mod bucket {
             BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BUCKET_WEBSITE_CONFIG, BucketMetadata, OBJECT_LOCK_CONFIG,
             load_bucket_metadata, table_catalog_path_hash,
         };
-        pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
     }
 
     pub mod durability {
@@ -145,29 +130,6 @@ pub mod bucket {
         };
     }
 
-    pub mod on_demand_migration {
-        pub use crate::bucket::on_demand_migration::{
-            ApplyOutcome, BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION,
-            Breaker, BreakerState, BreakerTransition, BreakerVerdict, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, GaugeGuard,
-            LastSourceError, LatencyBucketSnapshot, NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache, OdmBucketSnapshot, OdmLookup,
-            OdmOp, OdmOutcome, OdmStateError, OdmStats, OdmStatsSnapshot, OnDemandMigrationSys, PullError, PullFailureReason,
-            PullFollower, PullLeader, PullOutcome, PullPath, PullResult, PullSlot, SOURCE_LATENCY_BUCKET_BOUNDS_MS,
-            SourceLatencySnapshot, source_client_spec,
-        };
-        pub use crate::bucket::on_demand_migration::{
-            ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION,
-            OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy,
-            SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
-        };
-        pub mod source_client {
-            pub use crate::bucket::on_demand_migration::source_client::{
-                SourceClient, SourceClientSpec, SourceError, SourceGet, SourceHead, SourceObject, SourcePage, SourceProbe,
-                SourceProvider, SourceSse, SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value,
-                resolve_path_style,
-            };
-        }
-    }
-
     pub mod metadata_sys {
         #[cfg(feature = "test-util")]
         pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
@@ -177,11 +139,11 @@ pub mod bucket {
             delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
             get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
             get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
-            get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, get_public_access_block_config,
-            get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config,
-            get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata,
-            remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock,
-            update_config_with, update_if_incarnation, update_quota_if_incarnation, update_under_transaction_lock,
+            get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config,
+            get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config,
+            get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
+            set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
+            update_quota_if_incarnation, update_under_transaction_lock,
         };
     }
 
@@ -222,13 +184,6 @@ pub mod bucket {
         }
     }
 
-    pub mod remote_s3_client {
-        pub use crate::bucket::remote_s3_client::{
-            PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, build_remote_s3_client,
-            validate_remote_endpoint,
-        };
-    }
-
     pub mod replication {
         pub use crate::bucket::replication::replication_pool::{
             DurableMrfBacklogSummary, DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBacklogObservabilitySummary,
@@ -451,7 +406,7 @@ pub mod notification {
     pub use crate::services::notification_sys::{
         CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
         acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
-        new_global_notification_sys, scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
+        new_global_notification_sys, start_remote_version_state_fleet_probe,
     };
 }
 
@@ -470,11 +425,6 @@ pub mod object {
         PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
         SnapshotConsistencyError,
     };
-
-    #[cfg(feature = "test-util")]
-    pub mod test_util {
-        pub use crate::store::DeleteAfterObjectLockSnapshotBarrier;
-    }
 }
 
 pub mod rebalance {
@@ -511,8 +461,8 @@ pub mod rpc {
         tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
         verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
         verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
-        verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
-        verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
+        verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+        verify_tonic_rpc_signature_with_bootstrap,
     };
 }
 
@@ -539,9 +489,9 @@ pub mod storage {
     pub use crate::core::pools::HealLifecycleExpiryContext;
     pub use crate::store::HealWalkVersion;
     pub use crate::store::{
-        ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path,
-        find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
-        prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
+        ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
+        init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
+        prewarm_local_disk_id_map_with_instance_ctx,
     };
 }
 

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -41,7 +41,7 @@ use crate::bucket::lifecycle::tier_free_version_recovery::{
     DEFAULT_FREE_VERSION_RECOVERY_LIMIT, FreeVersionRecoveryStats, recover_tier_free_versions_with_cancel,
 };
 use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
-use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_with_lease_idempotent};
+use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity};
 use crate::bucket::lifecycle::transition_transaction::run_transition_transaction_recovery_loop;
 use crate::bucket::object_lock::ObjectLockApi;
 use crate::bucket::versioning::VersioningApi as _;
@@ -50,10 +50,7 @@ use crate::disk::error::DiskError;
 use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE};
 use crate::error::Error;
 use crate::error::StorageError;
-use crate::error::{
-    is_err_object_not_found, is_err_read_quorum, is_err_strict_volume_not_found, is_err_version_not_found,
-    is_network_or_host_down,
-};
+use crate::error::{is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down};
 use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
 use crate::object_api::{ObjectEncryptionResolver, ReadPlan};
 use crate::services::tier::{
@@ -589,215 +586,23 @@ impl ExpiryOp for FreeVersionTask {
     }
 }
 
-async fn acquire_free_version_tier_lease(
-    oi: &ObjectInfo,
-    tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
-) -> Result<(TierOperationLease, bool), std::io::Error> {
-    let version_id_exact = validate_transition_remote_version(oi)?;
-    let identity = tier_destination_id_from_metadata(&oi.user_defined)?
-        .ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?;
-    let lease =
-        TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity)
-            .await
-            .map_err(std::io::Error::other)?;
-    Ok((lease, version_id_exact))
-}
-
-async fn delete_free_version_remote_object_with_lease(
-    oi: &ObjectInfo,
-    lease: &TierOperationLease,
-    version_id_exact: bool,
-) -> Result<(), std::io::Error> {
-    delete_object_from_remote_tier_with_lease_idempotent(
-        &oi.transitioned_object.name,
-        &oi.transitioned_object.version_id,
-        lease,
-        version_id_exact,
-    )
-    .await?;
-    Ok(())
-}
-
-fn free_version_physical_topology_generation(api: &ECStore) -> String {
-    let mut hasher = Sha256::new();
-    for pool in &api.pools {
-        hasher.update(pool.pool_idx.to_be_bytes());
-        hasher.update(pool.disk_set.len().to_be_bytes());
-        for set in &pool.disk_set {
-            hasher.update(set.set_index.to_be_bytes());
-        }
-    }
-    rustfs_utils::crypto::hex(hasher.finalize().as_slice())
-}
-
-fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectInfo) -> std::io::Result<bool> {
-    if candidate.transitioned_object.tier != expected.transitioned_object.tier
-        || candidate.transitioned_object.name != expected.transitioned_object.name
-    {
-        return Ok(false);
-    }
-    let candidate_identity = tier_destination_id_from_metadata(&candidate.user_defined)?
-        .ok_or_else(|| std::io::Error::other("tier free-version is missing its backend identity"))?;
-    let expected_identity = tier_destination_id_from_metadata(&expected.user_defined)?
-        .ok_or_else(|| std::io::Error::other("tier free-version task is missing its backend identity"))?;
-    if candidate_identity != expected_identity {
-        return Ok(false);
-    }
-    if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
-        || expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
-    {
-        return Err(std::io::Error::new(
-            std::io::ErrorKind::WouldBlock,
-            "tier free-version remote version state is unknown",
-        ));
-    }
-    Ok(candidate.transition_version_state == expected.transition_version_state
-        && candidate.transitioned_object.version_id == expected.transitioned_object.version_id)
-}
-
-async fn scan_exact_free_version_targets(
-    api: &ECStore,
-    oi: &ObjectInfo,
-    local_object: &str,
-) -> std::io::Result<Vec<(Arc<SetDisks>, FileInfo)>> {
-    let mut targets = Vec::new();
-    for pool in &api.pools {
-        for set in &pool.disk_set {
-            let versions = match set.load_file_info_versions_exact(&oi.bucket, &oi.name).await {
-                Ok(Some(versions)) => versions,
-                Ok(None) => continue,
-                Err(err) if is_err_strict_volume_not_found(&err) => continue,
-                Err(err) => return Err(std::io::Error::other(err)),
-            };
-            for version in versions.versions.iter().chain(versions.free_versions.iter()) {
-                let candidate = ObjectInfo::from_file_info(version, &oi.bucket, &oi.name, true);
-                if free_version_remote_tuple_matches(&candidate, oi)? {
-                    if candidate.transitioned_object.free_version {
-                        // Data movement can leave the same remote tuple in
-                        // several physical pools. Ordinary deletion assigns a
-                        // fresh local free-version UUID to each copy, but all
-                        // of those markers own the same idempotent remote
-                        // DELETE. Consume them together while holding every
-                        // physical object lock; treating their local UUIDs as
-                        // conflicting would strand cleanup forever.
-                        let mut actual = version.clone();
-                        actual.name = local_object.to_string();
-                        targets.push((Arc::clone(set), actual));
-                    } else {
-                        return Err(std::io::Error::new(
-                            std::io::ErrorKind::WouldBlock,
-                            "a live transitioned source still references the free-version remote tuple",
-                        ));
-                    }
-                }
-            }
-        }
-    }
-    Ok(targets)
-}
-
-fn free_version_cleanup_fences_current(
-    topology_generation: &str,
-    api: &ECStore,
-    bucket_guard: &rustfs_lock::NamespaceLockGuard,
-    object_guards: &[crate::store::ObjectLockDiagGuard],
-    lease: &TierOperationLease,
-    cancel: &CancellationToken,
-    deadline: tokio::time::Instant,
-) -> bool {
-    !cancel.is_cancelled()
-        && tokio::time::Instant::now() < deadline
-        && !bucket_guard.is_lock_lost()
-        && object_guards.iter().all(|guard| !guard.is_lock_lost())
-        && lease.is_current_generation()
-        && free_version_physical_topology_generation(api) == topology_generation
-}
-
-async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel: &CancellationToken) -> std::io::Result<bool> {
-    const FREE_VERSION_REMOTE_DEADLINE: StdDuration = StdDuration::from_secs(30);
-
-    let topology_generation = free_version_physical_topology_generation(&api);
-    let bucket_guard = api
-        .acquire_bucket_lifecycle_read_lock(&oi.bucket)
-        .await
-        .map_err(std::io::Error::other)?;
-    let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
-    let local_object = encode_dir_object(&oi.name);
-    let object_guards = api
-        .acquire_all_physical_object_write_locks("tier_free_version_cleanup", &oi.bucket, &local_object)
-        .await
-        .map_err(std::io::Error::other)?;
-    let targets = scan_exact_free_version_targets(&api, oi, &local_object).await?;
-    if targets.is_empty() {
-        return Ok(false);
-    }
-
-    let deadline = tokio::time::Instant::now() + FREE_VERSION_REMOTE_DEADLINE;
-    if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
-        return Err(std::io::Error::new(
-            std::io::ErrorKind::WouldBlock,
-            "tier free-version cleanup fence is invalid before remote delete",
-        ));
-    }
-    tokio::select! {
-        _ = cancel.cancelled() => {
-            return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled"));
-        }
-        result = tokio::time::timeout_at(
-            deadline,
-            delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact),
-        ) => {
-            result
-                .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out"))??;
-        }
-    }
-    if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
-        // Remote DELETE is idempotent, but a changed fence makes the local
-        // outcome ambiguous. Keep every marker for a fully fenced retry.
-        return Err(std::io::Error::new(
-            std::io::ErrorKind::WouldBlock,
-            "tier free-version cleanup fence changed after remote delete",
-        ));
-    }
-
-    let mut first_error = None;
-    for (set, actual) in &targets {
-        let mut delete_request = FileInfo {
-            name: local_object.clone(),
-            version_id: actual.version_id,
-            ..Default::default()
-        };
-        delete_request.set_tier_free_version();
-        if let Err(err) = set
-            .delete_object_version(&oi.bucket, &local_object, &delete_request, false)
-            .await
-            && first_error.is_none()
-        {
-            first_error = Some(std::io::Error::other(err));
-        }
-    }
-    let remaining = scan_exact_free_version_targets(&api, oi, &local_object).await?;
-    if !remaining.is_empty() {
-        return Err(first_error.unwrap_or_else(|| {
-            std::io::Error::new(
-                std::io::ErrorKind::WouldBlock,
-                "tier free-version cleanup remained on at least one physical set",
-            )
-        }));
-    }
-    if let Some(err) = first_error {
-        return Err(err);
-    }
-    Ok(true)
-}
-
-#[cfg(all(test, feature = "test-util"))]
 async fn delete_free_version_remote_object(
     oi: &ObjectInfo,
     tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
 ) -> Result<(), std::io::Error> {
-    let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
-    delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await
+    let version_id_exact = validate_transition_remote_version(oi)?;
+    let identity = tier_destination_id_from_metadata(&oi.user_defined)?
+        .ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?;
+    delete_object_from_remote_tier_idempotent_with_manager_and_identity(
+        &oi.transitioned_object.name,
+        &oi.transitioned_object.version_id,
+        &oi.transitioned_object.tier,
+        identity,
+        tier_config_mgr,
+        version_id_exact,
+    )
+    .await?;
+    Ok(())
 }
 
 #[allow(
@@ -813,11 +618,8 @@ where
     F: FnOnce() -> Fut,
     Fut: std::future::Future<Output = T>,
 {
-    let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
-    delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await?;
-    let result = delete_local().await;
-    drop(lease);
-    Ok(result)
+    delete_free_version_remote_object(oi, tier_config_mgr).await?;
+    Ok(delete_local().await)
 }
 
 struct NewerNoncurrentTask {
@@ -888,10 +690,6 @@ impl ExpiryState {
         usize::try_from(self.stats.pending_tasks().max(0)).unwrap_or(usize::MAX)
     }
 
-    pub fn active_tasks(&self) -> usize {
-        usize::try_from(self.stats.active_tasks().max(0)).unwrap_or(usize::MAX)
-    }
-
     fn send_expiry_task(&self, wrkr: Sender<Option<ExpiryOpType>>, task: ExpiryOpType) -> bool {
         let queued = wrkr.try_send(Some(task)).is_ok();
         if queued {
@@ -921,7 +719,7 @@ impl ExpiryState {
         Ok(())
     }
 
-    pub fn enqueue_free_version(&self, oi: ObjectInfo) -> bool {
+    pub fn enqueue_free_version(&mut self, oi: ObjectInfo) -> bool {
         let task = FreeVersionTask(oi);
         let wrkr = self.get_worker_ch(task.op_hash());
         if wrkr.is_none() {
@@ -1028,7 +826,7 @@ impl ExpiryState {
     }
 
     pub async fn resize_workers(n: usize, api: Arc<ECStore>) {
-        let expiry_state = api.ctx.expiry_state();
+        let expiry_state = runtime_sources::expiry_state_handle();
         if n == expiry_state.read().await.tasks_tx.len() || n < 1 {
             return;
         }
@@ -1069,7 +867,7 @@ impl ExpiryState {
         stats: Arc<ExpiryStats>,
         recovery_notify: Arc<Notify>,
     ) {
-        let cancel_token = api.ctx.background_cancel_token().unwrap_or_else(|| {
+        let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_else(|| {
             static FALLBACK: std::sync::OnceLock<tokio_util::sync::CancellationToken> = std::sync::OnceLock::new();
             FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new).clone()
         });
@@ -1170,33 +968,119 @@ impl ExpiryState {
                     else if v.as_any().is::<FreeVersionTask>() {
                         let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("FreeVersionTask downcast failed");
                         let oi = v.0.clone();
-                        match cleanup_free_version_exact(api.clone(), &oi, &cancel_token).await {
-                            Ok(true) => {}
-                            Ok(false) => debug!(
+                        if let Err(err) = delete_free_version_remote_object(&oi, &api.tier_config_mgr()).await {
+                            recovery_notify.notify_one();
+                            debug!(
+                                bucket = %oi.bucket,
+                                object = %oi.name,
+                                remote_object = %oi.transitioned_object.name,
+                                remote_version_id = %oi.transitioned_object.version_id,
+                                tier = %oi.transitioned_object.tier,
+                                error = ?err,
+                                event = EVENT_LIFECYCLE_WORKER_STATE,
+                                component = LOG_COMPONENT_ECSTORE,
+                                subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                                reason = "remote_tier_delete_failed",
+                                "Lifecycle worker skipped remote tier delete"
+                            );
+                            continue;
+                        }
+
+                        let local_object = encode_dir_object(&oi.name);
+                        let mut fi = FileInfo {
+                            name: local_object.clone(),
+                            version_id: oi.version_id,
+                            ..Default::default()
+                        };
+                        // This removes an existing internal cleanup marker. Keeping
+                        // `deleted` false makes duplicate tasks return not-found
+                        // instead of creating an ordinary delete marker.
+                        fi.set_tier_free_version();
+
+                        let mut deleted_locally = false;
+                        for pool in &api.pools {
+                            let set = pool.get_disks_by_key(&local_object);
+                            let ns_lock = match set.new_ns_lock(&oi.bucket, &local_object).await {
+                                Ok(lock) => lock,
+                                Err(err) => {
+                                    recovery_notify.notify_one();
+                                    debug!(
+                                        event = EVENT_LIFECYCLE_WORKER_STATE,
+                                        component = LOG_COMPONENT_ECSTORE,
+                                        subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                                        bucket = %oi.bucket,
+                                        object = %oi.name,
+                                        pool_index = pool.pool_idx,
+                                        set_index = set.set_index,
+                                        error = ?err,
+                                        reason = "local_free_version_lock_failed",
+                                        "Lifecycle worker failed to create local free-version cleanup lock"
+                                    );
+                                    continue;
+                                }
+                            };
+                            let _object_lock_guard =
+                                match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
+                                    Ok(guard) => guard,
+                                    Err(err) => {
+                                        recovery_notify.notify_one();
+                                        debug!(
+                                            event = EVENT_LIFECYCLE_WORKER_STATE,
+                                            component = LOG_COMPONENT_ECSTORE,
+                                            subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                                            bucket = %oi.bucket,
+                                            object = %oi.name,
+                                            pool_index = pool.pool_idx,
+                                            set_index = set.set_index,
+                                            error = ?err,
+                                            reason = "local_free_version_lock_failed",
+                                            "Lifecycle worker failed to acquire local free-version cleanup lock"
+                                        );
+                                        continue;
+                                    }
+                                };
+                            match set
+                                .delete_object_version(&oi.bucket, &local_object, &fi, false)
+                                .await
+                            {
+                                Ok(()) => {
+                                    deleted_locally = true;
+                                    break;
+                                }
+                                Err(err) if is_err_version_not_found(&err) || is_err_object_not_found(&err) => continue,
+                                Err(err) => {
+                                    recovery_notify.notify_one();
+                                    debug!(
+                                        event = EVENT_LIFECYCLE_WORKER_STATE,
+                                        component = LOG_COMPONENT_ECSTORE,
+                                        subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                                        bucket = %oi.bucket,
+                                        object = %oi.name,
+                                        remote_object = %oi.transitioned_object.name,
+                                        remote_version_id = %oi.transitioned_object.version_id,
+                                        tier = %oi.transitioned_object.tier,
+                                        error = ?err,
+                                        reason = "local_free_version_delete_failed",
+                                        "Lifecycle worker failed local free-version cleanup"
+                                    );
+                                    break;
+                                }
+                            }
+                        }
+
+                        if !deleted_locally {
+                            debug!(
                                 event = EVENT_LIFECYCLE_WORKER_STATE,
                                 component = LOG_COMPONENT_ECSTORE,
                                 subsystem = LOG_SUBSYSTEM_LIFECYCLE,
                                 bucket = %oi.bucket,
                                 object = %oi.name,
+                                remote_object = %oi.transitioned_object.name,
+                                remote_version_id = %oi.transitioned_object.version_id,
+                                tier = %oi.transitioned_object.tier,
                                 reason = "local_free_version_missing",
-                                "Lifecycle worker found that the exact free-version was already absent"
-                            ),
-                            Err(err) => {
-                                recovery_notify.notify_one();
-                                debug!(
-                                    event = EVENT_LIFECYCLE_WORKER_STATE,
-                                    component = LOG_COMPONENT_ECSTORE,
-                                    subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                                    bucket = %oi.bucket,
-                                    object = %oi.name,
-                                    remote_object = %oi.transitioned_object.name,
-                                    remote_version_id = %oi.transitioned_object.version_id,
-                                    tier = %oi.transitioned_object.tier,
-                                    error = ?err,
-                                    reason = "free_version_exact_cleanup_deferred",
-                                    "Lifecycle worker retained the exact free-version for a fenced retry"
-                                );
-                            }
+                                "Lifecycle worker could not find transitioned free version locally"
+                            );
                         }
                     }
                     else {
@@ -1215,22 +1099,6 @@ impl ExpiryState {
     }
 }
 
-pub(crate) async fn enqueue_committed_free_versions(api: &ECStore, free_versions: Vec<ObjectInfo>) -> usize {
-    if free_versions.is_empty() {
-        return 0;
-    }
-
-    let expiry_state = api.ctx.expiry_state();
-    let state = expiry_state.read().await;
-    let mut queued = 0;
-    for free_version in free_versions {
-        if state.enqueue_free_version(free_version) {
-            queued += 1;
-        }
-    }
-    queued
-}
-
 async fn enqueue_recovered_free_version_with_state(state: &Arc<RwLock<ExpiryState>>, oi: ObjectInfo) -> bool {
     let task = FreeVersionTask(oi);
     let hash = task.op_hash();
@@ -1284,8 +1152,8 @@ fn set_recovered_free_version_enqueue_observer(
     RecoveredFreeVersionEnqueueObserverGuard
 }
 
-pub async fn enqueue_recovered_free_version(api: &ECStore, oi: ObjectInfo) -> bool {
-    let expiry_state = api.ctx.expiry_state();
+pub async fn enqueue_recovered_free_version(oi: ObjectInfo) -> bool {
+    let expiry_state = runtime_sources::expiry_state_handle();
     let queued = enqueue_recovered_free_version_with_state(&expiry_state, oi).await;
 
     #[cfg(test)]
@@ -2712,8 +2580,8 @@ fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<(
     }
 
     Some(tokio::spawn(async move {
-        let cancel_token = api.ctx.background_cancel_token().unwrap_or_default();
-        let expiry_state = api.ctx.expiry_state();
+        let cancel_token = runtime_sources::background_services_cancel_token().unwrap_or_default();
+        let expiry_state = runtime_sources::expiry_state_handle();
         run_tier_free_version_recovery_loop(
             cancel_token,
             expiry_state,
@@ -6361,18 +6229,9 @@ mod tests {
             rustfs_utils::crypto::hex(old_identity),
         );
         oi.user_defined = Arc::new(metadata.clone());
-        let lease_observed_during_local_delete = Arc::new(std::sync::atomic::AtomicBool::new(false));
         delete_free_version_remote_object_then(&oi, &manager, {
             let local_delete_calls = Arc::clone(&local_delete_calls);
-            let lease_observed_during_local_delete = Arc::clone(&lease_observed_during_local_delete);
-            let manager = manager.clone();
             move || async move {
-                assert_eq!(
-                    crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await,
-                    1,
-                    "the identity-bound tier lease must span the exact local marker delete"
-                );
-                lease_observed_during_local_delete.store(true, Ordering::Relaxed);
                 local_delete_calls.fetch_add(1, Ordering::Relaxed);
             }
         })
@@ -6380,12 +6239,6 @@ mod tests {
         .expect("matching destination identity should allow idempotent remote cleanup");
         assert_eq!(old_backend.remove_count().await, 1);
         assert_eq!(local_delete_calls.load(Ordering::Relaxed), 1);
-        assert!(lease_observed_during_local_delete.load(Ordering::Relaxed));
-        assert_eq!(
-            crate::services::tier::tier::TierConfigMgr::active_operation_lease_count(&manager, "WARM").await,
-            0,
-            "the tier lease should be released after the local marker delete completes"
-        );
 
         let mut single_prefix_metadata = HashMap::new();
         single_prefix_metadata.insert(
@@ -6619,7 +6472,6 @@ mod tests {
         let state = ExpiryState::new();
         let mut state = state.write().await;
         let je = Jentry {
-            persisted_version: 0,
             obj_name: "remote/object".to_string(),
             version_id: "remote-version".to_string(),
             tier_name: "WARM".to_string(),
@@ -6628,7 +6480,6 @@ mod tests {
             version_state: rustfs_filemeta::TransitionVersionState::Exact,
             state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
             source: None,
-            dispatch: None,
         };
 
         let err = state
@@ -6643,7 +6494,7 @@ mod tests {
     async fn enqueue_free_version_reports_false_without_worker_channel() {
         let state = ExpiryState::new();
         let recovery_notify = Arc::clone(&state.read().await.recovery_notify);
-        let state = state.write().await;
+        let mut state = state.write().await;
         let oi = ObjectInfo {
             bucket: "bucket".to_string(),
             name: "object".to_string(),
@@ -6769,7 +6620,6 @@ mod tests {
         let state = ExpiryState::new_with_unconsumed_worker_channel(1);
         let mut state = state.write().await;
         let je = Jentry {
-            persisted_version: 0,
             obj_name: "remote/object".to_string(),
             version_id: "remote-version".to_string(),
             tier_name: "WARM".to_string(),
@@ -6778,7 +6628,6 @@ mod tests {
             version_state: rustfs_filemeta::TransitionVersionState::Exact,
             state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
             source: None,
-            dispatch: None,
         };
 
         state
@@ -6835,7 +6684,7 @@ mod tests {
             },
             ..Default::default()
         };
-        let state = state.write().await;
+        let mut state = state.write().await;
 
         assert!(state.enqueue_free_version(oi.clone()));
         assert!(recovery_notify.notified().now_or_never().is_none());
@@ -6910,7 +6759,7 @@ mod tests {
         };
 
         assert!(
-            super::enqueue_recovered_free_version(&ecstore, oi).await,
+            super::enqueue_recovered_free_version(oi).await,
             "the resized production worker queue should accept the task"
         );
         stop_tx.send(None).await.expect("worker stop signal should be delivered");
@@ -7026,12 +6875,12 @@ mod tests {
             .await
             .expect("free-version task should reach the worker");
         tokio::time::timeout(StdDuration::from_secs(30), async {
-            while stats.active_tasks() == 0 {
+            while remote_backend.remove_count().await == 0 {
                 tokio::task::yield_now().await;
             }
         })
         .await
-        .expect("worker should mark the cleanup task active before the lock assertion");
+        .expect("worker should complete remote cleanup before taking the local lock");
         let completed_while_locked = tokio::time::timeout(StdDuration::from_millis(100), async {
             while stats.active_tasks() != 0 {
                 tokio::task::yield_now().await;
@@ -7040,12 +6889,7 @@ mod tests {
         .await;
         assert!(
             completed_while_locked.is_err(),
-            "the cleanup task must wait while a competing object writer owns the namespace lock"
-        );
-        assert_eq!(
-            remote_backend.remove_count().await,
-            0,
-            "the remote tuple must not be deleted before the all-physical namespace fence is acquired"
+            "local cleanup must wait while a competing object writer owns the namespace lock"
         );
         for disk_path in &disk_paths {
             assert!(
@@ -7056,13 +6900,6 @@ mod tests {
         }
 
         drop(object_lock_guard);
-        tokio::time::timeout(StdDuration::from_secs(30), async {
-            while remote_backend.remove_count().await == 0 {
-                tokio::task::yield_now().await;
-            }
-        })
-        .await
-        .expect("worker should delete the remote tuple after acquiring the released namespace fence");
         tx.send(None).await.expect("worker stop signal should be delivered");
         worker.await.expect("free-version worker should stop cleanly");
 
@@ -7158,7 +6995,6 @@ mod tests {
             .next()
             .expect("seeded free version should be recoverable");
         let stale_version_id = oi.version_id.expect("free version should have a concrete UUID");
-        let ordinary_marker_mod_time = OffsetDateTime::now_utc();
 
         for disk_path in &disk_paths {
             let metadata_path = disk_path.join(&bucket).join(object).join(STORAGE_FORMAT_FILE);
@@ -7181,7 +7017,7 @@ mod tests {
                     name: object.to_string(),
                     version_id: Some(stale_version_id),
                     deleted: true,
-                    mod_time: Some(ordinary_marker_mod_time),
+                    mod_time: Some(OffsetDateTime::now_utc()),
                     ..Default::default()
                 })
                 .expect("same-ID ordinary marker should replace the stale free version");
@@ -7195,13 +7031,6 @@ mod tests {
             .expect("same-ID ordinary marker metadata should be written");
         }
 
-        assert!(
-            !super::cleanup_free_version_exact(Arc::clone(&ecstore), &oi, &CancellationToken::new())
-                .await
-                .expect("a stale task whose local UUID now names an ordinary marker should be an idempotent no-op"),
-            "the stale free-version task must not report local cleanup"
-        );
-
         let state = ExpiryState::new();
         let (stats, recovery_notify) = {
             let state = state.read().await;
@@ -11693,7 +11522,7 @@ mod tests {
 
     #[cfg(feature = "test-util")]
     #[tokio::test]
-    async fn journal_replay_quarantines_legacy_unknown_version_state_before_backend_io() {
+    async fn journal_replay_rejects_unknown_version_state_before_backend_io() {
         let (_disk_paths, ecstore) = setup_test_env().await;
         let (backend, _) = register_recovery_mock_tier(&ecstore).await;
         let identity = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
@@ -11701,7 +11530,6 @@ mod tests {
             .expect("mock tier lease should be available")
             .backend_identity();
         let je = Jentry {
-            persisted_version: 0,
             obj_name: "remote/object".to_string(),
             version_id: "legacy-version".to_string(),
             tier_name: "WARM".to_string(),
@@ -11710,28 +11538,25 @@ mod tests {
             version_state: rustfs_filemeta::TransitionVersionState::Unknown,
             state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
             source: None,
-            dispatch: None,
         };
 
-        crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(ecstore.clone(), &je)
-            .await
-            .expect("legacy unknown journal should remain byte-compatible and persistable");
         let err = crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
             .await
-            .expect_err("legacy unknown journal must be quarantined before backend IO");
+            .expect_err("unknown journal state must fail before backend IO");
 
-        assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
         assert_eq!(backend.remove_count().await, 0);
     }
 
     #[cfg(feature = "test-util")]
     #[tokio::test]
-    async fn rejected_upload_cleanup_retries_confirmed_exact_provider_token_without_legacy_journal() {
+    async fn journal_replay_deletes_confirmed_exact_provider_token() {
         let (_disk_paths, ecstore) = setup_test_env().await;
         let (backend, _) = register_recovery_mock_tier(&ecstore).await;
         let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
             .await
             .expect("mock tier lease should be available");
+        let identity = lease.backend_identity();
         backend
             .set_put_remote_version(Some("provider-version-token".to_string()))
             .await;
@@ -11745,30 +11570,34 @@ mod tests {
             .expect("confirmed remote candidate should be seeded");
         backend.set_remove_failure(true);
         backend.set_reject_non_empty_remote_versions(true);
-        let err = crate::set_disk::cleanup_rejected_transition_upload_durably(
+        let je = Jentry {
+            obj_name: "remote/object".to_string(),
+            version_id: "provider-version-token".to_string(),
+            tier_name: "WARM".to_string(),
+            backend_identity: Some(identity),
+            version_id_exact: true,
+            version_state: rustfs_filemeta::TransitionVersionState::Exact,
+            state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
+            source: None,
+        };
+
+        crate::set_disk::cleanup_rejected_transition_upload_durably(
             &lease,
-            "remote/object",
-            "provider-version-token",
+            &je.obj_name,
+            &je.version_id,
             true,
             Some(ecstore.clone()),
         )
         .await
-        .expect_err("a failed immediate cleanup must remain owned by the caller's transition transaction");
-        assert_eq!(err.kind(), std::io::ErrorKind::Other);
-        assert!(backend.contains("remote/object").await);
+        .expect("failed immediate cleanup should remain durable in the journal");
+        assert!(backend.contains(&je.obj_name).await);
 
         backend.set_remove_failure(false);
-        crate::set_disk::cleanup_rejected_transition_upload_durably(
-            &lease,
-            "remote/object",
-            "provider-version-token",
-            true,
-            Some(ecstore),
-        )
-        .await
-        .expect("the transaction retry must delete the same confirmed candidate");
+        crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
+            .await
+            .expect("identity-bound exact journal must retry confirmed candidate cleanup");
 
-        assert!(!backend.contains("remote/object").await);
+        assert!(!backend.contains(&je.obj_name).await);
         assert_eq!(backend.exact_remove_count(), 2);
         assert_eq!(
             backend.remove_versions().await,
@@ -11931,14 +11760,11 @@ mod tests {
         };
         let mut recovery_rx = recovery_rx.lock().await;
         assert!(
-            super::enqueue_recovered_free_version(
-                &ecstore,
-                ObjectInfo {
-                    bucket: "prefill".to_string(),
-                    name: "prefill".to_string(),
-                    ..Default::default()
-                },
-            )
+            super::enqueue_recovered_free_version(ObjectInfo {
+                bucket: "prefill".to_string(),
+                name: "prefill".to_string(),
+                ..Default::default()
+            })
             .await,
             "the production recovery queue should accept its first task"
         );
@@ -12369,7 +12195,7 @@ mod tests {
     #[tokio::test]
     #[serial]
     async fn tier_free_version_recovery_continues_after_deleted_marker_bucket() {
-        let (disk_paths, ecstore) = setup_test_env().await;
+        let (_paths, ecstore) = setup_test_env().await;
         let suffix = Uuid::new_v4().simple();
         let earlier_bucket = format!("zzzz-recovery-{suffix}-a");
         let deleted_marker = format!("zzzz-recovery-{suffix}-m");
@@ -12377,7 +12203,11 @@ mod tests {
         let later_object = "a-before-stale-marker";
         create_test_bucket(&ecstore, &earlier_bucket).await;
         create_test_bucket(&ecstore, &later_bucket).await;
-        seed_recoverable_free_version(&disk_paths, &later_bucket, later_object, None, None).await;
+        let mut reader = PutObjReader::from_vec(b"cursor reset probe".to_vec());
+        ecstore
+            .put_object(&later_bucket, later_object, &mut reader, &ObjectOptions::default())
+            .await
+            .expect("successor bucket object should be created");
 
         let page = list_tier_free_versions(
             Arc::clone(&ecstore),
@@ -12390,10 +12220,14 @@ mod tests {
         .expect("recovery should resume at the first bucket after a deleted marker bucket");
 
         assert_eq!(page.buckets_scanned, 1, "the later bucket must not be skipped");
-        assert_eq!(page.items.len(), 1, "the successor bucket's recoverable object must be returned");
-        assert_eq!(page.items[0].bucket, later_bucket);
-        assert_eq!(page.items[0].name, later_object);
-        remove_seeded_free_version(&disk_paths, &later_bucket, later_object).await;
+        assert_eq!(
+            page.scanned_entries, 1,
+            "the deleted bucket's object marker must not skip objects in the successor bucket"
+        );
+        ecstore
+            .delete_object(&later_bucket, later_object, ObjectOptions::default())
+            .await
+            .expect("successor bucket object should be removed");
         for bucket in [&earlier_bucket, &later_bucket] {
             ecstore
                 .delete_bucket(bucket, &DeleteBucketOptions::default())

crates/ecstore/src/bucket/lifecycle/durable_namespace.rs

@@ -33,7 +33,6 @@ const MANUAL_TRANSITION_CURSOR_MARKER_PROOF_MAX_SIZE: usize = 1024;
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub(crate) enum DurableIlmRecordKind {
     TierDeleteJournal,
-    TierDeleteDispatchManifest,
     TransitionTransaction,
     ManualTransitionJob,
     ManualTransitionScope,
@@ -55,18 +54,6 @@ pub(crate) const TIER_DELETE_JOURNAL_NAMESPACE: DurableIlmNamespace = DurableIlm
     max_record_size: 64 * 1024,
     kind: DurableIlmRecordKind::TierDeleteJournal,
 };
-pub(crate) const TIER_DELETE_JOURNAL_V6_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
-    name: "tier-delete-journal-v6",
-    prefix: "ilm/tier-delete-journal-v6/",
-    max_record_size: 64 * 1024,
-    kind: DurableIlmRecordKind::TierDeleteJournal,
-};
-pub(crate) const TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
-    name: "tier-delete-dispatch-manifest",
-    prefix: tier_delete_journal::TIER_DELETE_DISPATCH_MANIFEST_PREFIX,
-    max_record_size: tier_delete_journal::MAX_TIER_DELETE_DISPATCH_MANIFEST_SIZE,
-    kind: DurableIlmRecordKind::TierDeleteDispatchManifest,
-};
 pub(crate) const TRANSITION_TRANSACTION_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
     name: "transition-transaction",
     prefix: "ilm/transition-transactions/records",
@@ -98,10 +85,8 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
     kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
 };
 
-pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 8] = [
+pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 6] = [
     TIER_DELETE_JOURNAL_NAMESPACE,
-    TIER_DELETE_JOURNAL_V6_NAMESPACE,
-    TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
     TRANSITION_TRANSACTION_NAMESPACE,
     MANUAL_TRANSITION_JOB_NAMESPACE,
     MANUAL_TRANSITION_SCOPE_NAMESPACE,
@@ -172,15 +157,6 @@ pub(crate) enum DurableIlmRecordCheckpoint {
         content_sha256: String,
         identity_sha256: String,
         committed: bool,
-        #[serde(default, skip_serializing_if = "Option::is_none")]
-        dispatch_identity_sha256: Option<String>,
-        #[serde(default, skip_serializing_if = "Option::is_none")]
-        state: Option<super::tier_sweeper::TierDeleteJournalState>,
-    },
-    TierDeleteDispatchManifest {
-        content_sha256: String,
-        identity_sha256: String,
-        state: tier_delete_journal::TierDeleteDispatchManifestState,
     },
     TransitionTransaction {
         content_sha256: String,
@@ -219,7 +195,6 @@ impl DurableIlmRecordCheckpoint {
     pub(crate) fn content_sha256(&self) -> &str {
         match self {
             Self::TierDeleteJournal { content_sha256, .. }
-            | Self::TierDeleteDispatchManifest { content_sha256, .. }
             | Self::TransitionTransaction { content_sha256, .. }
             | Self::ManualTransitionJob { content_sha256, .. }
             | Self::ManualTransitionScope { content_sha256, .. }
@@ -253,19 +228,6 @@ impl DurableIlmRecordCheckpoint {
     }
 
     pub(crate) fn validate_successor(&self, next: &Self) -> Result<()> {
-        for checkpoint in [self, next] {
-            if let Self::TierDeleteJournal {
-                committed,
-                dispatch_identity_sha256,
-                state,
-                ..
-            } = checkpoint
-                && (state.is_some() != dispatch_identity_sha256.is_some()
-                    || state.is_some_and(|state| *committed != (state == super::tier_sweeper::TierDeleteJournalState::Committed)))
-            {
-                return Err(Error::other("durable ILM tier delete journal checkpoint is invalid"));
-            }
-        }
         if self == next {
             if let Self::ManualTransitionJob {
                 progress,
@@ -282,64 +244,18 @@ impl DurableIlmRecordCheckpoint {
         let valid = match (self, next) {
             (
                 Self::TierDeleteJournal {
-                    content_sha256: previous_content,
                     identity_sha256: previous_identity,
                     committed: previous_committed,
-                    dispatch_identity_sha256: previous_dispatch_identity,
-                    state: previous_state,
                     ..
                 },
                 Self::TierDeleteJournal {
-                    content_sha256: next_content,
                     identity_sha256: next_identity,
                     committed: next_committed,
-                    dispatch_identity_sha256: next_dispatch_identity,
-                    state: next_state,
                     ..
                 },
             ) => {
-                use super::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};
-
-                let dispatch_identity_is_monotonic = match (previous_dispatch_identity, next_dispatch_identity) {
-                    (Some(previous), Some(next)) => previous == next,
-                    (None, None) => true,
-                    // Old receipts did not record the v6 dispatch binding. A
-                    // byte-identical observation may adopt the stronger proof,
-                    // but an in-flight mutation must fail closed instead of
-                    // guessing which operation owned the journal.
-                    (None, Some(_)) => previous_content == next_content,
-                    (Some(_), None) => false,
-                };
-                let state_is_monotonic = match (previous_state, next_state) {
-                    (Some(previous), Some(next)) => {
-                        previous == next || matches!((previous, next), (Prepared, Dispatched) | (Dispatched, Committed))
-                    }
-                    (None, None) => previous_committed == next_committed || (!previous_committed && *next_committed),
-                    (None, Some(_)) => previous_content == next_content,
-                    (Some(_), None) => false,
-                };
-                previous_identity == next_identity && dispatch_identity_is_monotonic && state_is_monotonic
-            }
-            (
-                Self::TierDeleteDispatchManifest {
-                    identity_sha256: previous_identity,
-                    state: previous_state,
-                    ..
-                },
-                Self::TierDeleteDispatchManifest {
-                    identity_sha256: next_identity,
-                    state: next_state,
-                    ..
-                },
-            ) => {
-                use tier_delete_journal::TierDeleteDispatchManifestState::{
-                    Aborted, Aborting, Completed, DispatchAuthorized, Preparing,
-                };
                 previous_identity == next_identity
-                    && matches!(
-                        (previous_state, next_state),
-                        (Preparing, DispatchAuthorized | Aborting) | (Aborting, Aborted) | (DispatchAuthorized, Completed)
-                    )
+                    && (previous_committed == next_committed || (!previous_committed && *next_committed))
             }
             (
                 Self::TransitionTransaction {
@@ -435,49 +351,6 @@ impl DurableIlmRecordCheckpoint {
             Err(Error::other("durable ILM record generation is not a monotonic successor"))
         }
     }
-
-    /// Whether `self` is an older generation of the same immutable record
-    /// that can reach `terminal` through one or more valid state transitions.
-    /// This is deliberately broader than `validate_successor`, which remains
-    /// adjacent-only for receipt advancement. Terminal cleanup uses this only
-    /// after the exact terminal ETag and terminal receipt were committed, to
-    /// purge older object versions exposed by that deletion.
-    pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool {
-        if self == terminal || self.validate_successor(terminal).is_ok() {
-            return true;
-        }
-        match (self, terminal) {
-            (
-                Self::TierDeleteJournal {
-                    identity_sha256: previous_identity,
-                    dispatch_identity_sha256: previous_dispatch,
-                    state: Some(super::tier_sweeper::TierDeleteJournalState::Prepared),
-                    ..
-                },
-                Self::TierDeleteJournal {
-                    identity_sha256: terminal_identity,
-                    dispatch_identity_sha256: terminal_dispatch,
-                    state: Some(super::tier_sweeper::TierDeleteJournalState::Committed),
-                    ..
-                },
-            ) => previous_identity == terminal_identity && previous_dispatch == terminal_dispatch,
-            (
-                Self::TierDeleteDispatchManifest {
-                    identity_sha256: previous_identity,
-                    state: tier_delete_journal::TierDeleteDispatchManifestState::Preparing,
-                    ..
-                },
-                Self::TierDeleteDispatchManifest {
-                    identity_sha256: terminal_identity,
-                    state:
-                        tier_delete_journal::TierDeleteDispatchManifestState::Aborted
-                        | tier_delete_journal::TierDeleteDispatchManifestState::Completed,
-                    ..
-                },
-            ) => previous_identity == terminal_identity,
-            _ => false,
-        }
-    }
 }
 
 fn transition_state_distance(
@@ -877,19 +750,10 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
             if tier_delete_journal::tier_delete_journal_object_name(&entry) != path {
                 return Err(Error::other("tier delete journal content does not match its path"));
             }
-            let legacy_operation_id = path
+            let operation_id = path
                 .strip_prefix(namespace.prefix)
                 .and_then(|suffix| suffix.strip_suffix(".json"))
                 .ok_or_else(|| Error::other("tier delete journal path is invalid"))?;
-            // Legacy v1-v5 paths already expose a 64-hex operation id and
-            // must remain receipt-compatible. V6 uses an operation-scoped
-            // nested path, so derive a fixed, path-unique receipt id instead
-            // of embedding slashes in the receipt locator.
-            let operation_id = if entry.persisted_version == 6 {
-                hex_sha256(path.as_bytes(), ToOwned::to_owned)
-            } else {
-                legacy_operation_id.to_string()
-            };
             let identity_sha256 = checkpoint_hash(&(
                 &entry.obj_name,
                 &entry.version_id,
@@ -899,29 +763,13 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
                 entry.version_state,
                 &entry.source,
             ))?;
-            let dispatch_identity_sha256 = entry.dispatch.as_ref().map(checkpoint_hash).transpose()?;
             (
                 "operation_id",
-                operation_id,
+                operation_id.to_string(),
                 DurableIlmRecordCheckpoint::TierDeleteJournal {
                     content_sha256,
                     identity_sha256,
                     committed: entry.state == super::tier_sweeper::TierDeleteJournalState::Committed,
-                    dispatch_identity_sha256,
-                    state: (entry.persisted_version == 6).then_some(entry.state),
-                },
-            )
-        }
-        DurableIlmRecordKind::TierDeleteDispatchManifest => {
-            let (operation_id, identity_sha256, state) =
-                tier_delete_journal::validate_tier_delete_dispatch_manifest_record(path, data)?;
-            (
-                "operation_id",
-                hex_sha256(operation_id.as_bytes(), ToOwned::to_owned),
-                DurableIlmRecordCheckpoint::TierDeleteDispatchManifest {
-                    content_sha256,
-                    identity_sha256,
-                    state,
                 },
             )
         }
@@ -1108,87 +956,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn tier_delete_dispatch_manifest_namespace_validates_monotonic_branches() {
-        use tier_delete_journal::TierDeleteDispatchManifestState::{Aborted, Aborting, Completed, DispatchAuthorized, Preparing};
-
-        let operation_id = Uuid::new_v4();
-        let checkpoint = |state| {
-            let (path, data) = tier_delete_journal::test_tier_delete_dispatch_manifest_record(operation_id, state);
-            let namespace = classify_durable_ilm_record(&path)
-                .expect("dispatch manifest namespace should classify")
-                .expect("dispatch manifest should be durable");
-            assert_eq!(namespace, &TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE);
-            validate_durable_ilm_record(&path, &data)
-                .expect("dispatch manifest should validate")
-                .checkpoint
-        };
-
-        let preparing = checkpoint(Preparing);
-        let authorized = checkpoint(DispatchAuthorized);
-        let completed = checkpoint(Completed);
-        let aborting = checkpoint(Aborting);
-        let aborted = checkpoint(Aborted);
-
-        preparing
-            .validate_successor(&authorized)
-            .expect("Preparing may become DispatchAuthorized");
-        authorized
-            .validate_successor(&completed)
-            .expect("DispatchAuthorized may become Completed");
-        preparing.validate_successor(&aborting).expect("Preparing may enter rollback");
-        aborting.validate_successor(&aborted).expect("Aborting may become Aborted");
-        assert!(authorized.validate_successor(&aborting).is_err());
-        assert!(completed.validate_successor(&authorized).is_err());
-        assert!(aborted.validate_successor(&preparing).is_err());
-    }
-
-    #[test]
-    fn tier_delete_journal_checkpoint_binds_dispatch_and_full_state_monotonically() {
-        use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::{Committed, Dispatched, Prepared};
-
-        let checkpoint = |content: &str, dispatch: Option<&str>, state| DurableIlmRecordCheckpoint::TierDeleteJournal {
-            content_sha256: content.repeat(64),
-            identity_sha256: "i".repeat(64),
-            committed: state == Some(Committed),
-            dispatch_identity_sha256: dispatch.map(|value| value.repeat(64)),
-            state,
-        };
-        let prepared = checkpoint("a", Some("d"), Some(Prepared));
-        let dispatched = checkpoint("b", Some("d"), Some(Dispatched));
-        let committed = checkpoint("c", Some("d"), Some(Committed));
-        prepared
-            .validate_successor(&dispatched)
-            .expect("Prepared may advance to Dispatched");
-        dispatched
-            .validate_successor(&committed)
-            .expect("Dispatched may advance to Committed");
-        assert!(prepared.validate_successor(&committed).is_err());
-        assert!(dispatched.validate_successor(&prepared).is_err());
-
-        let rebound = checkpoint("b", Some("e"), Some(Dispatched));
-        assert!(dispatched.validate_successor(&rebound).is_err());
-
-        let legacy: DurableIlmRecordCheckpoint = serde_json::from_value(serde_json::json!({
-            "kind": "tier_delete_journal",
-            "content_sha256": "a".repeat(64),
-            "identity_sha256": "i".repeat(64),
-            "committed": false
-        }))
-        .expect("legacy tier-delete checkpoint should remain decodable");
-        legacy
-            .validate_successor(&prepared)
-            .expect("byte-identical legacy receipt may adopt the stronger v6 proof");
-        let changed_legacy = DurableIlmRecordCheckpoint::TierDeleteJournal {
-            content_sha256: "z".repeat(64),
-            identity_sha256: "i".repeat(64),
-            committed: false,
-            dispatch_identity_sha256: None,
-            state: None,
-        };
-        assert!(changed_legacy.validate_successor(&prepared).is_err());
-    }
-
     #[test]
     fn manual_transition_job_checkpoint_compacts_legacy_progress_compatibly() {
         let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default();

crates/ecstore/src/bucket/lifecycle/mod.rs

@@ -34,6 +34,6 @@ pub mod tier_sweeper;
 pub mod transition_transaction;
 
 pub(crate) use durable_namespace::{
-    DurableIlmRecordCheckpoint, ILM_META_PREFIX, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE, ValidatedDurableIlmRecord,
-    classify_durable_ilm_record, validate_durable_ilm_record,
+    DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
+    validate_durable_ilm_record,
 };

crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs

@@ -172,8 +172,7 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
         return Err(std::io::Error::other("free-version recovery limit must be greater than zero").into());
     }
 
-    let page =
-        list_tier_free_versions(api.clone(), limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?;
+    let page = list_tier_free_versions(api, limit, bucket_marker.clone(), object_marker.clone(), cancel_token.clone()).await?;
     let mut stats = FreeVersionRecoveryStats {
         scanned: 0,
         enqueued: 0,
@@ -191,7 +190,7 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
             return Err(tier_free_version_recovery_cancelled());
         }
         retry_cursor.visit(&oi);
-        if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(&api, oi).await) {
+        if !record_recovered_free_version_enqueue(&mut stats, enqueue_recovered_free_version(oi).await) {
             let (bucket_marker, object_marker) = retry_cursor.retry_markers();
             stats.truncated = true;
             stats.next_bucket_marker = bucket_marker;

crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -255,7 +255,6 @@ impl ObjSweeper {
         }
         if del_tier {
             return Some(Jentry {
-                persisted_version: 0,
                 obj_name: self.remote_object.clone(),
                 version_id: self.transition_version_id.clone(),
                 tier_name: self.transition_tier.clone(),
@@ -267,7 +266,6 @@ impl ObjSweeper {
                 version_state: self.transition_version_state,
                 state: TierDeleteJournalState::Committed,
                 source: None,
-                dispatch: None,
             });
         }
         None
@@ -300,19 +298,9 @@ impl ObjSweeper {
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
 pub(crate) enum TierDeleteJournalState {
     Prepared,
-    Dispatched,
     Committed,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-#[serde(deny_unknown_fields)]
-pub(crate) struct TierDeleteDispatchBinding {
-    pub(crate) operation_id: Uuid,
-    pub(crate) manifest_object: String,
-    pub(crate) journal_set_sha256: String,
-    pub(crate) topology_generation: String,
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(deny_unknown_fields)]
 pub(crate) struct TierDeleteSourceIdentity {
@@ -354,10 +342,6 @@ impl TierDeleteSourceIdentity {
 #[derive(Debug, Clone)]
 #[allow(unused_assignments)]
 pub struct Jentry {
-    /// On-disk format version when decoded. Newly constructed entries use 0;
-    /// the encoder chooses their format from the durable ownership fields.
-    /// Recovery uses this value to quarantine v1-v5 without rewriting them.
-    pub(crate) persisted_version: u8,
     pub(crate) obj_name: String,
     pub(crate) version_id: String,
     pub(crate) tier_name: String,
@@ -366,23 +350,6 @@ pub struct Jentry {
     pub(crate) version_state: rustfs_filemeta::TransitionVersionState,
     pub(crate) state: TierDeleteJournalState,
     pub(crate) source: Option<TierDeleteSourceIdentity>,
-    pub(crate) dispatch: Option<TierDeleteDispatchBinding>,
-}
-
-impl Jentry {
-    /// Whether this prepared transaction is eligible to become the sole
-    /// cleanup owner for its transitioned source. The caller may use this to
-    /// decide whether to persist it, but must not set `skip_free_version`
-    /// until persistence succeeds.
-    pub(crate) fn can_replace_tier_free_version(&self) -> bool {
-        self.state == TierDeleteJournalState::Prepared
-            && self.backend_identity.is_some()
-            && self.version_state != rustfs_filemeta::TransitionVersionState::Unknown
-            && self
-                .source
-                .as_ref()
-                .is_some_and(TierDeleteSourceIdentity::has_stable_identity)
-    }
 }
 
 impl ExpiryOp for Jentry {
@@ -650,7 +617,6 @@ pub fn transitioned_force_delete_journal_entry(
     }
 
     Some(Jentry {
-        persisted_version: 0,
         obj_name: transitioned.name.clone(),
         version_id: transitioned.version_id.clone(),
         tier_name: transitioned.tier.clone(),
@@ -662,7 +628,6 @@ pub fn transitioned_force_delete_journal_entry(
         version_state: transition_version_state,
         state: TierDeleteJournalState::Committed,
         source: None,
-        dispatch: None,
     })
 }
 
@@ -708,73 +673,17 @@ mod test {
     use rustfs_s3_client::signer_error::invalid_utf8_header_error;
 
     use super::{
-        CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry,
-        RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity,
-        delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent,
-        delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error,
-        is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure,
-        transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
+        CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED,
+        RemoteDeleteBreaker, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
+        delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
+        is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
+        should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
     };
     use crate::storage_api_contracts::lifecycle::TransitionedObject;
     use rustfs_filemeta::TransitionVersionState;
     use std::io::{Error, ErrorKind};
     use std::time::{Duration, Instant};
 
-    fn stable_prepared_journal() -> Jentry {
-        Jentry {
-            persisted_version: 0,
-            obj_name: "remote/object".to_string(),
-            version_id: "remote-version".to_string(),
-            tier_name: "WARM".to_string(),
-            backend_identity: Some([7; 32]),
-            version_id_exact: true,
-            version_state: TransitionVersionState::Exact,
-            state: TierDeleteJournalState::Prepared,
-            source: Some(TierDeleteSourceIdentity {
-                bucket: "bucket".to_string(),
-                object: "object".to_string(),
-                version_id: Some(uuid::Uuid::new_v4().to_string()),
-                versioned: true,
-                version_suspended: false,
-                data_dir: None,
-                etag: None,
-                mod_time: None,
-            }),
-            dispatch: None,
-        }
-    }
-
-    #[test]
-    fn only_stable_prepared_journal_can_replace_tier_free_version() {
-        let stable = stable_prepared_journal();
-        assert!(stable.can_replace_tier_free_version());
-
-        let mut committed = stable.clone();
-        committed.state = TierDeleteJournalState::Committed;
-        assert!(!committed.can_replace_tier_free_version());
-
-        let mut unbound = stable.clone();
-        unbound.backend_identity = None;
-        assert!(!unbound.can_replace_tier_free_version());
-
-        let mut unknown = stable.clone();
-        unknown.version_state = TransitionVersionState::Unknown;
-        assert!(!unknown.can_replace_tier_free_version());
-
-        let mut unstable = stable;
-        unstable.source = Some(TierDeleteSourceIdentity {
-            bucket: "bucket".to_string(),
-            object: "object".to_string(),
-            version_id: None,
-            versioned: false,
-            version_suspended: false,
-            data_dir: None,
-            etag: Some("etag-only".to_string()),
-            mod_time: None,
-        });
-        assert!(!unstable.can_replace_tier_free_version());
-    }
-
     #[test]
     fn signer_header_error_detection_matches_utf8_failures() {
         let err = Error::new(

crates/ecstore/src/bucket/metadata.rs

@@ -270,7 +270,6 @@ pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
 pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
 pub const BUCKET_TABLE_CONFIG: &str = "table-bucket.json";
 pub const BUCKET_DURABILITY_CONFIG: &str = "durability.json";
-pub const BUCKET_ON_DEMAND_MIGRATION_CONFIG: &str = "on-demand-migration.json";
 pub const BUCKET_TABLE_RESERVED_PREFIX: &str = ".rustfs-table";
 pub const BUCKET_TABLE_CATALOG_META_PREFIX: &str = "s3tables/catalog";
 pub const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str = "table-buckets";
@@ -322,7 +321,6 @@ pub struct BucketMetadata {
     pub bucket_acl_config_json: Vec<u8>,
     pub table_bucket_config_json: Vec<u8>,
     pub durability_config_json: Vec<u8>,
-    pub on_demand_migration_config_json: Vec<u8>,
 
     pub policy_config_updated_at: OffsetDateTime,
     pub object_lock_config_updated_at: OffsetDateTime,
@@ -344,7 +342,6 @@ pub struct BucketMetadata {
     pub bucket_acl_config_updated_at: OffsetDateTime,
     pub table_bucket_config_updated_at: OffsetDateTime,
     pub durability_config_updated_at: OffsetDateTime,
-    pub on_demand_migration_config_updated_at: OffsetDateTime,
 
     pub new_field_updated_at: OffsetDateTime,
 
@@ -396,7 +393,6 @@ impl Default for BucketMetadata {
             bucket_acl_config_json: Default::default(),
             table_bucket_config_json: Default::default(),
             durability_config_json: Default::default(),
-            on_demand_migration_config_json: Default::default(),
             policy_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH,
@@ -417,7 +413,6 @@ impl Default for BucketMetadata {
             bucket_acl_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             table_bucket_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             durability_config_updated_at: OffsetDateTime::UNIX_EPOCH,
-            on_demand_migration_config_updated_at: OffsetDateTime::UNIX_EPOCH,
             new_field_updated_at: OffsetDateTime::UNIX_EPOCH,
             policy_config: Default::default(),
             notification_config: Default::default(),
@@ -482,23 +477,6 @@ impl BucketMetadata {
     /// Absent/empty/unparsable payloads all mean "no override" (the bucket
     /// follows the global durability mode); a parse failure is logged so a
     /// corrupted entry cannot silently change fsync behavior.
-    /// Parsed on-demand migration config, if one is stored.
-    ///
-    /// `Ok(None)` means no config (absent or cleared). A stored payload that
-    /// does not parse is an error, never a default: the runtime must not
-    /// pull from a source it cannot describe.
-    pub fn on_demand_migration_config(
-        &self,
-    ) -> std::result::Result<
-        Option<super::on_demand_migration::OnDemandMigrationConfig>,
-        super::on_demand_migration::OnDemandMigrationConfigError,
-    > {
-        if self.on_demand_migration_config_json.is_empty() {
-            return Ok(None);
-        }
-        super::on_demand_migration::OnDemandMigrationConfig::from_json(&self.on_demand_migration_config_json).map(Some)
-    }
-
     pub fn durability_config(&self) -> Option<super::durability::BucketDurabilityConfig> {
         if self.durability_config_json.is_empty() {
             return None;
@@ -577,9 +555,6 @@ impl BucketMetadata {
                 "BucketAclConfigJSON" | "BucketAclConfigJson" => self.bucket_acl_config_json = read_msgp_bin(rd)?,
                 "TableBucketConfigJSON" | "TableBucketConfigJson" => self.table_bucket_config_json = read_msgp_bin(rd)?,
                 "DurabilityConfigJSON" | "DurabilityConfigJson" => self.durability_config_json = read_msgp_bin(rd)?,
-                "OnDemandMigrationConfigJSON" | "OnDemandMigrationConfigJson" => {
-                    self.on_demand_migration_config_json = read_msgp_bin(rd)?
-                }
                 "CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?,
                 "LoggingConfigUpdatedAt" => self.logging_config_updated_at = read_msgp_time_value(rd)?,
                 "WebsiteConfigUpdatedAt" => self.website_config_updated_at = read_msgp_time_value(rd)?,
@@ -589,7 +564,6 @@ impl BucketMetadata {
                 "BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?,
                 "TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
                 "DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
-                "OnDemandMigrationConfigUpdatedAt" => self.on_demand_migration_config_updated_at = read_msgp_time_value(rd)?,
                 other => {
                     tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field");
                     skip_msgp_value(rd)?;
@@ -602,8 +576,8 @@ impl BucketMetadata {
 
     /// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
     pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
-        // Map size: MinIO fields (25) + RustFS extensions (21)
-        let map_len: u32 = 46;
+        // Map size: MinIO fields (25) + RustFS extensions (19)
+        let map_len: u32 = 44;
         rmp::encode::write_map_len(wr, map_len)?;
 
         // MinIO field order (same as Go struct)
@@ -663,7 +637,6 @@ impl BucketMetadata {
         write_bin_field(wr, "BucketAclConfigJSON", &self.bucket_acl_config_json)?;
         write_bin_field(wr, "TableBucketConfigJSON", &self.table_bucket_config_json)?;
         write_bin_field(wr, "DurabilityConfigJSON", &self.durability_config_json)?;
-        write_bin_field(wr, "OnDemandMigrationConfigJSON", &self.on_demand_migration_config_json)?;
         rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?;
         write_msgp_time(wr, self.cors_config_updated_at)?;
         rmp::encode::write_str(wr, "LoggingConfigUpdatedAt")?;
@@ -682,8 +655,6 @@ impl BucketMetadata {
         write_msgp_time(wr, self.table_bucket_config_updated_at)?;
         rmp::encode::write_str(wr, "DurabilityConfigUpdatedAt")?;
         write_msgp_time(wr, self.durability_config_updated_at)?;
-        rmp::encode::write_str(wr, "OnDemandMigrationConfigUpdatedAt")?;
-        write_msgp_time(wr, self.on_demand_migration_config_updated_at)?;
 
         Ok(())
     }
@@ -785,9 +756,6 @@ impl BucketMetadata {
         if self.durability_config_updated_at == OffsetDateTime::UNIX_EPOCH {
             self.durability_config_updated_at = self.created
         }
-        if self.on_demand_migration_config_updated_at == OffsetDateTime::UNIX_EPOCH {
-            self.on_demand_migration_config_updated_at = self.created
-        }
     }
 
     pub fn update_config(&mut self, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
@@ -903,17 +871,6 @@ impl BucketMetadata {
                 self.durability_config_json = data;
                 self.durability_config_updated_at = updated;
             }
-            BUCKET_ON_DEMAND_MIGRATION_CONFIG => {
-                // Structural check only (shape, unknown fields); the
-                // deployment-relative rules run in the admin handler with a
-                // `ValidationContext`. A blob this build cannot read must not
-                // be persisted for every later reader to trip over.
-                if !data.is_empty() {
-                    super::on_demand_migration::OnDemandMigrationConfig::from_json(&data).map_err(Error::other)?;
-                }
-                self.on_demand_migration_config_json = data;
-                self.on_demand_migration_config_updated_at = updated;
-            }
             _ => return Err(Error::other(format!("config file not found : {config_file}"))),
         }
 
@@ -1822,117 +1779,6 @@ mod test {
         assert!(!bm.table_bucket_enabled());
     }
 
-    const ODM_JSON: &[u8] = br#"{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
-
-    /// rustfs/backlog#2148: the on-demand migration config is a RustFS
-    /// extension entry that round-trips through `update_config` and the
-    /// msgpack codec, clears on delete, and never parses corruption into a
-    /// default.
-    #[test]
-    fn on_demand_migration_config_round_trips_and_tracks_updates() {
-        use crate::bucket::on_demand_migration::{OnDemandMigrationConfig, OnDemandMigrationConfigError};
-
-        let mut bm = BucketMetadata::new("odm-bucket");
-        assert_eq!(bm.on_demand_migration_config(), Ok(None), "fresh metadata carries no config");
-
-        let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
-        bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .expect("valid config is accepted");
-        assert_ne!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
-        assert_eq!(bm.on_demand_migration_config(), Ok(Some(expected.clone())));
-
-        let back = BucketMetadata::unmarshal(&bm.marshal_msg().unwrap()).unwrap();
-        assert_eq!(back.on_demand_migration_config_json, bm.on_demand_migration_config_json);
-        assert_eq!(
-            back.on_demand_migration_config_updated_at.unix_timestamp(),
-            bm.on_demand_migration_config_updated_at.unix_timestamp()
-        );
-        assert_eq!(back.on_demand_migration_config(), Ok(Some(expected)));
-
-        // A blob this build cannot read is rejected at the write boundary
-        // rather than persisted for every reader to trip over.
-        let before = bm.on_demand_migration_config_json.clone();
-        assert!(
-            bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec())
-                .is_err()
-        );
-        assert_eq!(bm.on_demand_migration_config_json, before, "a rejected update leaves the blob untouched");
-
-        // Delete clears the entry.
-        let stamped = bm.on_demand_migration_config_updated_at;
-        bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, Vec::new()).unwrap();
-        assert!(bm.on_demand_migration_config_json.is_empty());
-        assert_eq!(bm.on_demand_migration_config(), Ok(None));
-        assert!(bm.on_demand_migration_config_updated_at >= stamped);
-
-        // Corruption that bypassed `update_config` (disk, another writer)
-        // is a typed error, never a default.
-        bm.on_demand_migration_config_json = b"not-json".to_vec();
-        assert!(matches!(bm.on_demand_migration_config(), Err(OnDemandMigrationConfigError::Malformed(_))));
-    }
-
-    /// rustfs/backlog#2148: a `.metadata.bin` written before the on-demand
-    /// migration keys existed decodes with an empty blob and an epoch
-    /// timestamp that `default_timestamps` back-fills from `created`.
-    #[test]
-    fn on_demand_migration_config_absent_in_legacy_blob_defaults_to_created() {
-        let blob = decode_hex(include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex"));
-        let mut bm = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal MinIO bucket metadata");
-        assert!(bm.on_demand_migration_config_json.is_empty());
-        assert_eq!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
-        assert_eq!(bm.on_demand_migration_config(), Ok(None));
-
-        bm.default_timestamps();
-        assert_ne!(bm.created, OffsetDateTime::UNIX_EPOCH, "fixture must carry a real creation time");
-        assert_eq!(bm.on_demand_migration_config_updated_at, bm.created);
-
-        // A metadata blob from this build with no config set stays
-        // indistinguishable from the legacy one for these fields.
-        let fresh = BucketMetadata::unmarshal(&BucketMetadata::new("fresh").marshal_msg().unwrap()).unwrap();
-        assert!(fresh.on_demand_migration_config_json.is_empty());
-        assert_eq!(fresh.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
-    }
-
-    /// rustfs/backlog#2148: a reader that predates the two on-demand
-    /// migration keys takes `decode_from`'s unknown-field branch, which is
-    /// `skip_msgp_value`. Walk the new-format blob with exactly that
-    /// primitive and prove both keys are skipped without desynchronising the
-    /// stream, so the fields that follow them still decode.
-    #[test]
-    fn old_decoder_skips_on_demand_migration_fields_without_desync() {
-        let mut bm = BucketMetadata::new("odm-skip");
-        bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .unwrap();
-        bm.update_config(BUCKET_DURABILITY_CONFIG, br#"{"mode":"relaxed"}"#.to_vec())
-            .unwrap();
-        let buf = bm.marshal_msg().unwrap();
-
-        let mut rd = std::io::Cursor::new(buf.as_slice());
-        let fields = rmp::decode::read_map_len(&mut rd).unwrap();
-        let mut skipped = Vec::new();
-        let mut durability_json = Vec::new();
-        for _ in 0..fields {
-            let key_len = rmp::decode::read_str_len(&mut rd).unwrap();
-            let mut key = vec![0u8; key_len as usize];
-            rd.read_exact(&mut key).unwrap();
-            let key = String::from_utf8(key).unwrap();
-            match key.as_str() {
-                // The field an old reader knows that is encoded *after* the
-                // unknown JSON key and *before* the unknown timestamp key.
-                "DurabilityConfigJSON" => durability_json = read_msgp_bin(&mut rd).unwrap(),
-                other => {
-                    if other.starts_with("OnDemandMigration") {
-                        skipped.push(other.to_string());
-                    }
-                    skip_msgp_value(&mut rd).unwrap();
-                }
-            }
-        }
-        assert_eq!(skipped, ["OnDemandMigrationConfigJSON", "OnDemandMigrationConfigUpdatedAt"]);
-        assert_eq!(durability_json, br#"{"mode":"relaxed"}"#);
-        assert_eq!(rd.position() as usize, buf.len(), "old-style walk must consume the blob exactly");
-    }
-
     /// HP-5b (rustfs/backlog#938): the durability override is a RustFS
     /// extension entry and must survive an encode/decode round trip.
     #[test]

crates/ecstore/src/bucket/metadata_sys.rs

@@ -19,10 +19,9 @@ use super::quota::BucketQuota;
 use super::target::BucketTargets;
 use crate::bucket::bucket_target_sys::BucketTargetSys;
 use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_parse_with_presence};
-use crate::bucket::on_demand_migration::{ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig};
 use crate::bucket::utils::is_meta_bucketname;
 use crate::disk::RUSTFS_META_BUCKET;
-use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
+use crate::error::{Error, Result, is_err_bucket_not_found};
 use crate::runtime::sources as runtime_sources;
 use crate::storage_api_contracts::heal::HealOperations as _;
 use crate::storage_api_contracts::namespace::NamespaceLocking as _;
@@ -385,42 +384,6 @@ fn clear_bucket_durability(bucket: &str) {
     crate::disk::local::bucket_durability::set(bucket, None);
 }
 
-/// Publish the bucket's on-demand migration config (or its absence) to the
-/// runtime registered in `ON_DEMAND_MIGRATION_CONFIG_HOOK`.
-///
-/// Called from the same five cache-install paths as
-/// [`sync_bucket_durability`]. A stored payload this build cannot parse is
-/// published as `None`: the runtime must stop pulling for that bucket rather
-/// than keep an older config or guess.
-fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
-    let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() else {
-        return;
-    };
-    match bm.on_demand_migration_config() {
-        Ok(config) => hook(bucket, config.as_ref()),
-        Err(err) => {
-            warn!(
-                event = "bucket_metadata_parse_failed",
-                component = "ecstore",
-                subsystem = "bucket_metadata",
-                bucket = %bucket,
-                config = "on_demand_migration",
-                error = %err,
-                "Failed to parse bucket metadata config"
-            );
-            hook(bucket, None);
-        }
-    }
-}
-
-/// Withdraw a bucket's on-demand migration config when its metadata leaves
-/// the cache.
-fn clear_on_demand_migration(bucket: &str) {
-    if let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() {
-        hook(bucket, None);
-    }
-}
-
 pub async fn get(bucket: &str) -> Result<Arc<BucketMetadata>> {
     let sys = get_bucket_metadata_sys()?;
     let lock = sys.read().await;
@@ -449,14 +412,8 @@ pub(crate) fn require_bucket_metadata_sys_in(
 }
 
 pub(crate) async fn object_store_in(ctx: &crate::runtime::instance::InstanceContext) -> Result<Arc<ECStore>> {
-    object_store_if_initialized_in(ctx)
-        .await
-        .ok_or_else(|| Error::other("bucket metadata sys not initialized for this instance"))
-}
-
-pub(crate) async fn object_store_if_initialized_in(ctx: &crate::runtime::instance::InstanceContext) -> Option<Arc<ECStore>> {
-    let sys = ctx.bucket_metadata_sys().or_else(get_global_bucket_metadata_sys)?;
-    Some(sys.read().await.api.clone())
+    let sys = bucket_metadata_sys_of(ctx)?;
+    Ok(sys.read().await.api.clone())
 }
 
 pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<Arc<BucketMetadata>> {
@@ -696,12 +653,13 @@ async fn acquire_config_write_guard_for_incarnation(
             async {
                 match metadata_sys
                     .api
-                    .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
+                    .peer_sys
+                    .get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
                     .await
                 {
                     Ok(_) => Ok(()),
-                    Err(err) if is_err_strict_volume_not_found(&err) => Err(Error::BucketNotFound(bucket.to_string())),
-                    Err(err) => Err(err),
+                    Err(crate::disk::error::Error::VolumeNotFound) => Err(Error::BucketNotFound(bucket.to_string())),
+                    Err(err) => Err(err.into()),
                 }
             },
         ),
@@ -1007,16 +965,6 @@ pub async fn get_durability_config(
     Ok((bm.durability_config(), bm.durability_config_updated_at))
 }
 
-/// The bucket's on-demand migration config with its update time, or
-/// `Ok(None)` when the bucket has none. A stored payload that does not parse
-/// is a typed error (`OnDemandMigrationConfigError` inside `Error::Io`).
-pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
-    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
-    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
-
-    bucket_meta_sys.get_on_demand_migration_config(bucket).await
-}
-
 pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -1182,16 +1130,6 @@ pub(crate) async fn has_authoritative_never_versioned_state(bucket: &str) -> Res
     bucket_meta_sys.has_authoritative_never_versioned_state(bucket).await
 }
 
-pub(crate) async fn has_authoritative_never_versioned_state_in(
-    ctx: &crate::runtime::instance::InstanceContext,
-    bucket: &str,
-) -> Result<bool> {
-    let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
-    let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
-
-    bucket_meta_sys.has_authoritative_never_versioned_state(bucket).await
-}
-
 pub async fn get_website_config(bucket: &str) -> Result<(WebsiteConfiguration, OffsetDateTime)> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
     let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -1320,6 +1258,7 @@ pub struct BucketMetadataSys {
     /// name floods while avoiding repeated namespace and erasure reads.
     missing_buckets: moka::future::Cache<String, ()>,
     api: Arc<ECStore>,
+    initialized: Arc<RwLock<bool>>,
 }
 
 impl BucketMetadataSys {
@@ -1348,6 +1287,7 @@ impl BucketMetadataSys {
                 .time_to_live(MISSING_BUCKET_TTL)
                 .build(),
             api,
+            initialized: Arc::new(RwLock::new(false)),
         }
     }
 
@@ -1408,12 +1348,13 @@ impl BucketMetadataSys {
         await_bucket_namespace_operation(Some(namespace_guard), bucket, operation, async {
             match self
                 .api
-                .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
+                .peer_sys
+                .get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
                 .await
             {
                 Ok(_) => Ok(true),
-                Err(Error::VolumeNotFound) => Ok(false),
-                Err(err) => Err(err),
+                Err(crate::disk::error::Error::VolumeNotFound) => Ok(false),
+                Err(err) => Err(err.into()),
             }
         })
         .await
@@ -1423,15 +1364,9 @@ impl BucketMetadataSys {
         let _ = self.init_internal(buckets).await;
     }
     async fn init_internal(&self, buckets: Vec<String>) -> Result<()> {
-        let count = self
-            .api
-            .pools
-            .iter()
-            .map(|pool| pool.disk_set.len())
-            .sum::<usize>()
-            .checked_mul(10)
-            .filter(|count| *count != 0)
-            .ok_or_else(|| Error::other("bucket metadata store has no erasure sets"))?;
+        let count = runtime_sources::endpoint_erasure_set_count()
+            .map(|count| count * 10)
+            .ok_or_else(|| Error::other("endpoint pools not initialized"))?;
 
         let mut failed_buckets: HashSet<String> = HashSet::new();
         let mut buckets = buckets.as_slice();
@@ -1449,6 +1384,9 @@ impl BucketMetadataSys {
             buckets = &buckets[count..]
         }
 
+        let mut initialized = self.initialized.write().await;
+        *initialized = true;
+
         Ok(())
     }
 
@@ -1525,6 +1463,14 @@ impl BucketMetadataSys {
         expected: Option<&Arc<BucketMetadata>>,
         namespace_guard: &rustfs_lock::NamespaceLockGuard,
     ) -> Result<()> {
+        await_bucket_namespace_operation(
+            Some(namespace_guard),
+            bucket,
+            "bucket metadata heal",
+            self.api.heal_bucket(bucket, &HealOpts::default()),
+        )
+        .await?;
+
         if !self
             .bucket_exists(bucket, namespace_guard, "bucket metadata existence check")
             .await?
@@ -1539,26 +1485,11 @@ impl BucketMetadataSys {
                 if removed {
                     BucketTargetSys::get().delete(bucket).await;
                     clear_bucket_durability(bucket);
-                    clear_on_demand_migration(bucket);
                 }
             }
             return Ok(());
         }
 
-        await_bucket_namespace_operation(
-            Some(namespace_guard),
-            bucket,
-            "bucket metadata heal",
-            self.api.heal_bucket(
-                bucket,
-                &HealOpts {
-                    recreate: true,
-                    ..Default::default()
-                },
-            ),
-        )
-        .await?;
-
         let (bm, persisted) = await_bucket_namespace_operation(
             Some(namespace_guard),
             bucket,
@@ -1577,7 +1508,6 @@ impl BucketMetadataSys {
                 self.missing_buckets.invalidate(bucket).await;
                 sync_bucket_target_sys(bucket, &bm).await;
                 sync_bucket_durability(bucket, &bm);
-                sync_on_demand_migration(bucket, &bm);
             }
             MetadataLoadMode::Initial => {
                 let _publish_guard = self
@@ -1624,7 +1554,6 @@ impl BucketMetadataSys {
             if removed {
                 BucketTargetSys::get().delete(bucket).await;
                 clear_bucket_durability(bucket);
-                clear_on_demand_migration(bucket);
             }
             return Ok(());
         }
@@ -1647,7 +1576,6 @@ impl BucketMetadataSys {
         self.missing_buckets.invalidate(bucket).await;
         sync_bucket_target_sys(bucket, &metadata).await;
         sync_bucket_durability(bucket, &metadata);
-        sync_on_demand_migration(bucket, &metadata);
         Ok(())
     }
 
@@ -1675,7 +1603,6 @@ impl BucketMetadataSys {
             self.missing_buckets.invalidate(&bucket).await;
             sync_bucket_target_sys(&bucket, &bm).await;
             sync_bucket_durability(&bucket, &bm);
-            sync_on_demand_migration(&bucket, &bm);
         }
     }
 
@@ -1696,7 +1623,6 @@ impl BucketMetadataSys {
         if removed {
             BucketTargetSys::get().delete(bucket).await;
             clear_bucket_durability(bucket);
-            clear_on_demand_migration(bucket);
         }
         removed || removed_fabricated
     }
@@ -1945,13 +1871,23 @@ impl BucketMetadataSys {
                     "lazy metadata IO must start while the bucket namespace read lock is held"
                 );
             }
-            let (bm, persisted) = await_bucket_namespace_operation(
+            let (bm, persisted) = match await_bucket_namespace_operation(
                 Some(&guard),
                 bucket,
                 "lazy bucket metadata load",
                 Box::pin(load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true)),
             )
-            .await?;
+            .await
+            {
+                Ok(res) => res,
+                Err(err) => {
+                    return if *self.initialized.read().await {
+                        Err(Error::other("errBucketMetadataNotInitialized"))
+                    } else {
+                        Err(err)
+                    };
+                }
+            };
 
             let bm = Arc::new(bm);
 
@@ -1962,9 +1898,11 @@ impl BucketMetadataSys {
                     "lazy bucket metadata existence check",
                     Box::pin(async {
                         self.api
-                            .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
+                            .peer_sys
+                            .get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
                             .await
                             .map(|_| ())
+                            .map_err(Into::into)
                     }),
                 )
                 .await?;
@@ -1986,7 +1924,6 @@ impl BucketMetadataSys {
                 self.missing_buckets.invalidate(bucket).await;
                 sync_bucket_target_sys(bucket, &bm).await;
                 sync_bucket_durability(bucket, &bm);
-                sync_on_demand_migration(bucket, &bm);
             } else {
                 let exists = self
                     .bucket_exists(bucket, &guard, "lazy bucket metadata existence check")
@@ -2244,8 +2181,10 @@ impl BucketMetadataSys {
             "legacy bucket metadata existence check",
             async {
                 self.api
-                    .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
+                    .peer_sys
+                    .get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
                     .await
+                    .map_err(crate::error::StorageError::from)
             },
         )
         .await
@@ -2325,7 +2264,6 @@ impl BucketMetadataSys {
         self.missing_buckets.invalidate(bucket).await;
         sync_bucket_target_sys(bucket, &metadata).await;
         sync_bucket_durability(bucket, &metadata);
-        sync_on_demand_migration(bucket, &metadata);
         Ok(BucketMetadataAuthority::Authoritative(metadata))
     }
 
@@ -2348,8 +2286,10 @@ impl BucketMetadataSys {
             "bucket metadata snapshot existence check",
             async {
                 self.api
-                    .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
+                    .peer_sys
+                    .get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
                     .await
+                    .map_err(crate::error::StorageError::from)
             },
         )
         .await
@@ -2518,17 +2458,6 @@ impl BucketMetadataSys {
             Err(Error::ConfigNotFound)
         }
     }
-
-    /// See [`get_on_demand_migration_config`].
-    pub async fn get_on_demand_migration_config(
-        &self,
-        bucket: &str,
-    ) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
-        let (bm, _) = self.get_config(bucket).await?;
-
-        let config = bm.on_demand_migration_config().map_err(Error::other)?;
-        Ok(config.map(|config| (config, bm.on_demand_migration_config_updated_at)))
-    }
 }
 
 /// Test-only fixture shared with sibling modules (e.g. the quota checker
@@ -2583,169 +2512,11 @@ pub(crate) mod test_support {
 mod tests {
     use super::test_support::isolated_store_over_temp_disks;
     use super::*;
-    use crate::bucket::metadata::{
-        BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_NOTIFICATION_CONFIG,
-        BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG,
-        BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG, BUCKET_WEBSITE_CONFIG, OBJECT_LOCK_CONFIG,
-    };
     use crate::bucket::target::{BucketTarget, BucketTargetType, Credentials};
-    use crate::config::com::read_config;
     use crate::storage_api_contracts::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
-    use byteorder::{ByteOrder as _, LittleEndian};
     use serial_test::serial;
     use tokio::time::timeout;
 
-    const NEW_WRITER_REPLICATION_XML: &[u8] = br#"<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Role>arn:aws:iam::111122223333:role/replication-role</Role><Rule><ID>rollback</ID><Priority>1</Priority><Filter><Prefix>documents/</Prefix></Filter><Status>Enabled</Status><Destination><Bucket>arn:aws:s3:::replica-bucket</Bucket></Destination><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication></Rule></ReplicationConfiguration>"#;
-
-    const NEW_WRITER_CONFIGS: [(&str, &[u8]); 14] = [
-        (BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#),
-        (BUCKET_NOTIFICATION_CONFIG, br#"<NotificationConfiguration/>"#),
-        (
-            BUCKET_LIFECYCLE_CONFIG,
-            br#"<LifecycleConfiguration><Rule><ID>expire</ID><Status>Enabled</Status><Filter><Prefix>logs/</Prefix></Filter><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>"#,
-        ),
-        (
-            OBJECT_LOCK_CONFIG,
-            br#"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>7</Days></DefaultRetention></Rule></ObjectLockConfiguration>"#,
-        ),
-        (
-            BUCKET_VERSIONING_CONFIG,
-            br#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#,
-        ),
-        (
-            BUCKET_SSECONFIG,
-            br#"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"#,
-        ),
-        (
-            BUCKET_TAGGING_CONFIG,
-            r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
-        ),
-        (BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
-        (
-            BUCKET_CORS_CONFIG,
-            br#"<CORSConfiguration><CORSRule><AllowedMethod>GET</AllowedMethod><AllowedOrigin>https://example.test</AllowedOrigin></CORSRule></CORSConfiguration>"#,
-        ),
-        (BUCKET_LOGGING_CONFIG, br#"<BucketLoggingStatus/>"#),
-        (
-            BUCKET_WEBSITE_CONFIG,
-            br#"<WebsiteConfiguration><IndexDocument><Suffix>index.html</Suffix></IndexDocument></WebsiteConfiguration>"#,
-        ),
-        (
-            BUCKET_ACCELERATE_CONFIG,
-            br#"<AccelerateConfiguration><Status>Enabled</Status></AccelerateConfiguration>"#,
-        ),
-        (
-            BUCKET_REQUEST_PAYMENT_CONFIG,
-            br#"<RequestPaymentConfiguration><Payer>Requester</Payer></RequestPaymentConfiguration>"#,
-        ),
-        (
-            BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG,
-            br#"<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><IgnorePublicAcls>true</IgnorePublicAcls><BlockPublicPolicy>true</BlockPublicPolicy><RestrictPublicBuckets>false</RestrictPublicBuckets></PublicAccessBlockConfiguration>"#,
-        ),
-    ];
-
-    #[tokio::test]
-    async fn g_d3_003_new_writer_replication_loads_without_fail_closed_state() {
-        let (dirs, store) = isolated_store_over_temp_disks().await;
-        let bucket = "rollback-new-replication";
-        for dir in &dirs {
-            std::fs::create_dir_all(dir.path().join(bucket)).expect("rollback fixture bucket should be created");
-        }
-
-        let writer = BucketMetadataSys::new(store.clone());
-        let mut metadata = BucketMetadata::new(bucket);
-        metadata
-            .update_config(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML.to_vec())
-            .expect("new-writer replication XML should be accepted before persistence");
-        writer
-            .persist_new_and_set(metadata)
-            .await
-            .expect("new-writer replication metadata should persist");
-
-        let old_reader = BucketMetadataSys::new(store);
-        let (loaded, _) = old_reader
-            .get_replication_config(bucket)
-            .await
-            .expect("old metadata_sys must not classify new-writer replication XML as invalid");
-        assert_eq!(loaded.role, "arn:aws:iam::111122223333:role/replication-role");
-        assert_eq!(loaded.rules.len(), 1);
-        assert_eq!(loaded.rules[0].id.as_deref(), Some("rollback"));
-    }
-
-    #[tokio::test]
-    async fn g_d3_004_new_writer_metadata_blob_keeps_legacy_header_and_configs() {
-        let (dirs, store) = isolated_store_over_temp_disks().await;
-        let bucket = "rollback-new-metadata";
-        for dir in &dirs {
-            std::fs::create_dir_all(dir.path().join(bucket)).expect("rollback fixture bucket should be created");
-        }
-
-        let writer = BucketMetadataSys::new(store.clone());
-        let mut metadata = BucketMetadata::new(bucket);
-        for (config_file, bytes) in NEW_WRITER_CONFIGS {
-            metadata
-                .update_config(config_file, bytes.to_vec())
-                .unwrap_or_else(|err| panic!("new-writer {config_file} fixture must be valid: {err}"));
-        }
-        writer
-            .persist_new_and_set(metadata)
-            .await
-            .expect("new-writer metadata should persist");
-
-        let path = BucketMetadata::new(bucket).save_file_path();
-        let blob = read_config(store.clone(), &path)
-            .await
-            .expect("persisted .metadata.bin should be readable");
-        assert_eq!(
-            LittleEndian::read_u16(&blob[0..2]),
-            1,
-            "bucket metadata format must stay rollback-readable"
-        );
-        assert_eq!(
-            LittleEndian::read_u16(&blob[2..4]),
-            1,
-            "bucket metadata version must stay rollback-readable"
-        );
-
-        let loaded = load_bucket_metadata(store, bucket)
-            .await
-            .expect("old read_bucket_metadata path must load the new-writer blob");
-        let loaded_configs: [(&str, &[u8]); 14] = [
-            (BUCKET_POLICY_CONFIG, &loaded.policy_config_json),
-            (BUCKET_NOTIFICATION_CONFIG, &loaded.notification_config_xml),
-            (BUCKET_LIFECYCLE_CONFIG, &loaded.lifecycle_config_xml),
-            (OBJECT_LOCK_CONFIG, &loaded.object_lock_config_xml),
-            (BUCKET_VERSIONING_CONFIG, &loaded.versioning_config_xml),
-            (BUCKET_SSECONFIG, &loaded.encryption_config_xml),
-            (BUCKET_TAGGING_CONFIG, &loaded.tagging_config_xml),
-            (BUCKET_REPLICATION_CONFIG, &loaded.replication_config_xml),
-            (BUCKET_CORS_CONFIG, &loaded.cors_config_xml),
-            (BUCKET_LOGGING_CONFIG, &loaded.logging_config_xml),
-            (BUCKET_WEBSITE_CONFIG, &loaded.website_config_xml),
-            (BUCKET_ACCELERATE_CONFIG, &loaded.accelerate_config_xml),
-            (BUCKET_REQUEST_PAYMENT_CONFIG, &loaded.request_payment_config_xml),
-            (BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, &loaded.public_access_block_config_xml),
-        ];
-        for ((expected_name, expected), (loaded_name, actual)) in NEW_WRITER_CONFIGS.into_iter().zip(loaded_configs) {
-            assert_eq!(loaded_name, expected_name);
-            assert_eq!(actual, expected, "old read_bucket_metadata changed {expected_name} bytes");
-        }
-        assert!(loaded.policy_config.is_some());
-        assert!(loaded.notification_config.is_some());
-        assert!(loaded.lifecycle_config.is_some());
-        assert!(loaded.object_lock_config.is_some());
-        assert!(loaded.versioning_config.is_some());
-        assert!(loaded.sse_config.is_some());
-        assert!(loaded.tagging_config.is_some());
-        assert!(loaded.replication_config.is_some());
-        assert!(loaded.cors_config.is_some());
-        assert!(loaded.logging_config.is_some());
-        assert!(loaded.website_config.is_some());
-        assert!(loaded.accelerate_config.is_some());
-        assert!(loaded.request_payment_config.is_some());
-        assert!(loaded.public_access_block_config.is_some());
-    }
-
     #[tokio::test]
     async fn malformed_delete_configs_are_not_treated_as_absent() {
         let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
@@ -4109,151 +3880,6 @@ mod tests {
         assert_eq!(bucket_durability::lookup(bucket), None);
     }
 
-    const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
-
-    /// Every `(bucket, config)` the recording hook has seen. Tests filter by
-    /// their own bucket name; the hook is process-wide and set once.
-    static ODM_HOOK_CALLS: std::sync::Mutex<Vec<(String, Option<OnDemandMigrationConfig>)>> = std::sync::Mutex::new(Vec::new());
-
-    fn install_recording_odm_hook() {
-        ON_DEMAND_MIGRATION_CONFIG_HOOK.get_or_init(|| {
-            Box::new(|bucket, config| {
-                ODM_HOOK_CALLS.lock().unwrap().push((bucket.to_string(), config.cloned()));
-            })
-        });
-    }
-
-    fn odm_hook_calls(bucket: &str) -> Vec<Option<OnDemandMigrationConfig>> {
-        ODM_HOOK_CALLS
-            .lock()
-            .unwrap()
-            .iter()
-            .filter(|(name, _)| name == bucket)
-            .map(|(_, config)| config.clone())
-            .collect()
-    }
-
-    /// rustfs/backlog#2148: the accessor reports absence as `Ok(None)` and a
-    /// stored payload it cannot parse as a typed error, never as a default
-    /// and never as `ConfigNotFound`.
-    #[tokio::test]
-    async fn get_on_demand_migration_config_distinguishes_absent_from_corrupt() {
-        use crate::bucket::on_demand_migration::OnDemandMigrationConfigError;
-
-        let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
-        let sys = BucketMetadataSys::new(ecstore);
-        let bucket = "odm-accessor";
-
-        sys.set(bucket.to_string(), Arc::new(BucketMetadata::new(bucket))).await;
-        assert_eq!(sys.get_on_demand_migration_config(bucket).await.unwrap(), None);
-
-        let mut corrupt = BucketMetadata::new(bucket);
-        corrupt.on_demand_migration_config_json = br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec();
-        sys.set(bucket.to_string(), Arc::new(corrupt)).await;
-        let err = sys
-            .get_on_demand_migration_config(bucket)
-            .await
-            .expect_err("corrupt config must not read as a default");
-        assert_ne!(err, Error::ConfigNotFound, "corruption must not be reported as absence");
-        let typed = match &err {
-            Error::Io(io) => io
-                .get_ref()
-                .and_then(|source| source.downcast_ref::<OnDemandMigrationConfigError>()),
-            _ => None,
-        };
-        assert!(
-            matches!(typed, Some(OnDemandMigrationConfigError::Malformed(_))),
-            "typed parse error must survive the Result boundary, got: {err:?}"
-        );
-
-        let mut valid = BucketMetadata::new(bucket);
-        valid
-            .update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .unwrap();
-        let stamped = valid.on_demand_migration_config_updated_at;
-        sys.set(bucket.to_string(), Arc::new(valid)).await;
-        let (config, updated_at) = sys
-            .get_on_demand_migration_config(bucket)
-            .await
-            .unwrap()
-            .expect("stored config is returned");
-        assert_eq!(config, OnDemandMigrationConfig::from_json(ODM_JSON).unwrap());
-        assert_eq!(updated_at, stamped);
-    }
-
-    /// rustfs/backlog#2148: the publish hook fires on every path that
-    /// installs bucket metadata into the cache (set, initial load, peer
-    /// reload, refresh loop, lazy load) and withdraws on removal, mirroring
-    /// `sync_bucket_durability`.
-    #[tokio::test]
-    async fn on_demand_migration_hook_fires_on_every_cache_install_path() {
-        install_recording_odm_hook();
-
-        let (dirs, ecstore) = isolated_store_over_temp_disks().await;
-        let bucket = "odm-hook-paths";
-        for dir in &dirs {
-            std::fs::create_dir_all(dir.path().join(bucket)).expect("physical bucket should exist");
-        }
-        let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
-        let expect_publish = |before: usize, label: &str| {
-            let calls = odm_hook_calls(bucket);
-            assert_eq!(calls.len(), before + 1, "{label} must publish exactly once");
-            assert_eq!(calls.last().unwrap().as_ref(), Some(&expected), "{label} must publish the stored config");
-        };
-
-        // set (via persist_new_and_set, which installs through `set`).
-        let mut bm = BucketMetadata::new(bucket);
-        bm.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .unwrap();
-        let writer = BucketMetadataSys::new(ecstore.clone());
-        let before = odm_hook_calls(bucket).len();
-        writer.persist_new_and_set(bm).await.expect("metadata should persist");
-        expect_publish(before, "set");
-
-        // init (initial load on a cold system).
-        let mut cold = BucketMetadataSys::new(ecstore.clone());
-        let before = odm_hook_calls(bucket).len();
-        cold.init(vec![bucket.to_string()]).await;
-        assert!(cold.get(bucket).await.is_ok(), "initial load must cache the bucket");
-        expect_publish(before, "init");
-
-        // peer reload.
-        let before = odm_hook_calls(bucket).len();
-        cold.reload_from_store(bucket).await.expect("peer reload should publish");
-        expect_publish(before, "peer reload");
-
-        // refresh loop.
-        let before = odm_hook_calls(bucket).len();
-        let mut failed = HashSet::new();
-        cold.concurrent_load(&[bucket.to_string()], &mut failed, MetadataLoadMode::Refresh)
-            .await;
-        assert!(failed.is_empty(), "refresh must succeed");
-        expect_publish(before, "refresh loop");
-
-        // lazy load on another cold system.
-        let lazy = BucketMetadataSys::new(ecstore);
-        let before = odm_hook_calls(bucket).len();
-        let (_, loaded) = lazy.get_config(bucket).await.expect("lazy load should publish");
-        assert!(loaded, "the lazy path must have gone to disk");
-        expect_publish(before, "lazy load");
-
-        // Removal withdraws the config.
-        let before = odm_hook_calls(bucket).len();
-        assert!(lazy.remove(bucket).await);
-        let calls = odm_hook_calls(bucket);
-        assert_eq!(calls.len(), before + 1, "remove must withdraw exactly once");
-        assert_eq!(calls.last().unwrap(), &None);
-
-        // A corrupt payload is withdrawn, never published as a config.
-        let mut corrupt = BucketMetadata::new(bucket);
-        corrupt.on_demand_migration_config_json = b"not-json".to_vec();
-        let before = odm_hook_calls(bucket).len();
-        lazy.set(bucket.to_string(), Arc::new(corrupt)).await;
-        let calls = odm_hook_calls(bucket);
-        assert_eq!(calls.len(), before + 1);
-        assert_eq!(calls.last().unwrap(), &None, "unreadable config must publish absence");
-    }
-
     #[tokio::test]
     async fn refresh_wait_exits_when_cancelled() {
         let cancel_token = CancellationToken::new();

crates/ecstore/src/bucket/mod.rs

@@ -26,10 +26,8 @@ mod metadata_test;
 pub mod migration;
 mod msgp_decode;
 pub mod object_lock;
-pub mod on_demand_migration;
 pub mod policy_sys;
 pub mod quota;
-pub mod remote_s3_client;
 pub mod replication;
 pub mod tagging;
 pub mod target;

crates/ecstore/src/bucket/object_lock/objectlock_sys.rs

@@ -177,28 +177,6 @@ pub fn replication_write_may_pass_worm_gate(
     Ok(!(retention_locked && opts.replication_retention_timestamp.is_none()))
 }
 
-/// Whether an authorized replication delete (`ObjectOptions::replication_request`)
-/// addressed to an explicit version may bypass GOVERNANCE retention on the
-/// local replica, exactly as an `x-amz-bypass-governance-retention` caller
-/// with the bypass permission would.
-///
-/// The source is authoritative for a replicated version purge (issue #6850):
-/// the same WORM deletion gate already ran there, and GOVERNANCE retention
-/// with an authorized bypass is the only lock state it can purge through.
-/// Requiring the bypass header again here makes the purge permanently
-/// undeliverable — replication senders never carry it — and the sites diverge
-/// forever. COMPLIANCE retention and legal hold stay blocking: the source
-/// gate can never purge through them, so a replication purge that meets one
-/// here is divergence or forgery and fails closed.
-///
-/// The trust judgment is the same one the write-path exemption uses:
-/// `replication_request` is only set once the receiving handler has
-/// authorized the caller for the replication action
-/// (`ReplicateDeleteAction`), never straight from request headers.
-pub fn replication_delete_may_bypass_governance(opts: &ObjectOptions) -> bool {
-    opts.replication_request && opts.version_id.is_some()
-}
-
 /// Check if an object is locked based on its metadata.
 /// This is a common function used by both lifecycle evaluation and deletion checks.
 ///
@@ -702,32 +680,6 @@ mod tests {
         assert!(err.to_string().contains("modification time"));
     }
 
-    /// The replicated-purge GOVERNANCE bypass (#6850) applies only to an
-    /// authorized replication delete addressed to an explicit version: a
-    /// local delete never gets it, and a replicated delete without a version
-    /// id creates a delete marker rather than purging anything.
-    #[test]
-    fn replication_delete_bypasses_governance_only_for_authorized_version_purges() {
-        let version_purge = ObjectOptions {
-            replication_request: true,
-            version_id: Some("6b6ffbc0-b0d3-4a86-8f6c-fe19163b8dcd".to_string()),
-            ..Default::default()
-        };
-        assert!(replication_delete_may_bypass_governance(&version_purge));
-
-        let local_version_delete = ObjectOptions {
-            replication_request: false,
-            ..version_purge.clone()
-        };
-        assert!(!replication_delete_may_bypass_governance(&local_version_delete));
-
-        let replicated_marker_creation = ObjectOptions {
-            version_id: None,
-            ..version_purge
-        };
-        assert!(!replication_delete_may_bypass_governance(&replicated_marker_creation));
-    }
-
     /// A local PutObjectRetention / PutObjectLegalHold "clear" persists the
     /// lock keys as empty strings (the MinIO on-disk shape, see
     /// `parse_object_lock_retention`); that is "no lock", not corruption, and

crates/ecstore/src/bucket/on_demand_migration/breaker.rs

@@ -1,362 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Per-bucket three-state circuit breaker protecting an on-demand migration
-//! source (rustfs/backlog#2152).
-//!
-//! `Closed` lets every request through and counts consecutive failures
-//! inside a sliding window; reaching the threshold opens the breaker. `Open`
-//! rejects everything until the open duration elapses, then moves to
-//! `HalfOpen`, which admits a single probe: success closes the breaker,
-//! failure re-opens it. Timing uses `tokio::time::Instant` so tests can drive
-//! it with `tokio::time::pause`.
-//!
-//! Only transport-level failures count (`Throttled`, `Timeout`, `Connect`,
-//! `ServerError`). `NotFound` is a healthy answer and resets the failure
-//! streak; `AccessDenied`, `Unsupported` and `Other` are configuration or
-//! object problems that neither open nor close the breaker.
-
-use super::source_client::SourceError;
-use parking_lot::Mutex;
-use serde::{Deserialize, Serialize};
-use std::time::Duration;
-use tokio::time::Instant;
-
-/// Consecutive counted failures that open the breaker.
-pub const BREAKER_FAILURE_THRESHOLD: u32 = 5;
-/// Failures further apart than this do not accumulate.
-pub const BREAKER_FAILURE_WINDOW: Duration = Duration::from_secs(30);
-/// How long an open breaker rejects before admitting a probe.
-pub const BREAKER_OPEN_DURATION: Duration = Duration::from_secs(30);
-/// Probes admitted while half-open.
-pub const BREAKER_HALF_OPEN_MAX_PROBES: u32 = 1;
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub enum BreakerState {
-    Closed,
-    Open,
-    HalfOpen,
-}
-
-impl BreakerState {
-    pub fn as_str(self) -> &'static str {
-        match self {
-            BreakerState::Closed => "closed",
-            BreakerState::Open => "open",
-            BreakerState::HalfOpen => "half_open",
-        }
-    }
-}
-
-/// A state change the caller may want to log.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub struct BreakerTransition {
-    pub from: BreakerState,
-    pub to: BreakerState,
-}
-
-/// How a source result is scored by the breaker.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum BreakerVerdict {
-    /// Resets the failure streak; closes a half-open breaker.
-    Success,
-    /// Counts toward the threshold; re-opens a half-open breaker.
-    Failure,
-    /// Leaves the breaker untouched.
-    Neutral,
-}
-
-impl BreakerVerdict {
-    /// `None` is a successful source call.
-    pub fn for_result(error: Option<&SourceError>) -> Self {
-        match error {
-            None | Some(SourceError::NotFound) => BreakerVerdict::Success,
-            Some(SourceError::Throttled | SourceError::Timeout | SourceError::Connect(_) | SourceError::ServerError(_)) => {
-                BreakerVerdict::Failure
-            }
-            Some(SourceError::AccessDenied | SourceError::Unsupported(_) | SourceError::Other(_)) => BreakerVerdict::Neutral,
-        }
-    }
-}
-
-#[derive(Debug)]
-struct Inner {
-    state: BreakerState,
-    consecutive_failures: u32,
-    last_failure_at: Option<Instant>,
-    opened_at: Option<Instant>,
-    half_open_probes: u32,
-}
-
-#[derive(Debug)]
-pub struct Breaker {
-    inner: Mutex<Inner>,
-}
-
-impl Default for Breaker {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl Breaker {
-    pub fn new() -> Self {
-        Self {
-            inner: Mutex::new(Inner {
-                state: BreakerState::Closed,
-                consecutive_failures: 0,
-                last_failure_at: None,
-                opened_at: None,
-                half_open_probes: 0,
-            }),
-        }
-    }
-
-    /// Current state after applying the open-duration timeout.
-    pub fn state(&self) -> BreakerState {
-        let mut inner = self.inner.lock();
-        Self::advance(&mut inner, Instant::now());
-        inner.state
-    }
-
-    /// Whether a request may reach the source right now. Consumes the
-    /// half-open probe budget when it grants one.
-    pub fn allow_request(&self) -> bool {
-        let mut inner = self.inner.lock();
-        Self::advance(&mut inner, Instant::now());
-        match inner.state {
-            BreakerState::Closed => true,
-            BreakerState::Open => false,
-            BreakerState::HalfOpen => {
-                if inner.half_open_probes < BREAKER_HALF_OPEN_MAX_PROBES {
-                    inner.half_open_probes += 1;
-                    true
-                } else {
-                    false
-                }
-            }
-        }
-    }
-
-    /// Scores a source result; returns the transition it caused, if any.
-    pub fn record(&self, verdict: BreakerVerdict) -> Option<BreakerTransition> {
-        match verdict {
-            BreakerVerdict::Success => self.record_success(),
-            BreakerVerdict::Failure => self.record_failure(),
-            BreakerVerdict::Neutral => None,
-        }
-    }
-
-    pub fn record_success(&self) -> Option<BreakerTransition> {
-        let mut inner = self.inner.lock();
-        let now = Instant::now();
-        Self::advance(&mut inner, now);
-        inner.consecutive_failures = 0;
-        inner.last_failure_at = None;
-        match inner.state {
-            BreakerState::Closed => None,
-            // A success while open can only come from a request admitted
-            // before the breaker opened; it says nothing about recovery.
-            BreakerState::Open => None,
-            BreakerState::HalfOpen => Some(Self::transition(&mut inner, BreakerState::Closed, now)),
-        }
-    }
-
-    pub fn record_failure(&self) -> Option<BreakerTransition> {
-        let mut inner = self.inner.lock();
-        let now = Instant::now();
-        Self::advance(&mut inner, now);
-        match inner.state {
-            BreakerState::Closed => {
-                let within_window = inner
-                    .last_failure_at
-                    .is_some_and(|last| now.saturating_duration_since(last) <= BREAKER_FAILURE_WINDOW);
-                inner.consecutive_failures = if within_window { inner.consecutive_failures + 1 } else { 1 };
-                inner.last_failure_at = Some(now);
-                if inner.consecutive_failures >= BREAKER_FAILURE_THRESHOLD {
-                    Some(Self::transition(&mut inner, BreakerState::Open, now))
-                } else {
-                    None
-                }
-            }
-            BreakerState::Open => None,
-            BreakerState::HalfOpen => Some(Self::transition(&mut inner, BreakerState::Open, now)),
-        }
-    }
-
-    fn advance(inner: &mut Inner, now: Instant) {
-        if inner.state == BreakerState::Open
-            && inner
-                .opened_at
-                .is_some_and(|opened| now.saturating_duration_since(opened) >= BREAKER_OPEN_DURATION)
-        {
-            Self::transition(inner, BreakerState::HalfOpen, now);
-        }
-    }
-
-    fn transition(inner: &mut Inner, to: BreakerState, now: Instant) -> BreakerTransition {
-        let from = inner.state;
-        inner.state = to;
-        match to {
-            BreakerState::Open => {
-                inner.opened_at = Some(now);
-                inner.half_open_probes = 0;
-            }
-            BreakerState::HalfOpen => {
-                inner.half_open_probes = 0;
-            }
-            BreakerState::Closed => {
-                inner.opened_at = None;
-                inner.half_open_probes = 0;
-                inner.consecutive_failures = 0;
-                inner.last_failure_at = None;
-            }
-        }
-        BreakerTransition { from, to }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    fn server_error() -> SourceError {
-        SourceError::ServerError(503)
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn five_failures_open_then_half_open_after_timeout() {
-        let breaker = Breaker::new();
-        for i in 0..BREAKER_FAILURE_THRESHOLD - 1 {
-            assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&server_error()))), None, "failure {i}");
-            assert_eq!(breaker.state(), BreakerState::Closed);
-        }
-        assert_eq!(
-            breaker.record(BreakerVerdict::for_result(Some(&server_error()))),
-            Some(BreakerTransition {
-                from: BreakerState::Closed,
-                to: BreakerState::Open
-            })
-        );
-        assert_eq!(breaker.state(), BreakerState::Open);
-        assert!(!breaker.allow_request());
-
-        tokio::time::advance(BREAKER_OPEN_DURATION - Duration::from_secs(1)).await;
-        assert!(!breaker.allow_request());
-        assert_eq!(breaker.state(), BreakerState::Open);
-
-        tokio::time::advance(Duration::from_secs(1)).await;
-        assert_eq!(breaker.state(), BreakerState::HalfOpen);
-        assert!(breaker.allow_request(), "one probe is admitted");
-        assert!(!breaker.allow_request(), "second probe is rejected");
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn half_open_probe_success_closes_and_failure_reopens() {
-        let breaker = Breaker::new();
-        for _ in 0..BREAKER_FAILURE_THRESHOLD {
-            breaker.record_failure();
-        }
-        tokio::time::advance(BREAKER_OPEN_DURATION).await;
-        assert!(breaker.allow_request());
-        assert_eq!(
-            breaker.record_failure(),
-            Some(BreakerTransition {
-                from: BreakerState::HalfOpen,
-                to: BreakerState::Open
-            })
-        );
-        assert!(!breaker.allow_request());
-
-        tokio::time::advance(BREAKER_OPEN_DURATION).await;
-        assert!(breaker.allow_request());
-        assert_eq!(
-            breaker.record_success(),
-            Some(BreakerTransition {
-                from: BreakerState::HalfOpen,
-                to: BreakerState::Closed
-            })
-        );
-        assert_eq!(breaker.state(), BreakerState::Closed);
-        assert!(breaker.allow_request());
-        // The streak restarts from zero after closing.
-        for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
-            assert_eq!(breaker.record_failure(), None);
-        }
-        assert_eq!(breaker.state(), BreakerState::Closed);
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn failures_outside_window_do_not_accumulate() {
-        let breaker = Breaker::new();
-        for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
-            breaker.record_failure();
-        }
-        tokio::time::advance(BREAKER_FAILURE_WINDOW + Duration::from_secs(1)).await;
-        assert_eq!(breaker.record_failure(), None, "stale streak restarts at one");
-        assert_eq!(breaker.state(), BreakerState::Closed);
-    }
-
-    #[test]
-    fn not_found_and_access_denied_do_not_count() {
-        let breaker = Breaker::new();
-        for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
-            breaker.record(BreakerVerdict::for_result(Some(&server_error())));
-        }
-        assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::AccessDenied))), None);
-        assert_eq!(breaker.state(), BreakerState::Closed);
-        // AccessDenied is neutral: the streak is still one short of opening.
-        assert_eq!(
-            breaker.record(BreakerVerdict::for_result(Some(&SourceError::Unsupported("sse-c".into())))),
-            None
-        );
-        assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::Other("x".into())))), None);
-        // NotFound is a healthy answer and resets the streak entirely.
-        assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::NotFound))), None);
-        for _ in 0..BREAKER_FAILURE_THRESHOLD - 1 {
-            assert_eq!(breaker.record(BreakerVerdict::for_result(Some(&SourceError::Timeout))), None);
-        }
-        assert_eq!(breaker.state(), BreakerState::Closed);
-    }
-
-    #[test]
-    fn verdicts_cover_every_source_error_class() {
-        assert_eq!(BreakerVerdict::for_result(None), BreakerVerdict::Success);
-        assert_eq!(BreakerVerdict::for_result(Some(&SourceError::NotFound)), BreakerVerdict::Success);
-        for failure in [
-            SourceError::Throttled,
-            SourceError::Timeout,
-            SourceError::Connect("refused".into()),
-            SourceError::ServerError(500),
-        ] {
-            assert_eq!(BreakerVerdict::for_result(Some(&failure)), BreakerVerdict::Failure, "{failure:?}");
-        }
-        for neutral in [
-            SourceError::AccessDenied,
-            SourceError::Unsupported("sse-c".into()),
-            SourceError::Other("x".into()),
-        ] {
-            assert_eq!(BreakerVerdict::for_result(Some(&neutral)), BreakerVerdict::Neutral, "{neutral:?}");
-        }
-    }
-
-    #[test]
-    fn state_labels_are_stable() {
-        assert_eq!(BreakerState::Closed.as_str(), "closed");
-        assert_eq!(BreakerState::Open.as_str(), "open");
-        assert_eq!(BreakerState::HalfOpen.as_str(), "half_open");
-        assert_eq!(serde_json::to_string(&BreakerState::HalfOpen).unwrap(), "\"half_open\"");
-    }
-}

crates/ecstore/src/bucket/on_demand_migration/mod.rs

@@ -1,47 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! On-Demand Migration (ODM): a bucket can name an external S3-compatible
-//! source bucket; GET misses are served from that source and backfilled
-//! locally. This module owns the bucket-level configuration model
-//! (`on-demand-migration.json` in the bucket metadata file), the source
-//! client, and the per-node runtime (`sys`) that turns configs into live
-//! clients guarded by a breaker, a negative cache, singleflight and a pull
-//! concurrency limit (rustfs/backlog#2147).
-
-pub mod breaker;
-pub mod config;
-pub mod negative_cache;
-pub mod source_client;
-pub mod stats;
-pub mod sys;
-
-pub use breaker::{
-    BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION, Breaker,
-    BreakerState, BreakerTransition, BreakerVerdict,
-};
-pub use config::{
-    ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION,
-    OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, SourceConfig,
-    SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
-};
-pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache};
-pub use stats::{
-    GaugeGuard, LastSourceError, LatencyBucketSnapshot, OdmOp, OdmOutcome, OdmStats, OdmStatsSnapshot, PullFailureReason,
-    PullPath, SOURCE_LATENCY_BUCKET_BOUNDS_MS, SourceLatencySnapshot,
-};
-pub use sys::{
-    ApplyOutcome, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, OdmBucketSnapshot, OdmLookup, OdmStateError,
-    OnDemandMigrationSys, PullError, PullFollower, PullLeader, PullOutcome, PullResult, PullSlot, source_client_spec,
-};

crates/ecstore/src/bucket/on_demand_migration/negative_cache.rs

@@ -1,130 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Per-bucket cache of keys the source answered 404 for
-//! (rustfs/backlog#2152). A hit short-circuits the source lookup for
-//! `policy.negative_cache_ttl_secs`; a TTL of zero disables the cache.
-//!
-//! Entries are never invalidated on a local PUT: once the object exists
-//! locally the handler never consults ODM for it, so a stale negative entry
-//! is harmless.
-
-use std::time::Duration;
-
-/// Upper bound on remembered keys per bucket; LRU eviction beyond it.
-pub const NEGATIVE_CACHE_MAX_ENTRIES: u64 = 100_000;
-
-#[derive(Debug)]
-pub struct NegativeCache {
-    cache: Option<moka::sync::Cache<String, ()>>,
-    ttl: Duration,
-}
-
-impl NegativeCache {
-    /// `ttl == 0` builds a disabled cache that never records anything.
-    pub fn new(ttl: Duration) -> Self {
-        Self::with_capacity(ttl, NEGATIVE_CACHE_MAX_ENTRIES)
-    }
-
-    pub fn with_capacity(ttl: Duration, max_entries: u64) -> Self {
-        let cache = (!ttl.is_zero()).then(|| {
-            moka::sync::Cache::builder()
-                .max_capacity(max_entries)
-                .time_to_live(ttl)
-                .build()
-        });
-        Self { cache, ttl }
-    }
-
-    pub fn is_enabled(&self) -> bool {
-        self.cache.is_some()
-    }
-
-    pub fn ttl(&self) -> Duration {
-        self.ttl
-    }
-
-    /// Whether `key` is currently remembered as absent on the source.
-    pub fn contains(&self, key: &str) -> bool {
-        self.cache.as_ref().is_some_and(|cache| cache.get(key).is_some())
-    }
-
-    /// Remembers `key` as absent; no-op when disabled.
-    pub fn insert(&self, key: &str) {
-        if let Some(cache) = &self.cache {
-            cache.insert(key.to_string(), ());
-        }
-    }
-
-    /// Forgets `key` (e.g. after an admin-triggered backfill found it).
-    pub fn remove(&self, key: &str) {
-        if let Some(cache) = &self.cache {
-            cache.invalidate(key);
-        }
-    }
-
-    /// Approximate live entry count, for status snapshots only.
-    pub fn len(&self) -> u64 {
-        self.cache.as_ref().map_or(0, |cache| {
-            cache.run_pending_tasks();
-            cache.entry_count()
-        })
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.len() == 0
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn entry_expires_after_ttl() {
-        let cache = NegativeCache::new(Duration::from_millis(80));
-        assert!(cache.is_enabled());
-        cache.insert("a/x");
-        assert!(cache.contains("a/x"));
-        assert!(!cache.contains("a/y"));
-        std::thread::sleep(Duration::from_millis(160));
-        assert!(!cache.contains("a/x"), "entry must expire after the TTL");
-    }
-
-    #[test]
-    fn zero_ttl_disables_the_cache() {
-        let cache = NegativeCache::new(Duration::ZERO);
-        assert!(!cache.is_enabled());
-        cache.insert("a/x");
-        assert!(!cache.contains("a/x"));
-        assert!(cache.is_empty());
-    }
-
-    #[test]
-    fn remove_forgets_a_key() {
-        let cache = NegativeCache::new(Duration::from_secs(30));
-        cache.insert("a/x");
-        cache.remove("a/x");
-        assert!(!cache.contains("a/x"));
-    }
-
-    #[test]
-    fn capacity_bounds_entries() {
-        let cache = NegativeCache::with_capacity(Duration::from_secs(30), 4);
-        for i in 0..64 {
-            cache.insert(&format!("k{i}"));
-        }
-        assert!(cache.len() <= 4, "len {} exceeds capacity", cache.len());
-    }
-}

crates/ecstore/src/bucket/on_demand_migration/stats.rs

@@ -1,527 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Per-bucket on-demand migration counters (rustfs/backlog#2152).
-//!
-//! `OdmStats` is lock-free and survives config rebuilds; `snapshot()` turns
-//! it into the serializable `OdmStatsSnapshot` that the metrics collector
-//! and the admin status route (ODM-10/14/15) consume. Field names and label
-//! values are a wire contract: the golden JSON test below pins them.
-
-use super::breaker::BreakerState;
-use super::source_client::SourceError;
-use parking_lot::Mutex;
-use serde::{Deserialize, Serialize};
-use std::collections::BTreeMap;
-use std::sync::Arc;
-use std::sync::atomic::{AtomicU64, Ordering};
-use std::time::Duration;
-use time::OffsetDateTime;
-
-/// Request operations that can enter ODM.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum OdmOp {
-    Get,
-    Head,
-}
-
-impl OdmOp {
-    pub const ALL: [OdmOp; 2] = [OdmOp::Get, OdmOp::Head];
-
-    pub fn as_str(self) -> &'static str {
-        match self {
-            OdmOp::Get => "get",
-            OdmOp::Head => "head",
-        }
-    }
-}
-
-/// How a request that entered ODM ended. `local_hit` is deliberately absent:
-/// requests served locally never reach the runtime.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum OdmOutcome {
-    SourceHit,
-    SourceMiss,
-    SourceError,
-    BreakerOpen,
-    NegativeCached,
-    Filtered,
-    Unsupported,
-}
-
-impl OdmOutcome {
-    pub const ALL: [OdmOutcome; 7] = [
-        OdmOutcome::SourceHit,
-        OdmOutcome::SourceMiss,
-        OdmOutcome::SourceError,
-        OdmOutcome::BreakerOpen,
-        OdmOutcome::NegativeCached,
-        OdmOutcome::Filtered,
-        OdmOutcome::Unsupported,
-    ];
-
-    pub fn as_str(self) -> &'static str {
-        match self {
-            OdmOutcome::SourceHit => "source_hit",
-            OdmOutcome::SourceMiss => "source_miss",
-            OdmOutcome::SourceError => "source_error",
-            OdmOutcome::BreakerOpen => "breaker_open",
-            OdmOutcome::NegativeCached => "negative_cached",
-            OdmOutcome::Filtered => "filtered",
-            OdmOutcome::Unsupported => "unsupported",
-        }
-    }
-}
-
-/// Which pipeline stored a pulled object locally.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum PullPath {
-    /// Streamed to the client and written locally in one pass.
-    Inline,
-    /// Pulled by a background task after a partial/large read.
-    Background,
-    /// Pulled by the backfill job.
-    Backfill,
-}
-
-impl PullPath {
-    pub const ALL: [PullPath; 3] = [PullPath::Inline, PullPath::Background, PullPath::Backfill];
-
-    pub fn as_str(self) -> &'static str {
-        match self {
-            PullPath::Inline => "inline",
-            PullPath::Background => "background",
-            PullPath::Backfill => "backfill",
-        }
-    }
-}
-
-/// Why a pull did not produce a local object.
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub enum PullFailureReason {
-    SourceNotFound,
-    SourceAccessDenied,
-    SourceThrottled,
-    SourceTimeout,
-    SourceConnect,
-    SourceServerError,
-    SourceUnsupported,
-    SourceOther,
-    /// Source bytes did not match the ETag advertised by HEAD/GET.
-    EtagMismatch,
-    /// The local write (internal PUT) failed.
-    LocalWrite,
-    /// The bucket state was removed or the process is shutting down.
-    Canceled,
-    /// The background pull queue was full.
-    QueueFull,
-}
-
-impl PullFailureReason {
-    pub const ALL: [PullFailureReason; 12] = [
-        PullFailureReason::SourceNotFound,
-        PullFailureReason::SourceAccessDenied,
-        PullFailureReason::SourceThrottled,
-        PullFailureReason::SourceTimeout,
-        PullFailureReason::SourceConnect,
-        PullFailureReason::SourceServerError,
-        PullFailureReason::SourceUnsupported,
-        PullFailureReason::SourceOther,
-        PullFailureReason::EtagMismatch,
-        PullFailureReason::LocalWrite,
-        PullFailureReason::Canceled,
-        PullFailureReason::QueueFull,
-    ];
-
-    pub fn as_str(self) -> &'static str {
-        match self {
-            PullFailureReason::SourceNotFound => "source_not_found",
-            PullFailureReason::SourceAccessDenied => "source_access_denied",
-            PullFailureReason::SourceThrottled => "source_throttled",
-            PullFailureReason::SourceTimeout => "source_timeout",
-            PullFailureReason::SourceConnect => "source_connect",
-            PullFailureReason::SourceServerError => "source_server_error",
-            PullFailureReason::SourceUnsupported => "source_unsupported",
-            PullFailureReason::SourceOther => "source_other",
-            PullFailureReason::EtagMismatch => "etag_mismatch",
-            PullFailureReason::LocalWrite => "local_write",
-            PullFailureReason::Canceled => "canceled",
-            PullFailureReason::QueueFull => "queue_full",
-        }
-    }
-}
-
-impl From<&SourceError> for PullFailureReason {
-    fn from(err: &SourceError) -> Self {
-        match err {
-            SourceError::NotFound => PullFailureReason::SourceNotFound,
-            SourceError::AccessDenied => PullFailureReason::SourceAccessDenied,
-            SourceError::Throttled => PullFailureReason::SourceThrottled,
-            SourceError::Timeout => PullFailureReason::SourceTimeout,
-            SourceError::Connect(_) => PullFailureReason::SourceConnect,
-            SourceError::ServerError(_) => PullFailureReason::SourceServerError,
-            SourceError::Unsupported(_) => PullFailureReason::SourceUnsupported,
-            SourceError::Other(_) => PullFailureReason::SourceOther,
-        }
-    }
-}
-
-/// Upper bounds (milliseconds) of the source latency histogram buckets; the
-/// implicit last bucket is unbounded. Roughly logarithmic from 5 ms to 60 s.
-pub const SOURCE_LATENCY_BUCKET_BOUNDS_MS: [u64; 14] = [
-    5, 10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000, 20_000, 30_000, 60_000,
-];
-
-#[derive(Debug, Default)]
-struct LatencyHistogram {
-    /// One counter per bound plus one for the overflow bucket.
-    buckets: [AtomicU64; SOURCE_LATENCY_BUCKET_BOUNDS_MS.len() + 1],
-    count: AtomicU64,
-    sum_ms: AtomicU64,
-}
-
-impl LatencyHistogram {
-    fn observe(&self, latency: Duration) {
-        let ms = u64::try_from(latency.as_millis()).unwrap_or(u64::MAX);
-        let index = SOURCE_LATENCY_BUCKET_BOUNDS_MS
-            .iter()
-            .position(|bound| ms <= *bound)
-            .unwrap_or(SOURCE_LATENCY_BUCKET_BOUNDS_MS.len());
-        self.buckets[index].fetch_add(1, Ordering::Relaxed);
-        self.count.fetch_add(1, Ordering::Relaxed);
-        self.sum_ms.fetch_add(ms, Ordering::Relaxed);
-    }
-
-    fn snapshot(&self) -> SourceLatencySnapshot {
-        let mut cumulative = 0;
-        let buckets = SOURCE_LATENCY_BUCKET_BOUNDS_MS
-            .iter()
-            .zip(self.buckets.iter())
-            .map(|(bound, counter)| {
-                cumulative += counter.load(Ordering::Relaxed);
-                LatencyBucketSnapshot {
-                    le_ms: *bound,
-                    count: cumulative,
-                }
-            })
-            .collect();
-        SourceLatencySnapshot {
-            buckets,
-            count: self.count.load(Ordering::Relaxed),
-            sum_ms: self.sum_ms.load(Ordering::Relaxed),
-        }
-    }
-}
-
-/// The most recent source failure, kept for operators: class only, never the
-/// key or the message (which may echo attacker-controlled input).
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct LastSourceError {
-    pub class: String,
-    #[serde(with = "time::serde::rfc3339")]
-    pub at: OffsetDateTime,
-}
-
-#[derive(Debug, Default)]
-pub struct OdmStats {
-    requests_total: [[AtomicU64; OdmOutcome::ALL.len()]; OdmOp::ALL.len()],
-    pulled_bytes_total: AtomicU64,
-    pulled_objects_total: [AtomicU64; PullPath::ALL.len()],
-    pull_failures_total: [AtomicU64; PullFailureReason::ALL.len()],
-    inflight_pulls: AtomicU64,
-    queue_depth: AtomicU64,
-    source_latency: LatencyHistogram,
-    last_source_error: Mutex<Option<LastSourceError>>,
-}
-
-impl OdmStats {
-    pub fn new() -> Self {
-        Self::default()
-    }
-
-    pub fn record_request(&self, op: OdmOp, outcome: OdmOutcome) {
-        self.requests_total[op as usize][outcome as usize].fetch_add(1, Ordering::Relaxed);
-    }
-
-    pub fn record_pulled_bytes(&self, bytes: u64) {
-        self.pulled_bytes_total.fetch_add(bytes, Ordering::Relaxed);
-    }
-
-    pub fn record_pulled_object(&self, path: PullPath) {
-        self.pulled_objects_total[path as usize].fetch_add(1, Ordering::Relaxed);
-    }
-
-    pub fn record_pull_failure(&self, reason: PullFailureReason) {
-        self.pull_failures_total[reason as usize].fetch_add(1, Ordering::Relaxed);
-    }
-
-    pub fn record_source_latency(&self, latency: Duration) {
-        self.source_latency.observe(latency);
-    }
-
-    pub fn record_source_error(&self, err: &SourceError) {
-        self.record_source_error_at(err, OffsetDateTime::now_utc());
-    }
-
-    pub fn record_source_error_at(&self, err: &SourceError, at: OffsetDateTime) {
-        *self.last_source_error.lock() = Some(LastSourceError {
-            class: err.class_label().to_string(),
-            at,
-        });
-    }
-
-    pub fn last_source_error(&self) -> Option<LastSourceError> {
-        self.last_source_error.lock().clone()
-    }
-
-    pub fn inflight_pulls(&self) -> u64 {
-        self.inflight_pulls.load(Ordering::Relaxed)
-    }
-
-    pub fn queue_depth(&self) -> u64 {
-        self.queue_depth.load(Ordering::Relaxed)
-    }
-
-    /// RAII increment of `inflight_pulls`.
-    pub fn inflight_guard(self: &Arc<Self>) -> GaugeGuard {
-        GaugeGuard::new(Arc::clone(self), OdmGauge::InflightPulls)
-    }
-
-    /// RAII increment of `queue_depth`.
-    pub fn queue_guard(self: &Arc<Self>) -> GaugeGuard {
-        GaugeGuard::new(Arc::clone(self), OdmGauge::QueueDepth)
-    }
-
-    fn gauge(&self, gauge: OdmGauge) -> &AtomicU64 {
-        match gauge {
-            OdmGauge::InflightPulls => &self.inflight_pulls,
-            OdmGauge::QueueDepth => &self.queue_depth,
-        }
-    }
-
-    /// Read-only, side-effect-free copy of every counter. The breaker lives
-    /// next to the stats in the bucket state; its state is passed in so the
-    /// snapshot stays a single document.
-    pub fn snapshot(&self, breaker_state: BreakerState) -> OdmStatsSnapshot {
-        let mut requests_total = BTreeMap::new();
-        for op in OdmOp::ALL {
-            let mut by_outcome = BTreeMap::new();
-            for outcome in OdmOutcome::ALL {
-                by_outcome.insert(
-                    outcome.as_str().to_string(),
-                    self.requests_total[op as usize][outcome as usize].load(Ordering::Relaxed),
-                );
-            }
-            requests_total.insert(op.as_str().to_string(), by_outcome);
-        }
-        let pulled_objects_total = PullPath::ALL
-            .iter()
-            .map(|path| {
-                (
-                    path.as_str().to_string(),
-                    self.pulled_objects_total[*path as usize].load(Ordering::Relaxed),
-                )
-            })
-            .collect();
-        let pull_failures_total = PullFailureReason::ALL
-            .iter()
-            .map(|reason| {
-                (
-                    reason.as_str().to_string(),
-                    self.pull_failures_total[*reason as usize].load(Ordering::Relaxed),
-                )
-            })
-            .collect();
-        OdmStatsSnapshot {
-            requests_total,
-            pulled_bytes_total: self.pulled_bytes_total.load(Ordering::Relaxed),
-            pulled_objects_total,
-            pull_failures_total,
-            inflight_pulls: self.inflight_pulls(),
-            queue_depth: self.queue_depth(),
-            source_latency: self.source_latency.snapshot(),
-            last_source_error: self.last_source_error(),
-            breaker_state,
-        }
-    }
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-enum OdmGauge {
-    InflightPulls,
-    QueueDepth,
-}
-
-/// Increments a gauge on creation and decrements it on drop. Owns its
-/// `OdmStats` so it can live inside the pull slot handed to callers.
-#[derive(Debug)]
-pub struct GaugeGuard {
-    stats: Arc<OdmStats>,
-    gauge: OdmGauge,
-}
-
-impl GaugeGuard {
-    fn new(stats: Arc<OdmStats>, gauge: OdmGauge) -> Self {
-        stats.gauge(gauge).fetch_add(1, Ordering::Relaxed);
-        Self { stats, gauge }
-    }
-}
-
-impl Drop for GaugeGuard {
-    fn drop(&mut self) {
-        self.stats.gauge(self.gauge).fetch_sub(1, Ordering::Relaxed);
-    }
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct LatencyBucketSnapshot {
-    /// Upper bound of the bucket in milliseconds.
-    pub le_ms: u64,
-    /// Cumulative observations at or below `le_ms`.
-    pub count: u64,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct SourceLatencySnapshot {
-    pub buckets: Vec<LatencyBucketSnapshot>,
-    /// Total observations, including those above the last bound.
-    pub count: u64,
-    pub sum_ms: u64,
-}
-
-/// Serializable copy of [`OdmStats`]. Every key is snake_case and every
-/// label set is fixed, so consumers can rely on the document shape.
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub struct OdmStatsSnapshot {
-    /// `op -> outcome -> count`.
-    pub requests_total: BTreeMap<String, BTreeMap<String, u64>>,
-    pub pulled_bytes_total: u64,
-    /// `path -> count`.
-    pub pulled_objects_total: BTreeMap<String, u64>,
-    /// `reason -> count`.
-    pub pull_failures_total: BTreeMap<String, u64>,
-    pub inflight_pulls: u64,
-    pub queue_depth: u64,
-    pub source_latency: SourceLatencySnapshot,
-    pub last_source_error: Option<LastSourceError>,
-    pub breaker_state: BreakerState,
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use serde_json::json;
-    use time::macros::datetime;
-
-    #[test]
-    fn snapshot_matches_golden_json() {
-        let stats = Arc::new(OdmStats::new());
-        stats.record_request(OdmOp::Get, OdmOutcome::SourceHit);
-        stats.record_request(OdmOp::Get, OdmOutcome::SourceHit);
-        stats.record_request(OdmOp::Head, OdmOutcome::NegativeCached);
-        stats.record_pulled_bytes(4096);
-        stats.record_pulled_object(PullPath::Inline);
-        stats.record_pull_failure(PullFailureReason::from(&SourceError::Timeout));
-        stats.record_source_latency(Duration::from_millis(3));
-        stats.record_source_latency(Duration::from_millis(750));
-        stats.record_source_latency(Duration::from_secs(90));
-        stats.record_source_error_at(&SourceError::ServerError(502), datetime!(2026-09-02 10:00:00 UTC));
-        let _inflight = stats.inflight_guard();
-        let _queued = stats.queue_guard();
-
-        let snapshot = stats.snapshot(BreakerState::HalfOpen);
-        let actual = serde_json::to_value(&snapshot).unwrap();
-        let expected = json!({
-            "requests_total": {
-                "get": {
-                    "breaker_open": 0, "filtered": 0, "negative_cached": 0, "source_error": 0,
-                    "source_hit": 2, "source_miss": 0, "unsupported": 0
-                },
-                "head": {
-                    "breaker_open": 0, "filtered": 0, "negative_cached": 1, "source_error": 0,
-                    "source_hit": 0, "source_miss": 0, "unsupported": 0
-                }
-            },
-            "pulled_bytes_total": 4096,
-            "pulled_objects_total": { "backfill": 0, "background": 0, "inline": 1 },
-            "pull_failures_total": {
-                "canceled": 0, "etag_mismatch": 0, "local_write": 0, "queue_full": 0,
-                "source_access_denied": 0, "source_connect": 0, "source_not_found": 0, "source_other": 0,
-                "source_server_error": 0, "source_throttled": 0, "source_timeout": 1, "source_unsupported": 0
-            },
-            "inflight_pulls": 1,
-            "queue_depth": 1,
-            "source_latency": {
-                "buckets": [
-                    { "le_ms": 5, "count": 1 }, { "le_ms": 10, "count": 1 }, { "le_ms": 20, "count": 1 },
-                    { "le_ms": 50, "count": 1 }, { "le_ms": 100, "count": 1 }, { "le_ms": 200, "count": 1 },
-                    { "le_ms": 500, "count": 1 }, { "le_ms": 1000, "count": 2 }, { "le_ms": 2000, "count": 2 },
-                    { "le_ms": 5000, "count": 2 }, { "le_ms": 10000, "count": 2 }, { "le_ms": 20000, "count": 2 },
-                    { "le_ms": 30000, "count": 2 }, { "le_ms": 60000, "count": 2 }
-                ],
-                "count": 3,
-                "sum_ms": 90753
-            },
-            "last_source_error": { "class": "server_error", "at": "2026-09-02T10:00:00Z" },
-            "breaker_state": "half_open"
-        });
-        assert_eq!(actual, expected);
-
-        let round_trip: OdmStatsSnapshot = serde_json::from_value(actual).unwrap();
-        assert_eq!(round_trip, snapshot);
-    }
-
-    #[test]
-    fn gauges_return_to_zero_when_guards_drop() {
-        let stats = Arc::new(OdmStats::new());
-        {
-            let _a = stats.inflight_guard();
-            let _b = stats.inflight_guard();
-            let _c = stats.queue_guard();
-            assert_eq!(stats.inflight_pulls(), 2);
-            assert_eq!(stats.queue_depth(), 1);
-        }
-        assert_eq!(stats.inflight_pulls(), 0);
-        assert_eq!(stats.queue_depth(), 0);
-    }
-
-    #[test]
-    fn pull_failure_reason_covers_every_source_error_class() {
-        let cases = [
-            (SourceError::NotFound, PullFailureReason::SourceNotFound),
-            (SourceError::AccessDenied, PullFailureReason::SourceAccessDenied),
-            (SourceError::Throttled, PullFailureReason::SourceThrottled),
-            (SourceError::Timeout, PullFailureReason::SourceTimeout),
-            (SourceError::Connect("x".into()), PullFailureReason::SourceConnect),
-            (SourceError::ServerError(500), PullFailureReason::SourceServerError),
-            (SourceError::Unsupported("x".into()), PullFailureReason::SourceUnsupported),
-            (SourceError::Other("x".into()), PullFailureReason::SourceOther),
-        ];
-        for (err, reason) in cases {
-            assert_eq!(PullFailureReason::from(&err), reason, "{err:?}");
-            assert_eq!(serde_json::to_string(&reason).unwrap(), format!("\"{}\"", reason.as_str()));
-        }
-    }
-
-    #[test]
-    fn label_lists_are_exhaustive_and_unique() {
-        let outcomes: std::collections::BTreeSet<_> = OdmOutcome::ALL.iter().map(|o| o.as_str()).collect();
-        assert_eq!(outcomes.len(), OdmOutcome::ALL.len());
-        let reasons: std::collections::BTreeSet<_> = PullFailureReason::ALL.iter().map(|r| r.as_str()).collect();
-        assert_eq!(reasons.len(), PullFailureReason::ALL.len());
-        let paths: std::collections::BTreeSet<_> = PullPath::ALL.iter().map(|p| p.as_str()).collect();
-        assert_eq!(paths.len(), PullPath::ALL.len());
-    }
-}

crates/ecstore/src/bucket/remote_s3_client.rs

@@ -1,789 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Shared builder for outbound `aws_sdk_s3::Client`s.
-//!
-//! Replication targets (`bucket_target_sys`) and the on-demand migration
-//! source client build their remote clients from one neutral
-//! [`RemoteS3EndpointSpec`]: endpoint assembly, credential handling, path-style
-//! selection, custom CA / skip-TLS transports and the outbound SSRF gate all
-//! live here so both callers share exactly one policy. The gate keeps the
-//! relaxed replication semantics documented in
-//! `docs/operations/outbound-connection-policy.md`: private addresses are
-//! always allowed, loopback only behind `RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET`.
-
-use aws_credential_types::Credentials as SdkCredentials;
-use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future};
-use aws_sdk_s3::config::Region as SdkRegion;
-use aws_sdk_s3::config::RequestChecksumCalculation;
-use aws_sdk_s3::config::SharedCredentialsProvider;
-use aws_sdk_s3::config::SharedHttpClient;
-use aws_sdk_s3::{Client as S3Client, Config as S3Config};
-use aws_smithy_http_client::{Builder as SmithyHttpClientBuilder, tls as smithy_tls};
-use aws_smithy_runtime_api::box_error::BoxError;
-use aws_smithy_runtime_api::client::http::{
-    HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
-};
-use aws_smithy_runtime_api::client::interceptors::Intercept;
-use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
-use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
-use aws_smithy_runtime_api::client::result::ConnectorError;
-use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
-use aws_smithy_types::body::SdkBody;
-use aws_smithy_types::config_bag::ConfigBag;
-use aws_smithy_types::timeout::TimeoutConfig;
-use http::Uri;
-use hyper_util::client::legacy::Client as HyperClient;
-use hyper_util::rt::{TokioExecutor, TokioTimer};
-use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RUSTFS_CA_CERT, RUSTFS_TLS_CERT};
-use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url};
-use rustls_pki_types::pem::PemObject;
-use std::fmt;
-use std::path::Path;
-use std::sync::Arc;
-use std::time::{Duration, SystemTime};
-use tower::Service;
-use tracing::warn;
-use url::Url;
-
-const REDACTED_CREDENTIAL: &str = "<redacted>";
-pub(crate) const EXPIRED_REMOTE_TARGET_CREDENTIALS: &str = "remote target credentials have expired";
-
-/// Request addressing style for a remote S3-compatible endpoint.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum PathStyle {
-    /// Caller did not choose; the builder defaults to path-style because that
-    /// is what custom S3-compatible endpoints accept most reliably.
-    Auto,
-    /// `https://endpoint/bucket/key`.
-    Path,
-    /// `https://bucket.endpoint/key`.
-    VirtualHost,
-}
-
-impl PathStyle {
-    /// Resolves the style to the SDK `force_path_style` flag. `Auto` keeps
-    /// the historical replication default (path-style).
-    pub fn force_path_style(self) -> bool {
-        !matches!(self, PathStyle::VirtualHost)
-    }
-}
-
-/// Static or temporary credentials for a remote endpoint. `expiration` without
-/// a `session_token` is rejected at build time: only STS-style temporary
-/// credentials expire, so that combination is a corrupted configuration
-/// rather than a static key.
-#[derive(Clone)]
-pub struct RemoteCredentials {
-    pub access_key: String,
-    pub secret_key: String,
-    pub session_token: Option<String>,
-    pub expiration: Option<SystemTime>,
-    /// SDK credential `account_id`; replication targets pass their reset id.
-    pub account_id: String,
-}
-
-impl fmt::Debug for RemoteCredentials {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("RemoteCredentials")
-            .field("access_key", &self.access_key)
-            .field("secret_key", &REDACTED_CREDENTIAL)
-            .field("session_token", &self.session_token.as_ref().map(|_| REDACTED_CREDENTIAL))
-            .field("expiration", &self.expiration)
-            .field("account_id", &self.account_id)
-            .finish()
-    }
-}
-
-/// Neutral description of a remote S3 endpoint from which an
-/// `aws_sdk_s3::Client` is built.
-#[derive(Clone, Debug)]
-pub struct RemoteS3EndpointSpec {
-    /// `host[:port]` without a scheme; `secure` selects `https` or `http`.
-    pub endpoint: String,
-    pub secure: bool,
-    pub region: String,
-    pub path_style: PathStyle,
-    pub credentials: Option<RemoteCredentials>,
-    /// Accept any server certificate. Takes priority over `ca_cert_pem`.
-    pub skip_tls_verify: bool,
-    /// Extra PEM bundle trusted alongside the platform roots and the
-    /// `RUSTFS_TLS_PATH` bundle. `None` and whitespace-only mean "not set".
-    pub ca_cert_pem: Option<String>,
-    pub connect_timeout: Option<Duration>,
-    pub read_timeout: Option<Duration>,
-    /// Appended to the SDK `User-Agent` (space separated) so the remote side
-    /// can identify the caller; empty means no suffix.
-    pub user_agent_suffix: &'static str,
-}
-
-impl RemoteS3EndpointSpec {
-    /// Full endpoint URL (`scheme://host[:port]`) as handed to the SDK.
-    pub fn endpoint_url(&self) -> String {
-        if self.secure {
-            format!("https://{}", self.endpoint)
-        } else {
-            format!("http://{}", self.endpoint)
-        }
-    }
-
-    fn custom_ca_pem(&self) -> Option<&str> {
-        self.ca_cert_pem.as_deref().filter(|pem| !pem.trim().is_empty())
-    }
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum RemoteS3ClientError {
-    #[error("remote endpoint requires credentials")]
-    MissingCredentials,
-    #[error("{0}")]
-    Credentials(&'static str),
-    #[error("invalid target endpoint: {0}")]
-    InvalidEndpoint(String),
-    #[error("target endpoint is not allowed: {0}")]
-    EndpointNotAllowed(#[source] OutboundUrlError),
-    #[error("invalid target CA PEM: {0}")]
-    InvalidCaPem(String),
-}
-
-#[derive(Clone)]
-pub(crate) struct RemoteTargetCredentialsProvider {
-    pub(crate) credentials: SdkCredentials,
-}
-
-impl RemoteTargetCredentialsProvider {
-    pub(crate) fn resolve_at(&self, now: SystemTime) -> aws_credential_types::provider::Result {
-        if self.credentials.expiry().is_some_and(|expiration| expiration <= now) {
-            return Err(CredentialsError::provider_error(std::io::Error::other(EXPIRED_REMOTE_TARGET_CREDENTIALS)));
-        }
-        Ok(self.credentials.clone())
-    }
-}
-
-impl fmt::Debug for RemoteTargetCredentialsProvider {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("RemoteTargetCredentialsProvider")
-            .field("temporary", &self.credentials.session_token().is_some())
-            .field("expiration", &self.credentials.expiry())
-            .finish()
-    }
-}
-
-impl ProvideCredentials for RemoteTargetCredentialsProvider {
-    fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a>
-    where
-        Self: 'a,
-    {
-        future::ProvideCredentials::ready(self.resolve_at(SystemTime::now()))
-    }
-
-    fn fallback_on_interrupt(&self) -> Option<SdkCredentials> {
-        self.resolve_at(SystemTime::now()).ok()
-    }
-}
-
-pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: SystemTime) -> Result<SdkCredentials, &'static str> {
-    if credentials.expiration.is_some() && credentials.session_token.is_none() {
-        return Err("remote target credential expiration requires a session token");
-    }
-    if credentials.expiration.is_some_and(|expiration| expiration <= now) {
-        return Err(EXPIRED_REMOTE_TARGET_CREDENTIALS);
-    }
-
-    let mut builder = SdkCredentials::builder()
-        .access_key_id(credentials.access_key.clone())
-        .secret_access_key(credentials.secret_key.clone())
-        .account_id(credentials.account_id.clone())
-        .provider_name("bucket_target_sys");
-    if let Some(session_token) = &credentials.session_token {
-        builder = builder.session_token(session_token.clone());
-    }
-    if let Some(expiration) = credentials.expiration {
-        builder = builder.expiry(expiration);
-    }
-    Ok(builder.build())
-}
-
-/// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
-/// signing: SigV4 excludes `user-agent` from the canonical request, so the
-/// signature stays valid.
-#[derive(Debug)]
-struct UserAgentSuffixInterceptor {
-    suffix: &'static str,
-}
-
-impl Intercept for UserAgentSuffixInterceptor {
-    fn name(&self) -> &'static str {
-        "RustfsUserAgentSuffix"
-    }
-
-    fn modify_before_transmit(
-        &self,
-        context: &mut BeforeTransmitInterceptorContextMut<'_>,
-        _runtime_components: &RuntimeComponents,
-        _cfg: &mut ConfigBag,
-    ) -> Result<(), BoxError> {
-        let headers = context.request_mut().headers_mut();
-        let user_agent = match headers.get(http::header::USER_AGENT.as_str()) {
-            Some(existing) => format!("{existing} {}", self.suffix),
-            None => self.suffix.to_string(),
-        };
-        headers.try_insert(http::header::USER_AGENT.as_str(), user_agent)?;
-        Ok(())
-    }
-}
-
-/// Builds the SDK config for `spec` without finalizing it, so callers can add
-/// interceptors or (in tests) swap the HTTP client before `build()`.
-pub(crate) async fn build_remote_s3_config(
-    spec: &RemoteS3EndpointSpec,
-) -> Result<aws_sdk_s3::config::Builder, RemoteS3ClientError> {
-    let Some(credentials) = &spec.credentials else {
-        return Err(RemoteS3ClientError::MissingCredentials);
-    };
-    let creds = remote_sdk_credentials(credentials, SystemTime::now()).map_err(RemoteS3ClientError::Credentials)?;
-
-    let endpoint = spec.endpoint_url();
-    let parsed_endpoint = Url::parse(&endpoint).map_err(|err| RemoteS3ClientError::InvalidEndpoint(err.to_string()))?;
-    validate_remote_endpoint(&parsed_endpoint).map_err(RemoteS3ClientError::EndpointNotAllowed)?;
-
-    let mut config_builder = S3Config::builder()
-        .endpoint_url(endpoint)
-        .credentials_provider(SharedCredentialsProvider::new(RemoteTargetCredentialsProvider { credentials: creds }))
-        .region(SdkRegion::new(spec.region.clone()))
-        .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
-        .request_checksum_calculation(replication_request_checksum_calculation());
-
-    if spec.path_style.force_path_style() {
-        config_builder = config_builder.force_path_style(true);
-    }
-
-    if let Some(http_client) = build_aws_s3_http_client_for_spec(spec).await? {
-        config_builder = config_builder.http_client(http_client);
-    }
-
-    if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
-        let mut timeouts = TimeoutConfig::builder();
-        if let Some(connect_timeout) = spec.connect_timeout {
-            timeouts = timeouts.connect_timeout(connect_timeout);
-        }
-        if let Some(read_timeout) = spec.read_timeout {
-            timeouts = timeouts.read_timeout(read_timeout);
-        }
-        config_builder = config_builder.timeout_config(timeouts.build());
-    }
-
-    if !spec.user_agent_suffix.is_empty() {
-        config_builder = config_builder.interceptor(UserAgentSuffixInterceptor {
-            suffix: spec.user_agent_suffix,
-        });
-    }
-
-    Ok(config_builder)
-}
-
-/// Builds an `aws_sdk_s3::Client` for `spec`, applying the outbound endpoint
-/// gate, credential validation and the TLS transport selection.
-pub async fn build_remote_s3_client(spec: &RemoteS3EndpointSpec) -> Result<S3Client, RemoteS3ClientError> {
-    Ok(S3Client::from_conf(build_remote_s3_config(spec).await?.build()))
-}
-
-#[derive(Debug)]
-struct AcceptAnyServerCertVerifier;
-
-impl rustls::client::danger::ServerCertVerifier for AcceptAnyServerCertVerifier {
-    fn verify_server_cert(
-        &self,
-        _end_entity: &rustls_pki_types::CertificateDer<'_>,
-        _intermediates: &[rustls_pki_types::CertificateDer<'_>],
-        _server_name: &rustls_pki_types::ServerName<'_>,
-        _ocsp_response: &[u8],
-        _now: rustls_pki_types::UnixTime,
-    ) -> Result<rustls::client::danger::ServerCertVerified, rustls::Error> {
-        Ok(rustls::client::danger::ServerCertVerified::assertion())
-    }
-
-    fn verify_tls12_signature(
-        &self,
-        _message: &[u8],
-        _cert: &rustls_pki_types::CertificateDer<'_>,
-        _dss: &rustls::DigitallySignedStruct,
-    ) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
-        Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
-    }
-
-    fn verify_tls13_signature(
-        &self,
-        _message: &[u8],
-        _cert: &rustls_pki_types::CertificateDer<'_>,
-        _dss: &rustls::DigitallySignedStruct,
-    ) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
-        Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
-    }
-
-    fn supported_verify_schemes(&self) -> Vec<rustls::SignatureScheme> {
-        rustls::crypto::aws_lc_rs::default_provider()
-            .signature_verification_algorithms
-            .supported_schemes()
-    }
-}
-
-#[derive(Clone)]
-struct TargetHyperHttpConnector<C> {
-    client: HyperClient<C, SdkBody>,
-}
-
-impl<C> fmt::Debug for TargetHyperHttpConnector<C> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("TargetHyperHttpConnector")
-            .field("client", &"** hyper client **")
-            .finish()
-    }
-}
-
-impl<C> SmithyHttpConnector for TargetHyperHttpConnector<C>
-where
-    C: Clone + Send + Sync + 'static,
-    C: Service<Uri>,
-    C::Response:
-        hyper::rt::Read + hyper::rt::Write + hyper_util::client::legacy::connect::Connection + Send + Sync + Unpin + 'static,
-    C::Future: Unpin + Send + 'static,
-    C::Error: Into<BoxError>,
-{
-    fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
-        let request = match request.try_into_http1x() {
-            Ok(request) => request,
-            Err(err) => return HttpConnectorFuture::ready(Err(ConnectorError::user(err.into()))),
-        };
-
-        let mut client = self.client.clone();
-        let fut = client.call(request);
-        HttpConnectorFuture::new(async move {
-            let response = fut
-                .await
-                .map_err(|err| ConnectorError::io(err.into()))?
-                .map(SdkBody::from_body_1_x);
-            HttpResponse::try_from(response).map_err(|err| ConnectorError::other(err.into(), None))
-        })
-    }
-}
-
-pub(crate) fn ensure_rustls_crypto_provider() {
-    if rustls::crypto::CryptoProvider::get_default().is_none() {
-        let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
-    }
-}
-
-/// Env opt-in that re-enables loopback replication targets. Loopback (`127.0.0.1`,
-/// `::1`, `localhost`) is a classic SSRF vector and stays rejected by default, but
-/// single-host multi-instance dev setups and the e2e harness legitimately replicate
-/// over loopback. Never set this in production.
-const ALLOW_LOOPBACK_REPLICATION_TARGET_ENV: &str = "RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET";
-
-fn loopback_replication_targets_allowed() -> bool {
-    std::env::var(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV)
-        .map(|v| v.eq_ignore_ascii_case("true") || v == "1")
-        .unwrap_or(false)
-}
-
-const REPLICATION_STREAMING_CHECKSUMS_ENV: &str = "RUSTFS_REPLICATION_STREAMING_CHECKSUMS";
-
-/// Streaming trailer checksums make the SDK frame request bodies as
-/// `aws-chunked`; a target that does not decode that framing stores the frames
-/// verbatim, silently corrupting every replica while the transfer itself
-/// succeeds (#6853). Plain signed payloads are the compatible default; the env
-/// knob restores trailer checksums for fleets whose targets are all known to
-/// decode them.
-pub(crate) fn replication_request_checksum_calculation() -> RequestChecksumCalculation {
-    if std::env::var(REPLICATION_STREAMING_CHECKSUMS_ENV)
-        .map(|v| v.eq_ignore_ascii_case("true") || v == "1")
-        .unwrap_or(false)
-    {
-        RequestChecksumCalculation::WhenSupported
-    } else {
-        RequestChecksumCalculation::WhenRequired
-    }
-}
-
-/// Outbound gate for operator-configured remote endpoints (replication
-/// targets, on-demand migration sources). See
-/// `docs/operations/outbound-connection-policy.md`.
-pub fn validate_remote_endpoint(url: &Url) -> Result<(), OutboundUrlError> {
-    validate_remote_endpoint_inner(url, loopback_replication_targets_allowed())
-}
-
-pub(crate) fn validate_remote_endpoint_inner(url: &Url, allow_loopback: bool) -> Result<(), OutboundUrlError> {
-    match validate_outbound_url(url) {
-        Ok(()) => Ok(()),
-        // Replication targets are trusted infrastructure the operator configures, and
-        // legitimately live on private networks, so private addresses are always allowed.
-        Err(OutboundUrlError::ForbiddenHost {
-            reason: "private address",
-            ..
-        }) => Ok(()),
-        // Loopback is far higher SSRF risk, so it is allowed only under the explicit,
-        // off-by-default opt-in above (single-host multi-instance / the e2e harness).
-        Err(OutboundUrlError::ForbiddenHost {
-            reason: "loopback address" | "loopback host",
-            ..
-        }) if allow_loopback => Ok(()),
-        Err(err) => Err(err),
-    }
-}
-
-pub(crate) fn build_insecure_aws_s3_http_client() -> SharedHttpClient {
-    ensure_rustls_crypto_provider();
-
-    let tls_config = rustls::ClientConfig::builder()
-        .dangerous()
-        .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCertVerifier))
-        .with_no_client_auth();
-
-    let https = hyper_rustls::HttpsConnectorBuilder::new()
-        .with_tls_config(tls_config)
-        .https_or_http()
-        .enable_http1()
-        .enable_http2()
-        .build();
-    let mut client_builder = HyperClient::builder(TokioExecutor::new());
-    client_builder.pool_timer(TokioTimer::new());
-    let client = client_builder.build(https);
-    let connector = SharedHttpConnector::new(TargetHyperHttpConnector { client });
-
-    http_client_fn(move |_settings, _components| connector.clone())
-}
-
-fn validate_ca_pem_bundle(ca_cert_pem: &[u8]) -> Result<(), String> {
-    let certs = rustls_pki_types::CertificateDer::pem_slice_iter(ca_cert_pem)
-        .collect::<Result<Vec<_>, _>>()
-        .map_err(|err| format!("invalid PEM encoding: {err}"))?;
-
-    if certs.is_empty() {
-        return Err("no certificates found".to_string());
-    }
-
-    // Smithy's rustls adapter defers parsing custom certificates and assumes
-    // they are valid when the HTTPS connector is built. Validate every DER
-    // certificate first so malformed configuration is reported rather than
-    // reaching an `expect` in the dependency.
-    let mut validation_store = rustls::RootCertStore::empty();
-    for cert in certs {
-        validation_store
-            .add(cert)
-            .map_err(|err| format!("invalid X.509 certificate: {err}"))?;
-    }
-
-    Ok(())
-}
-
-pub(crate) fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), RemoteS3ClientError> {
-    validate_ca_pem_bundle(ca_cert_pem.as_bytes()).map_err(RemoteS3ClientError::InvalidCaPem)
-}
-
-pub(crate) fn compose_replication_trust_store(
-    certificate_bundles: impl IntoIterator<Item = Vec<u8>>,
-) -> (smithy_tls::TrustStore, usize) {
-    // `TrustStore::default()` keeps the platform-native roots enabled. Target
-    // and RUSTFS_TLS_PATH certificates extend that baseline instead of
-    // replacing it with a target-specific trust island.
-    let mut trust_store = smithy_tls::TrustStore::default();
-    let mut custom_bundle_count = 0;
-    for pem in certificate_bundles {
-        trust_store.add_pem_certificate(pem);
-        custom_bundle_count += 1;
-    }
-
-    (trust_store, custom_bundle_count)
-}
-
-pub(crate) fn build_aws_s3_http_client_with_trust_store(
-    trust_store: smithy_tls::TrustStore,
-) -> Result<SharedHttpClient, RemoteS3ClientError> {
-    let tls_context = smithy_tls::TlsContext::builder()
-        .with_trust_store(trust_store)
-        .build()
-        .map_err(|err| RemoteS3ClientError::InvalidCaPem(err.to_string()))?;
-
-    Ok(SmithyHttpClientBuilder::new()
-        .tls_provider(smithy_tls::Provider::rustls(smithy_tls::rustls_provider::CryptoMode::AwsLc))
-        .tls_context(tls_context)
-        .build_https())
-}
-
-pub(crate) async fn load_tls_path_ca_bundles(tls_dir: &Path, trust_leaf_cert_as_ca: bool) -> Vec<Vec<u8>> {
-    let mut certificate_bundles = Vec::new();
-
-    let ca_path = tls_dir.join(RUSTFS_CA_CERT);
-    match tokio::fs::read(&ca_path).await {
-        Ok(pem) => match validate_ca_pem_bundle(&pem) {
-            Ok(()) => certificate_bundles.push(pem),
-            Err(err) => warn!("ignoring invalid custom CA bundle {:?} for replication client: {}", ca_path, err),
-        },
-        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
-        Err(e) => warn!("failed to read custom CA bundle {:?} for replication client: {}", ca_path, e),
-    }
-
-    if trust_leaf_cert_as_ca {
-        let leaf_cert_path = tls_dir.join(RUSTFS_TLS_CERT);
-        match tokio::fs::read(&leaf_cert_path).await {
-            Ok(pem) => match validate_ca_pem_bundle(&pem) {
-                Ok(()) => certificate_bundles.push(pem),
-                Err(err) => warn!(
-                    "ignoring invalid leaf certificate {:?} for replication client trust store: {}",
-                    leaf_cert_path, err
-                ),
-            },
-            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
-            Err(e) => warn!("failed to read leaf cert {:?} for replication client trust store: {}", leaf_cert_path, e),
-        }
-    }
-
-    certificate_bundles
-}
-
-async fn load_configured_tls_ca_bundles() -> Vec<Vec<u8>> {
-    let tls_path = rustfs_utils::get_env_str(rustfs_config::ENV_RUSTFS_TLS_PATH, rustfs_config::DEFAULT_RUSTFS_TLS_PATH);
-    if tls_path.is_empty() {
-        return Vec::new();
-    }
-
-    load_tls_path_ca_bundles(
-        Path::new(&tls_path),
-        rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA),
-    )
-    .await
-}
-
-pub(crate) async fn build_aws_s3_http_client_from_target_ca_pem(
-    ca_cert_pem: &str,
-) -> Result<SharedHttpClient, RemoteS3ClientError> {
-    validate_target_ca_pem(ca_cert_pem)?;
-
-    let mut certificate_bundles = load_configured_tls_ca_bundles().await;
-    certificate_bundles.push(ca_cert_pem.as_bytes().to_vec());
-    let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
-
-    build_aws_s3_http_client_with_trust_store(trust_store)
-}
-
-/// Selects the HTTP client for `spec`: `None` keeps the SDK default (plain
-/// HTTP, or HTTPS with platform roots when no custom trust is configured).
-pub(crate) async fn build_aws_s3_http_client_for_spec(
-    spec: &RemoteS3EndpointSpec,
-) -> Result<Option<SharedHttpClient>, RemoteS3ClientError> {
-    if !spec.secure {
-        return Ok(None);
-    }
-
-    if spec.skip_tls_verify {
-        return Ok(Some(build_insecure_aws_s3_http_client()));
-    }
-
-    if let Some(ca_cert_pem) = spec.custom_ca_pem() {
-        return build_aws_s3_http_client_from_target_ca_pem(ca_cert_pem).await.map(Some);
-    }
-
-    Ok(build_aws_s3_http_client_from_tls_path().await)
-}
-
-async fn build_aws_s3_http_client_from_tls_path() -> Option<SharedHttpClient> {
-    let certificate_bundles = load_configured_tls_ca_bundles().await;
-    if certificate_bundles.is_empty() {
-        return None;
-    }
-
-    let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
-    match build_aws_s3_http_client_with_trust_store(trust_store) {
-        Ok(client) => Some(client),
-        Err(e) => {
-            warn!("failed to build AWS SDK TLS context for replication client: {}", e);
-            None
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use aws_smithy_runtime_api::http::StatusCode as SmithyStatusCode;
-    use std::sync::Mutex;
-
-    fn spec(endpoint: &str, secure: bool) -> RemoteS3EndpointSpec {
-        RemoteS3EndpointSpec {
-            endpoint: endpoint.to_string(),
-            secure,
-            region: "us-east-1".to_string(),
-            path_style: PathStyle::Auto,
-            credentials: Some(RemoteCredentials {
-                access_key: "access".to_string(),
-                secret_key: "secret".to_string(),
-                session_token: None,
-                expiration: None,
-                account_id: String::new(),
-            }),
-            skip_tls_verify: false,
-            ca_cert_pem: None,
-            connect_timeout: None,
-            read_timeout: None,
-            user_agent_suffix: "",
-        }
-    }
-
-    type RecordedHeaders = Arc<Mutex<Vec<Vec<(String, String)>>>>;
-
-    #[derive(Clone, Debug)]
-    struct RecordingHeaderConnector {
-        request_headers: RecordedHeaders,
-    }
-
-    impl SmithyHttpConnector for RecordingHeaderConnector {
-        fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
-            self.request_headers
-                .lock()
-                .expect("recorded header lock should not be poisoned")
-                .push(
-                    request
-                        .headers()
-                        .iter()
-                        .map(|(k, v)| (k.to_string(), v.to_string()))
-                        .collect(),
-                );
-            HttpConnectorFuture::ready(Ok(HttpResponse::new(
-                SmithyStatusCode::try_from(200_u16).expect("200 should be a valid response status"),
-                SdkBody::empty(),
-            )))
-        }
-    }
-
-    #[tokio::test]
-    async fn build_remote_s3_client_rejects_loopback_and_metadata_endpoints() {
-        // Default (no loopback opt-in): loopback in IPv4, IPv6 and hostname
-        // forms plus the metadata endpoint all return the typed gate error.
-        for endpoint in ["127.0.0.1:9000", "[::1]:9000", "localhost:9000", "169.254.169.254"] {
-            let err = build_remote_s3_client(&spec(endpoint, false))
-                .await
-                .err()
-                .unwrap_or_else(|| panic!("{endpoint} must be rejected by the outbound gate"));
-            assert!(
-                matches!(err, RemoteS3ClientError::EndpointNotAllowed(OutboundUrlError::ForbiddenHost { .. })),
-                "{endpoint}: unexpected error {err:?}"
-            );
-            assert!(err.to_string().contains("not allowed"), "{endpoint}: {err}");
-        }
-    }
-
-    #[tokio::test]
-    async fn build_remote_s3_client_allows_private_and_public_endpoints() {
-        for endpoint in ["10.0.0.1:9000", "192.168.1.20", "s3.example.com"] {
-            build_remote_s3_client(&spec(endpoint, false))
-                .await
-                .unwrap_or_else(|err| panic!("{endpoint} should be allowed: {err}"));
-        }
-    }
-
-    #[tokio::test]
-    async fn build_remote_s3_client_requires_credentials() {
-        let mut spec = spec("s3.example.com", true);
-        spec.credentials = None;
-        let err = build_remote_s3_client(&spec)
-            .await
-            .expect_err("missing credentials must be a typed error");
-        assert!(matches!(err, RemoteS3ClientError::MissingCredentials));
-    }
-
-    #[tokio::test]
-    async fn build_remote_s3_client_rejects_expiration_without_session_token() {
-        let mut spec = spec("s3.example.com", true);
-        spec.credentials
-            .as_mut()
-            .expect("spec fixture carries credentials")
-            .expiration = Some(SystemTime::now() + Duration::from_secs(3_600));
-        let err = build_remote_s3_client(&spec)
-            .await
-            .expect_err("expiration without session token must be rejected");
-        assert_eq!(err.to_string(), "remote target credential expiration requires a session token");
-    }
-
-    #[tokio::test]
-    async fn build_remote_s3_client_rejects_invalid_custom_ca_pem() {
-        let mut spec = spec("192.168.1.10:9000", true);
-        spec.ca_cert_pem = Some("not a pem".to_string());
-        let err = build_remote_s3_client(&spec)
-            .await
-            .expect_err("invalid custom CA PEM must be rejected");
-        assert!(matches!(err, RemoteS3ClientError::InvalidCaPem(_)));
-        assert!(err.to_string().contains("invalid target CA PEM"));
-    }
-
-    #[test]
-    fn path_style_auto_and_path_force_path_style() {
-        assert!(PathStyle::Auto.force_path_style());
-        assert!(PathStyle::Path.force_path_style());
-        assert!(!PathStyle::VirtualHost.force_path_style());
-    }
-
-    #[test]
-    fn remote_credentials_debug_redacts_secrets() {
-        let credentials = RemoteCredentials {
-            access_key: "access".to_string(),
-            secret_key: "very-secret".to_string(),
-            session_token: Some("session-token".to_string()),
-            expiration: None,
-            account_id: String::new(),
-        };
-        let rendered = format!("{credentials:?}");
-        assert!(rendered.contains("access"));
-        assert!(!rendered.contains("very-secret"));
-        assert!(!rendered.contains("session-token"));
-    }
-
-    #[tokio::test]
-    async fn user_agent_suffix_is_appended_after_signing() {
-        let request_headers: RecordedHeaders = Arc::new(Mutex::new(Vec::new()));
-        let connector = SharedHttpConnector::new(RecordingHeaderConnector {
-            request_headers: Arc::clone(&request_headers),
-        });
-        let http_client = http_client_fn(move |_settings, _components| connector.clone());
-
-        let mut spec = spec("s3.example.com", true);
-        spec.user_agent_suffix = "RustFS-Test/0.0";
-        spec.connect_timeout = Some(Duration::from_secs(5));
-        spec.read_timeout = Some(Duration::from_secs(5));
-        let config = build_remote_s3_config(&spec)
-            .await
-            .expect("spec should build")
-            .http_client(http_client)
-            .build();
-        S3Client::from_conf(config)
-            .head_bucket()
-            .bucket("bucket")
-            .send()
-            .await
-            .expect("recording connector should accept the request");
-
-        let recorded = request_headers.lock().expect("recorded header lock should not be poisoned");
-        let headers = &recorded[0];
-        let user_agent = headers
-            .iter()
-            .find(|(k, _)| k.eq_ignore_ascii_case("user-agent"))
-            .map(|(_, v)| v.as_str())
-            .expect("SDK request must carry a user-agent");
-        assert!(user_agent.ends_with(" RustFS-Test/0.0"), "user-agent was {user_agent}");
-        assert!(user_agent.starts_with("aws-sdk-rust/"), "SDK identity must be preserved: {user_agent}");
-        assert!(
-            headers.iter().any(|(k, _)| k.eq_ignore_ascii_case("authorization")),
-            "request must still be signed"
-        );
-    }
-}

crates/ecstore/src/bucket/replication/replication_object_config.rs

@@ -436,21 +436,16 @@ pub(crate) async fn check_replicate_delete_strict(
     }
 
     for target in decision.targets_map.values_mut() {
-        let replicate_sync = ReplicationTargetStore::remote_target_client(bucket, &target.arn)
-            .await
-            .map(|client| client.replicate_sync);
-        apply_target_delivery_mode(target, replicate_sync);
+        if let Some(client) = ReplicationTargetStore::remote_target_client(bucket, &target.arn).await {
+            target.synchronous = client.replicate_sync;
+        } else {
+            target.replicate = false;
+            target.synchronous = false;
+        }
     }
     Ok(decision)
 }
 
-fn apply_target_delivery_mode(target: &mut ReplicateTargetDecision, replicate_sync: Option<bool>) {
-    // A missing runtime client is a delivery failure, not a rule mismatch.
-    // Preserve admission and fall back to the asynchronous worker, which can
-    // persist FAILED state for the heal/retry path.
-    target.synchronous = replicate_sync.unwrap_or(false);
-}
-
 pub(crate) fn check_replicate_delete_with_snapshot(
     dobj: &ObjectToDelete,
     oi: &ObjectInfo,
@@ -634,23 +629,6 @@ mod tests {
         }));
     }
 
-    #[test]
-    fn missing_target_client_preserves_delete_admission_as_async() {
-        let mut target = ReplicateTargetDecision::new("arn:target".to_string(), true, true);
-
-        apply_target_delivery_mode(&mut target, None);
-
-        assert!(target.replicate, "a runtime client miss must not erase the replication rule decision");
-        assert!(
-            !target.synchronous,
-            "unavailable synchronous targets must fall back to the async retry path"
-        );
-
-        apply_target_delivery_mode(&mut target, Some(true));
-        assert!(target.replicate);
-        assert!(target.synchronous);
-    }
-
     #[test]
     fn must_replicate_options_preserve_request_flag() {
         let user_defined = HashMap::new();

crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs

@@ -20,9 +20,8 @@ pub use rustfs_replication::{
 pub(crate) use rustfs_replication::{
     ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, delete_marker_purge_mrf_entry,
     delete_marker_purge_version_id, delete_replication_creates_marker, delete_replication_missing_source_decision,
-    delete_replication_object_opts, heal_uses_delete_replication_path, is_object_lock_denied_delete,
-    is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
-    replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error,
-    resync_existing_delete_replication_info, resync_target_for_object, should_retry_delete_marker_purge,
-    single_part_replica_etag_mismatch, target_delete_version_id,
+    delete_replication_object_opts, heal_uses_delete_replication_path, is_retryable_delete_replication_head_error,
+    is_version_delete_replication, replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
+    replication_multipart_part_plan, resync_existing_delete_replication_info, resync_target_for_object,
+    should_retry_delete_marker_purge, target_delete_version_id,
 };

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -3177,19 +3177,6 @@ pub(crate) async fn queue_replication_heal_internal(
             }
         }
         ReplicationHealQueueAction::QueueDelete(dv) => {
-            // A purge the peer denied under object lock cannot succeed until
-            // the lock lapses (#6850); requeuing it every heal cycle only
-            // burns bandwidth and failure counters. The backoff expires on
-            // its own, so the purge is probed again — and converges — once
-            // the retention window has a chance of being over.
-            if super::replication_object_decision_boundary::is_version_delete_replication(&dv.delete_object)
-                && super::replication_resyncer::object_lock_denied_purge_backoff_active(&dv)
-            {
-                return ReplicationHealQueueResult {
-                    object_info: roi,
-                    admission: ReplicationQueueAdmission::Skipped,
-                };
-            }
             let admission = if let Some(pool) = runtime_sources::replication_pool() {
                 pool.queue_replica_delete_task(dv).await
             } else {

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -36,8 +36,8 @@ use time::OffsetDateTime;
 use time::format_description::well_known::Rfc3339;
 
 pub(crate) use crate::bucket::bucket_target_sys::{
-    AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemotePutObjectResponse, RemoveObjectOptions,
-    S3ClientError, TargetClient, resolve_read_api_version_id,
+    AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, S3ClientError,
+    TargetClient, resolve_read_api_version_id,
 };
 #[cfg(test)]
 pub(crate) use crate::bucket::target::BucketTarget;
@@ -260,15 +260,8 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
             // the response path relies on). Keep the object's own multipart
             // flag so encrypted objects stay on the multipart route.
         } else {
-            let (checksum_meta, checksum_record_is_multipart) = object_info.decrypt_checksums(0, &HeaderMap::new())?;
-            // The checksum record describes how the *checksum* is composed,
-            // not how the object is stored. A full-object checksum carries no
-            // MULTIPART flag even on a multipart upload, so trusting it here
-            // routed a 768-part object through a single PutObject and the
-            // target rejected the 6 GiB body with EntityTooLarge
-            // (rustfs#6825). The object's own shape is the authority: the
-            // record may only add multipart-ness, never take it away.
-            is_multipart = object_info.is_multipart() || checksum_record_is_multipart;
+            let (checksum_meta, is_mp) = object_info.decrypt_checksums(0, &HeaderMap::new())?;
+            is_multipart = is_mp;
 
             for (key, value) in checksum_meta.iter() {
                 if key != AMZ_CHECKSUM_TYPE {
@@ -525,109 +518,6 @@ mod tests {
     use time::Duration;
     use uuid::Uuid;
 
-    /// Serialize an object-level checksum record the way
-    /// `complete_multipart_upload` persists it for a **full-object** checksum:
-    /// the record carries the plain algorithm type, without the MULTIPART
-    /// flags that a composite record gets.
-    fn full_object_multipart_checksum_record() -> bytes::Bytes {
-        let checksum_type = rustfs_rio::ChecksumType::from_string_with_obj_type("crc32", "FULL_OBJECT");
-        assert!(checksum_type.is_set(), "crc32 FULL_OBJECT must be a valid checksum type");
-        assert!(checksum_type.full_object_requested());
-
-        let mut combined = Vec::new();
-        let mut checksum = rustfs_rio::Checksum {
-            checksum_type,
-            ..Default::default()
-        };
-        for part in [b"part-one".as_slice(), b"part-two".as_slice()] {
-            let part_checksum = rustfs_rio::Checksum::new_from_data(checksum_type, part).expect("part checksum");
-            combined.extend_from_slice(part_checksum.raw.as_slice());
-            checksum.add_part(&part_checksum, part.len() as i64).expect("add part");
-        }
-
-        checksum.to_bytes(&combined)
-    }
-
-    #[test]
-    fn multipart_object_with_full_object_checksum_keeps_the_multipart_route() {
-        // rustfs#6825: a 768-part upload was replicated with a single
-        // PutObject and rejected by the target with EntityTooLarge. The
-        // object's storage shape says multipart; only the checksum record
-        // looked single-part, and the checksum record must not decide the
-        // transport.
-        let object_info = ObjectInfo {
-            etag: Some("0123456789abcdef0123456789abcdef-768".to_string()),
-            checksum: Some(full_object_multipart_checksum_record()),
-            size: 6 * 1024 * 1024 * 1024,
-            ..Default::default()
-        };
-
-        assert!(object_info.is_multipart(), "the fixture must be a multipart object");
-        let (_, checksum_says_multipart) = object_info
-            .decrypt_checksums(0, &HeaderMap::new())
-            .expect("checksum record must decode");
-        assert!(
-            !checksum_says_multipart,
-            "fixture precondition: a full-object record carries no MULTIPART flag, which is what used to \
-             downgrade the transport"
-        );
-
-        let (_, is_multipart) = replication_put_object_options("STANDARD", &object_info).expect("build put options");
-
-        assert!(
-            is_multipart,
-            "a multipart object must replicate over multipart whatever its checksum record looks like"
-        );
-    }
-
-    #[test]
-    fn checksum_record_never_changes_the_transport_a_single_part_object_needs() {
-        // The mirror of the rustfs#6825 guard: an object stored as one PUT
-        // must keep the single-PUT transport, or its replica's ETag would
-        // change shape and every ETag-based convergence check would re-copy it.
-        let checksum =
-            rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::CRC32, b"whole-object").expect("checksum fixture");
-        let object_info = ObjectInfo {
-            etag: Some("0123456789abcdef0123456789abcdef".to_string()),
-            checksum: Some(checksum.to_bytes(&[])),
-            ..Default::default()
-        };
-
-        assert!(!object_info.is_multipart(), "the fixture must be a single-part object");
-
-        let (_, is_multipart) = replication_put_object_options("STANDARD", &object_info).expect("build put options");
-
-        assert!(!is_multipart, "a single-part object must not be promoted onto the multipart transport");
-    }
-
-    #[test]
-    fn composite_checksum_multipart_object_keeps_the_multipart_route() {
-        // The checksum shape that already worked before rustfs#6825, pinned so
-        // the fix cannot regress it.
-        let mut checksum_type = rustfs_rio::ChecksumType::from_string("crc32");
-        checksum_type
-            .merge(rustfs_rio::ChecksumType::MULTIPART)
-            .merge(rustfs_rio::ChecksumType::INCLUDES_MULTIPART);
-
-        let mut combined = Vec::new();
-        for part in [b"part-one".as_slice(), b"part-two".as_slice()] {
-            let part_checksum =
-                rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::from_string("crc32"), part).expect("part checksum");
-            combined.extend_from_slice(part_checksum.raw.as_slice());
-        }
-        let checksum = rustfs_rio::Checksum::new_from_data(checksum_type, &combined).expect("composite checksum");
-
-        let object_info = ObjectInfo {
-            etag: Some("0123456789abcdef0123456789abcdef-2".to_string()),
-            checksum: Some(checksum.to_bytes(&combined)),
-            ..Default::default()
-        };
-
-        let (_, is_multipart) = replication_put_object_options("STANDARD", &object_info).expect("build put options");
-
-        assert!(is_multipart, "a composite-checksum multipart object must stay on the multipart transport");
-    }
-
     #[test]
     fn replication_action_for_target_head_existing_object_source_newer_null_version_requires_replication() {
         let source = ObjectInfo {

crates/ecstore/src/bucket/target/bucket_target.rs

@@ -25,8 +25,6 @@ use time::OffsetDateTime;
 use url::Url;
 
 const REDACTED_CREDENTIAL: &str = "<redacted>";
-const GO_YEAR_ONE_START_UNIX_SECONDS: i64 = -62_135_596_800;
-const GO_YEAR_TWO_START_UNIX_SECONDS: i64 = -62_104_060_800;
 
 #[derive(Deserialize, Serialize, Default, Clone)]
 pub struct Credentials {
@@ -43,26 +41,6 @@ pub struct Credentials {
 }
 
 impl Credentials {
-    /// Returns the session token used for request signing.
-    ///
-    /// MinIO-compatible payloads may carry an empty token. Treat whitespace-only
-    /// values as absent without rewriting a real token, whose bytes are opaque.
-    pub fn effective_session_token(&self) -> Option<&str> {
-        self.session_token.as_deref().filter(|token| !token.trim().is_empty())
-    }
-
-    /// Returns the credential expiry after normalizing Go's zero `time.Time`.
-    ///
-    /// Go JSON encoders emit year 1 for an unset `time.Time`; persisted MinIO
-    /// target metadata can therefore contain that sentinel even for static
-    /// credentials.
-    pub fn effective_expiration(&self) -> Option<Timestamp> {
-        self.expiration.filter(|expiration| {
-            let unix_seconds = expiration.as_second();
-            !(GO_YEAR_ONE_START_UNIX_SECONDS..GO_YEAR_TWO_START_UNIX_SECONDS).contains(&unix_seconds)
-        })
-    }
-
     pub fn redacted(&self) -> Self {
         Self {
             access_key: self.access_key.clone(),
@@ -377,24 +355,6 @@ mod tests {
     use std::time::Duration;
     use time::OffsetDateTime;
 
-    #[test]
-    fn credential_effective_values_normalize_only_compatibility_sentinels() {
-        let mut credentials = Credentials {
-            access_key: "access".to_string(),
-            secret_key: "secret".to_string(),
-            session_token: Some("  ".to_string()),
-            expiration: Some("0001-01-01T08:00:00+08:00".parse().expect("Go zero time should parse")),
-        };
-
-        assert!(credentials.effective_session_token().is_none());
-        assert!(credentials.effective_expiration().is_none());
-
-        credentials.session_token = Some(" opaque token ".to_string());
-        credentials.expiration = Some("2099-01-01T00:00:00Z".parse().expect("future timestamp should parse"));
-        assert_eq!(credentials.effective_session_token(), Some(" opaque token "));
-        assert_eq!(credentials.effective_expiration(), credentials.expiration);
-    }
-
     #[test]
     fn test_bucket_target_json_deserialize() {
         let json = r#"

crates/ecstore/src/bucket/utils.rs

@@ -73,7 +73,6 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
     check_bucket_name_common(bucket_name, true)
 }
 
-// RUSTFS_COMPAT_TODO(s3gate-metadata-xml): the s3s codec reads persisted XML during migration. Remove after every supported writer uses the gateway codec and every retained metadata object and backup archive is verified or rewritten.
 pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
 where
     T: for<'xml> xml::Deserialize<'xml>,

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -1307,32 +1307,6 @@ pub fn verify_tonic_mutation_body_digest<T>(request: &tonic::Request<T>, canonic
     verify_tonic_mutation_body_digest_with_strictness(request, canonical_body, internode_rpc_body_digest_strict())
 }
 
-/// Verify a non-disk mutation without accepting a newly-generated unsigned v2 body.
-///
-/// The disk mutation lane has a rolling-upgrade exception for `UNSIGNED-PAYLOAD`
-/// while peer replay-cache capability is being discovered. Historical v2 peers
-/// used the fixed `unsigned` nonce before body-digest rollout; preserve that
-/// exact marker for mixed-version compatibility, but reject unsigned v2
-/// requests that omit it or present a different nonce.
-pub fn verify_tonic_mutation_body_digest_reject_unsigned<T>(
-    request: &tonic::Request<T>,
-    canonical_body: &[u8],
-) -> std::io::Result<()> {
-    let version = request
-        .metadata()
-        .get(RPC_AUTH_VERSION_HEADER)
-        .and_then(|value| value.to_str().ok());
-    let digest = request
-        .metadata()
-        .get(RPC_CONTENT_SHA256_HEADER)
-        .and_then(|value| value.to_str().ok());
-    let nonce = request.metadata().get(RPC_NONCE_HEADER).and_then(|value| value.to_str().ok());
-    if version == Some(RPC_AUTH_VERSION_V2) && digest == Some(UNSIGNED_PAYLOAD) && nonce != Some("unsigned") {
-        return Err(std::io::Error::other("RPC mutation requires a body-bound v2 signature"));
-    }
-    verify_tonic_mutation_body_digest(request, canonical_body)
-}
-
 /// [`verify_tonic_mutation_body_digest`] with the strict gate injected as a parameter, so both
 /// rollout postures are unit-testable without racing on process-global environment variables.
 fn verify_tonic_mutation_body_digest_with_strictness<T>(

crates/ecstore/src/cluster/rpc/internode_data_transport.rs

@@ -36,7 +36,6 @@ use rustfs_rio::{ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter};
 use sha2::{Digest, Sha256};
 use std::collections::HashMap;
 use std::future::Future;
-use std::io;
 use std::pin::Pin;
 use std::sync::{Arc, LazyLock, OnceLock};
 use std::task::{Context, Poll};
@@ -106,13 +105,9 @@ struct PutFileCapabilityCacheState {
     cached: Option<PutFileCapabilityState>,
     generation: u64,
     in_flight: Option<PutFileCapabilityFlight>,
-    rejected_server_epoch: Option<Uuid>,
 }
 
-// The registry lock is released before taking an entry lock. Entry guards cover
-// only cache transitions, never a probe or await; poll-based writers must be
-// able to reject an epoch atomically with those transitions.
-type PutFileCapabilityCacheEntry = Arc<parking_lot::RwLock<PutFileCapabilityCacheState>>;
+type PutFileCapabilityCacheEntry = Arc<tokio::sync::RwLock<PutFileCapabilityCacheState>>;
 
 static PUT_FILE_CAPABILITY_CACHE: LazyLock<parking_lot::RwLock<HashMap<String, PutFileCapabilityCacheEntry>>> =
     LazyLock::new(|| parking_lot::RwLock::new(HashMap::new()));
@@ -124,7 +119,7 @@ fn put_file_capability_cache_entry(endpoint: &str) -> PutFileCapabilityCacheEntr
     PUT_FILE_CAPABILITY_CACHE
         .write()
         .entry(endpoint.to_owned())
-        .or_insert_with(|| Arc::new(parking_lot::RwLock::new(PutFileCapabilityCacheState::default())))
+        .or_insert_with(|| Arc::new(tokio::sync::RwLock::new(PutFileCapabilityCacheState::default())))
         .clone()
 }
 
@@ -139,23 +134,6 @@ fn fresh_put_file_capability(state: Option<PutFileCapabilityState>, now: Instant
     }
 }
 
-fn reject_put_file_server_epoch(endpoint: &str, server_epoch: Uuid) {
-    let entry = PUT_FILE_CAPABILITY_CACHE.read().get(endpoint).cloned();
-    if let Some(entry) = entry {
-        let mut state = entry.write();
-        if matches!(state.cached, Some(PutFileCapabilityState::V1 { server_epoch: cached, .. }) if cached == server_epoch) {
-            state.rejected_server_epoch = Some(server_epoch);
-        }
-    }
-}
-
-fn usable_put_file_capability(state: &PutFileCapabilityCacheState, now: Instant) -> Option<Option<Uuid>> {
-    match fresh_put_file_capability(state.cached, now)? {
-        Some(server_epoch) if state.rejected_server_epoch == Some(server_epoch) => None,
-        capability => Some(capability),
-    }
-}
-
 fn put_file_capability_status_is_legacy(status: u16) -> bool {
     status == 404
 }
@@ -344,14 +322,13 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
 
     async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
         let server_epoch = self.put_file_auth_capability(&request.endpoint).await?;
-        let auth_scope = server_epoch.map(|server_epoch| (Uuid::new_v4(), server_epoch));
-        let url = build_put_file_stream_url(&request, auth_scope);
-        let endpoint = request.endpoint;
+        let nonce = server_epoch.map(|_| Uuid::new_v4());
+        let url = build_put_file_stream_url(&request, nonce.zip(server_epoch));
         let mut headers = json_headers();
         build_auth_headers(&url, &Method::PUT, &mut headers)?;
         let writer = HttpWriter::new(url.clone(), Method::PUT, headers).await?;
-        match auth_scope {
-            Some((nonce, server_epoch)) => Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce, endpoint, server_epoch))),
+        match nonce {
+            Some(nonce) => Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce))),
             None => Ok(Box::new(writer)),
         }
     }
@@ -521,15 +498,15 @@ where
 {
     let entry = put_file_capability_cache_entry(endpoint);
     {
-        let state = entry.read();
-        if let Some(cached) = usable_put_file_capability(&state, Instant::now()) {
+        let state = entry.read().await;
+        if let Some(cached) = fresh_put_file_capability(state.cached, Instant::now()) {
             return Ok(cached);
         }
     }
 
     let flight = {
-        let mut state = entry.write();
-        if let Some(cached) = usable_put_file_capability(&state, Instant::now()) {
+        let mut state = entry.write().await;
+        if let Some(cached) = fresh_put_file_capability(state.cached, Instant::now()) {
             return Ok(cached);
         }
         if let Some(flight) = state.in_flight.clone() {
@@ -555,7 +532,7 @@ where
         .await;
 
     {
-        let mut state = entry.write();
+        let mut state = entry.write().await;
         let is_current_flight = state
             .in_flight
             .as_ref()
@@ -563,9 +540,6 @@ where
         if is_current_flight {
             match outcome {
                 Ok(Some(server_epoch)) => {
-                    if state.rejected_server_epoch != Some(*server_epoch) {
-                        state.rejected_server_epoch = None;
-                    }
                     state.cached = Some(PutFileCapabilityState::V1 {
                         server_epoch: *server_epoch,
                         revalidate_after: Instant::now() + PUT_FILE_V1_CAPABILITY_TTL,
@@ -656,23 +630,17 @@ struct PutFileAuthWriter<W> {
     inner: W,
     url: String,
     nonce: Uuid,
-    endpoint: String,
-    server_epoch: Uuid,
-    server_epoch_rejected: bool,
     hasher: Sha256,
     trailer: Option<Vec<u8>>,
     trailer_offset: usize,
 }
 
 impl<W> PutFileAuthWriter<W> {
-    fn new(inner: W, url: String, nonce: Uuid, endpoint: String, server_epoch: Uuid) -> Self {
+    fn new(inner: W, url: String, nonce: Uuid) -> Self {
         Self {
             inner,
             url,
             nonce,
-            endpoint,
-            server_epoch,
-            server_epoch_rejected: false,
             hasher: Sha256::new(),
             trailer: None,
             trailer_offset: 0,
@@ -688,14 +656,6 @@ impl<W> PutFileAuthWriter<W> {
         Ok(())
     }
 
-    fn reject_server_epoch_on_conflict(&mut self, error: &io::Error) {
-        if self.server_epoch_rejected || !io_error_has_put_file_epoch_conflict(error) {
-            return;
-        }
-        reject_put_file_server_epoch(&self.endpoint, self.server_epoch);
-        self.server_epoch_rejected = true;
-    }
-
     fn poll_write_trailer(&mut self, cx: &mut Context<'_>) -> Poll<std::io::Result<()>>
     where
         W: AsyncWrite + Unpin,
@@ -713,10 +673,7 @@ impl<W> PutFileAuthWriter<W> {
                     )));
                 }
                 Poll::Ready(Ok(written)) => written,
-                Poll::Ready(Err(err)) => {
-                    self.reject_server_epoch_on_conflict(&err);
-                    return Poll::Ready(Err(err));
-                }
+                Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
                 Poll::Pending => return Poll::Pending,
             };
             self.trailer_offset += written;
@@ -725,15 +682,6 @@ impl<W> PutFileAuthWriter<W> {
     }
 }
 
-fn io_error_has_put_file_epoch_conflict(error: &io::Error) -> bool {
-    error
-        .get_ref()
-        .and_then(|source| source.downcast_ref::<rustfs_rio::InternodeHttpError>())
-        .is_some_and(
-            |error| matches!(error.kind(), rustfs_rio::InternodeHttpErrorKind::HttpStatus(status) if status.as_u16() == 409),
-        )
-}
-
 impl<W> AsyncWrite for PutFileAuthWriter<W>
 where
     W: AsyncWrite + Unpin,
@@ -750,22 +698,12 @@ where
                 self.hasher.update(&buf[..written]);
                 Poll::Ready(Ok(written))
             }
-            Poll::Ready(Err(err)) => {
-                self.reject_server_epoch_on_conflict(&err);
-                Poll::Ready(Err(err))
-            }
-            Poll::Pending => Poll::Pending,
+            other => other,
         }
     }
 
     fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
-        match Pin::new(&mut self.inner).poll_flush(cx) {
-            Poll::Ready(Err(err)) => {
-                self.reject_server_epoch_on_conflict(&err);
-                Poll::Ready(Err(err))
-            }
-            other => other,
-        }
+        Pin::new(&mut self.inner).poll_flush(cx)
     }
 
     fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
@@ -774,13 +712,7 @@ where
             Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
             Poll::Pending => return Poll::Pending,
         }
-        match Pin::new(&mut self.inner).poll_shutdown(cx) {
-            Poll::Ready(Err(err)) => {
-                self.reject_server_epoch_on_conflict(&err);
-                Poll::Ready(Err(err))
-            }
-            other => other,
-        }
+        Pin::new(&mut self.inner).poll_shutdown(cx)
     }
 }
 
@@ -908,6 +840,7 @@ mod tests {
             loop {
                 let strong_count = entry
                     .read()
+                    .await
                     .in_flight
                     .as_ref()
                     .map(|flight| Arc::strong_count(&flight.outcome))
@@ -925,50 +858,6 @@ mod tests {
     #[derive(Debug)]
     struct LegacyTestTransport;
 
-    #[derive(Clone, Copy, Debug)]
-    enum PutFileFailurePhase {
-        Write,
-        Flush,
-        Shutdown,
-    }
-
-    struct PutFileFailureWriter {
-        phase: PutFileFailurePhase,
-        status: reqwest::StatusCode,
-    }
-
-    impl PutFileFailureWriter {
-        fn error(&self) -> io::Error {
-            rustfs_rio::new_test_internode_http_io_error(rustfs_rio::InternodeHttpErrorKind::HttpStatus(self.status))
-        }
-    }
-
-    impl tokio::io::AsyncWrite for PutFileFailureWriter {
-        fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
-            Poll::Ready(if matches!(self.phase, PutFileFailurePhase::Write) {
-                Err(self.error())
-            } else {
-                Ok(buf.len())
-            })
-        }
-
-        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
-            Poll::Ready(if matches!(self.phase, PutFileFailurePhase::Flush) {
-                Err(self.error())
-            } else {
-                Ok(())
-            })
-        }
-
-        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
-            Poll::Ready(if matches!(self.phase, PutFileFailurePhase::Shutdown) {
-                Err(self.error())
-            } else {
-                Ok(())
-            })
-        }
-    }
-
     #[async_trait::async_trait]
     impl InternodeDataTransport for LegacyTestTransport {
         async fn open_read(&self, _request: ReadStreamRequest) -> Result<FileReader> {
@@ -1159,7 +1048,7 @@ mod tests {
         let v1_endpoint = format!("http://v1-{}.invalid", Uuid::new_v4());
         let v1_entry = put_file_capability_cache_entry(&v1_endpoint);
         let server_epoch = Uuid::new_v4();
-        v1_entry.write().cached = Some(PutFileCapabilityState::V1 {
+        v1_entry.write().await.cached = Some(PutFileCapabilityState::V1 {
             server_epoch,
             revalidate_after: Instant::now() + PUT_FILE_V1_CAPABILITY_TTL,
         });
@@ -1178,7 +1067,7 @@ mod tests {
             Some(server_epoch)
         );
         assert!(!cache_probe_called.load(Ordering::SeqCst));
-        v1_entry.write().cached = Some(PutFileCapabilityState::V1 {
+        v1_entry.write().await.cached = Some(PutFileCapabilityState::V1 {
             server_epoch,
             revalidate_after: Instant::now(),
         });
@@ -1197,7 +1086,8 @@ mod tests {
 
         let legacy_endpoint = format!("http://legacy-{}.invalid", Uuid::new_v4());
         let legacy_entry = put_file_capability_cache_entry(&legacy_endpoint);
-        legacy_entry.write().cached = Some(PutFileCapabilityState::LegacyUntil(Instant::now() + PUT_FILE_LEGACY_CAPABILITY_TTL));
+        legacy_entry.write().await.cached =
+            Some(PutFileCapabilityState::LegacyUntil(Instant::now() + PUT_FILE_LEGACY_CAPABILITY_TTL));
         assert!(
             transport
                 .put_file_auth_capability(&legacy_endpoint)
@@ -1208,7 +1098,7 @@ mod tests {
 
         let expired_endpoint = format!("http://expired-legacy-{}.invalid", Uuid::new_v4());
         let expired_entry = put_file_capability_cache_entry(&expired_endpoint);
-        expired_entry.write().cached = Some(PutFileCapabilityState::LegacyUntil(Instant::now()));
+        expired_entry.write().await.cached = Some(PutFileCapabilityState::LegacyUntil(Instant::now()));
         let reprobed = std::sync::atomic::AtomicBool::new(false);
         assert_eq!(
             resolve_put_file_auth_capability(&expired_endpoint, || async {
@@ -1459,7 +1349,7 @@ mod tests {
         };
         probe_started.notified().await;
         {
-            let mut state = entry.write();
+            let mut state = entry.write().await;
             state.generation = state.generation.checked_add(1).expect("test generation should advance");
             state.cached = Some(PutFileCapabilityState::V1 {
                 server_epoch: newer_epoch,
@@ -1472,7 +1362,10 @@ mod tests {
             task.await.expect("stale task should finish").expect("stale probe result"),
             Some(stale_epoch)
         );
-        assert_eq!(fresh_put_file_capability(entry.read().cached, Instant::now()), Some(Some(newer_epoch)));
+        assert_eq!(
+            fresh_put_file_capability(entry.read().await.cached, Instant::now()),
+            Some(Some(newer_epoch))
+        );
     }
 
     #[test]
@@ -1505,8 +1398,6 @@ mod tests {
 
         let _ = rustfs_credentials::set_global_rpc_secret("put-file-auth-writer-test-secret".to_string());
         let nonce = Uuid::parse_str("11111111-2222-4333-8444-555555555555").expect("nonce");
-        let server_epoch = Uuid::parse_str("aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee").expect("server epoch");
-        let endpoint = "http://node1:9000".to_string();
         let url = concat!(
             "http://node1:9000/rustfs/rpc/put_file_stream?disk=disk-a&volume=bucket&path=object%2Fpart.1",
             "&append=false&size=11&put_file_auth=digest-trailer-v1&put_file_nonce=11111111-2222-4333-8444-555555555555"
@@ -1515,7 +1406,7 @@ mod tests {
         let mut sink = Vec::new();
 
         {
-            let mut writer = PutFileAuthWriter::new(&mut sink, url.clone(), nonce, endpoint, server_epoch);
+            let mut writer = PutFileAuthWriter::new(&mut sink, url.clone(), nonce);
             writer.write_all(b"hello world").await.expect("body write should succeed");
             writer.shutdown().await.expect("shutdown should append auth trailer");
             let err = writer
@@ -1533,143 +1424,6 @@ mod tests {
         assert_eq!(verified, expected_digest);
     }
 
-    #[tokio::test]
-    async fn put_file_auth_writer_reprobes_after_server_epoch_conflict() {
-        use tokio::io::AsyncWriteExt;
-
-        let _ = rustfs_credentials::set_global_rpc_secret("put-file-epoch-conflict-test-secret".to_string());
-        for status in [reqwest::StatusCode::CONFLICT, reqwest::StatusCode::BAD_REQUEST] {
-            for (phase, trailer_write) in [
-                (PutFileFailurePhase::Write, false),
-                (PutFileFailurePhase::Write, true),
-                (PutFileFailurePhase::Flush, false),
-                (PutFileFailurePhase::Shutdown, false),
-            ] {
-                let endpoint = format!("http://epoch-conflict-{}.invalid", Uuid::new_v4());
-                let stale_epoch = Uuid::new_v4();
-                let replacement_epoch = Uuid::new_v4();
-                resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(stale_epoch)) })
-                    .await
-                    .expect("initial capability should resolve");
-                let mut writer = PutFileAuthWriter::new(
-                    PutFileFailureWriter { phase, status },
-                    format!("{endpoint}{PUT_FILE_AUTH_STREAM_PATH}"),
-                    Uuid::new_v4(),
-                    endpoint.clone(),
-                    stale_epoch,
-                );
-                let error = match (phase, trailer_write) {
-                    (PutFileFailurePhase::Write, false) => writer.write_all(b"body").await,
-                    (PutFileFailurePhase::Flush, _) => writer.flush().await,
-                    _ => writer.shutdown().await,
-                }
-                .expect_err("injected writer error must reach the caller");
-                let conflict = status == reqwest::StatusCode::CONFLICT;
-                assert_eq!(io_error_has_put_file_epoch_conflict(&error), conflict);
-
-                let probe_called = AtomicBool::new(false);
-                let resolved = resolve_put_file_auth_capability(&endpoint, || async {
-                    probe_called.store(true, Ordering::SeqCst);
-                    Ok(Some(replacement_epoch))
-                })
-                .await
-                .expect("capability should remain usable or be reprobed");
-                assert_eq!(probe_called.load(Ordering::SeqCst), conflict, "phase={phase:?}, trailer={trailer_write}");
-                assert_eq!(resolved, Some(if conflict { replacement_epoch } else { stale_epoch }));
-            }
-        }
-    }
-
-    #[tokio::test]
-    async fn late_put_file_epoch_rejection_preserves_current_rejection() {
-        let endpoint = format!("http://late-epoch-conflict-{}.invalid", Uuid::new_v4());
-        let old_epoch = Uuid::new_v4();
-        let current_epoch = Uuid::new_v4();
-        let replacement_epoch = Uuid::new_v4();
-        assert_eq!(
-            resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(old_epoch)) })
-                .await
-                .expect("initial epoch should be cached"),
-            Some(old_epoch)
-        );
-        reject_put_file_server_epoch(&endpoint, old_epoch);
-        assert_eq!(
-            resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(current_epoch)) })
-                .await
-                .expect("first restart should install a new epoch"),
-            Some(current_epoch)
-        );
-
-        reject_put_file_server_epoch(&endpoint, current_epoch);
-        // A writer opened before the first restart can report its 409 after
-        // a newer writer has already rejected the second server incarnation.
-        reject_put_file_server_epoch(&endpoint, old_epoch);
-        let probe_called = AtomicBool::new(false);
-        let resolved = resolve_put_file_auth_capability(&endpoint, || async {
-            probe_called.store(true, Ordering::SeqCst);
-            Ok(Some(replacement_epoch))
-        })
-        .await
-        .expect("late old-epoch rejection must preserve the current rejection");
-
-        assert!(probe_called.load(Ordering::SeqCst), "known-rejected current epoch must be reprobed");
-        assert_eq!(resolved, Some(replacement_epoch));
-    }
-
-    #[tokio::test]
-    async fn put_file_epoch_rejection_is_endpoint_and_epoch_scoped() {
-        let endpoint = format!("http://scoped-epoch-{}.invalid", Uuid::new_v4());
-        let other_endpoint = format!("http://other-epoch-{}.invalid", Uuid::new_v4());
-        let current_epoch = Uuid::new_v4();
-        for endpoint in [&endpoint, &other_endpoint] {
-            resolve_put_file_auth_capability(endpoint, || async { Ok(Some(current_epoch)) })
-                .await
-                .expect("initial epoch should resolve");
-        }
-        reject_put_file_server_epoch(&endpoint, Uuid::new_v4());
-        assert_eq!(
-            resolve_put_file_auth_capability(&endpoint, || async { panic!("old writer must not invalidate a new epoch") })
-                .await
-                .expect("new epoch must remain cached"),
-            Some(current_epoch)
-        );
-        reject_put_file_server_epoch(&endpoint, current_epoch);
-        assert_eq!(
-            resolve_put_file_auth_capability(&other_endpoint, || async { panic!("another endpoint must stay cached") })
-                .await
-                .expect("other endpoint must remain cached"),
-            Some(current_epoch)
-        );
-    }
-
-    #[tokio::test]
-    async fn put_file_rejected_epoch_survives_failed_stale_and_downgrade_probes() {
-        let endpoint = format!("http://rejected-probe-{}.invalid", Uuid::new_v4());
-        let rejected_epoch = Uuid::new_v4();
-        let replacement_epoch = Uuid::new_v4();
-        resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(rejected_epoch)) })
-            .await
-            .expect("initial epoch should resolve");
-        reject_put_file_server_epoch(&endpoint, rejected_epoch);
-        let failure = resolve_put_file_auth_capability(&endpoint, || async { Err(Error::other("injected probe failure")) })
-            .await
-            .expect_err("probe failure must be returned");
-        assert!(failure.to_string().contains("injected probe failure"));
-        let downgrade = resolve_put_file_auth_capability(&endpoint, || async { Ok(None) })
-            .await
-            .expect_err("rejection must not unpin authenticated v1");
-        assert!(downgrade.to_string().contains("downgrade rejected"));
-        resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(rejected_epoch)) })
-            .await
-            .expect("a probe racing a restart can still return the old epoch");
-        assert_eq!(
-            resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(replacement_epoch)) })
-                .await
-                .expect("same-epoch probe must not clear known rejection"),
-            Some(replacement_epoch)
-        );
-    }
-
     #[test]
     fn walk_dir_url_encodes_disk_ref() {
         let url = build_walk_dir_url(&WalkDirStreamRequest {

crates/ecstore/src/cluster/rpc/mod.rs

@@ -39,8 +39,8 @@ pub use http_auth::{
     sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
     verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
     verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
-    verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
-    verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
+    verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+    verify_tonic_rpc_signature_with_bootstrap,
 };
 #[cfg(test)]
 pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -49,8 +49,8 @@ use rustfs_protos::proto_gen::node_service::{
     ScannerActivityRequest, ScannerActivityResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest,
     ScannerPublicationLeaseResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
     StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
-    TierMutationControlResponse, TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest,
-    node_service_client::NodeServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient,
+    TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
+    tier_mutation_control_service_client::TierMutationControlServiceClient,
 };
 pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
 use rustfs_protos::{TierMutationRpcPhase, evict_failed_connection};
@@ -122,14 +122,6 @@ fn control_plane_failure(op: &str, bucket: Option<&str>, error_code: Option<i32>
     if error_code == Some(rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32) {
         return Error::RemoteNotInitialized;
     }
-    if error_code == Some(rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode::ControlPlaneErrorInvalidArgument as i32)
-    {
-        return Error::InvalidArgument(
-            "control-plane".to_string(),
-            op.to_string(),
-            error_info.unwrap_or_else(|| format!("{op}: peer rejected invalid argument without details")),
-        );
-    }
     match error_info {
         Some(msg) => Error::other(msg),
         None => peer_failure_without_details(op, bucket),
@@ -462,31 +454,6 @@ pub struct PeerTierMutationOutcome {
     pub applied: bool,
 }
 
-#[derive(Debug, thiserror::Error)]
-#[error("{message}")]
-struct TierMutationDefinitelyRejected {
-    message: String,
-}
-
-fn tier_mutation_definitely_rejected_error(message: String) -> Error {
-    Error::other(TierMutationDefinitelyRejected { message })
-}
-
-#[cfg(test)]
-pub(crate) fn test_tier_mutation_definitely_rejected_error(message: &str) -> Error {
-    tier_mutation_definitely_rejected_error(message.to_string())
-}
-
-pub(crate) fn tier_mutation_error_is_definitely_rejected(error: &Error) -> bool {
-    matches!(
-        error,
-        Error::Io(io_error)
-            if io_error
-                .get_ref()
-                .is_some_and(|source| source.downcast_ref::<TierMutationDefinitelyRejected>().is_some())
-    )
-}
-
 fn validate_tier_mutation_response_proof(
     version: u32,
     phase: TierMutationRpcPhase,
@@ -494,16 +461,6 @@ fn validate_tier_mutation_response_proof(
     canonical_payload: &[u8],
     response: &TierMutationControlResponse,
 ) -> Result<()> {
-    if response.response_proof.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE {
-        return Err(Error::other("peer tier mutation response proof exceeds size limit"));
-    }
-    if response
-        .error_info
-        .as_ref()
-        .is_some_and(|error| error.len() > rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE)
-    {
-        return Err(Error::other("peer tier mutation error response exceeds size limit"));
-    }
     let canonical_response =
         rustfs_protos::canonical_tier_mutation_rpc_response_body(rustfs_protos::TierMutationRpcResponseProofInput {
             version,
@@ -514,7 +471,6 @@ fn validate_tier_mutation_response_proof(
             state: response.state,
             applied: response.applied,
             error_info: response.error_info.as_deref(),
-            failure_class: response.failure_class,
         })
         .map_err(|_| Error::other("tier mutation response length cannot be represented"))?;
     verify_tonic_rpc_response_proof(&canonical_response, &response.response_proof)
@@ -536,9 +492,9 @@ fn validate_tier_mutation_payload_len(phase: TierMutationRpcPhase, payload_len:
         TierMutationRpcPhase::Commit => rustfs_protos::TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE,
         TierMutationRpcPhase::Abort => {
             if payload_len == 0 {
-                return Err(Error::other("tier mutation abort payload is empty"));
+                return Ok(());
             }
-            rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE
+            return Err(Error::other("tier mutation abort payload must be empty"));
         }
         _ => return Err(Error::other("tier mutation rpc phase is unsupported")),
     };
@@ -557,29 +513,8 @@ fn tier_mutation_phase_label(phase: TierMutationRpcPhase) -> &'static str {
     }
 }
 
-fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, requested_version: u32, status: tonic::Status) -> Error {
-    let message = format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase));
-    let legacy_rejection = format!("unsupported tier mutation peer protocol version: {requested_version}");
-    // RUSTFS_COMPAT_TODO(backlog-2097-tier-mutation-v4-error-text): retain this exact v3-server rejection classifier for mixed-version peers. Remove after every supported peer returns the signed v4 failure class.
-    if requested_version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
-        && status.code() == tonic::Code::FailedPrecondition
-        && status.message().as_bytes() == legacy_rejection.as_bytes()
-    {
-        return tier_mutation_definitely_rejected_error(message);
-    }
-    Error::other(message)
-}
-
-fn tier_mutation_failed_response_error(version: u32, failure_class: i32, error_info: Option<String>) -> Error {
-    let message = error_info.unwrap_or_else(|| "peer tier mutation failed without an error".to_string());
-    if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
-        && TierMutationFailureClass::try_from(failure_class).ok() == Some(TierMutationFailureClass::PreDispatchRejected)
-    {
-        return tier_mutation_definitely_rejected_error(message);
-    }
-    // Missing/zero, unknown, and explicit Ambiguous are deliberately the same
-    // fail-closed result: the coordinator must include this peer in Abort.
-    Error::other(message)
+fn tier_mutation_control_status_error(phase: TierMutationRpcPhase, status: tonic::Status) -> Error {
+    Error::other(format!("peer tier mutation {} RPC failed: {status}", tier_mutation_phase_label(phase)))
 }
 
 impl PeerRestClient {
@@ -1372,12 +1307,8 @@ impl PeerRestClient {
             .await
     }
 
-    pub async fn abort_tier_mutation(
-        &self,
-        mutation_id: Uuid,
-        canonical_prepare_payload: Bytes,
-    ) -> Result<PeerTierMutationOutcome> {
-        self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, canonical_prepare_payload)
+    pub async fn abort_tier_mutation(&self, mutation_id: Uuid) -> Result<PeerTierMutationOutcome> {
+        self.tier_mutation_control(TierMutationRpcPhase::Abort, mutation_id, Bytes::new())
             .await
     }
 
@@ -1411,7 +1342,7 @@ impl PeerRestClient {
                         client
                             .prepare_tier_mutation(request)
                             .await
-                            .map_err(|status| tier_mutation_control_status_error(phase, version, status))?
+                            .map_err(|status| tier_mutation_control_status_error(phase, status))?
                             .into_inner()
                     }
                     TierMutationRpcPhase::Commit => {
@@ -1424,7 +1355,7 @@ impl PeerRestClient {
                         client
                             .commit_tier_mutation(request)
                             .await
-                            .map_err(|status| tier_mutation_control_status_error(phase, version, status))?
+                            .map_err(|status| tier_mutation_control_status_error(phase, status))?
                             .into_inner()
                     }
                     TierMutationRpcPhase::Abort => {
@@ -1437,19 +1368,18 @@ impl PeerRestClient {
                         client
                             .abort_tier_mutation(request)
                             .await
-                            .map_err(|status| tier_mutation_control_status_error(phase, version, status))?
+                            .map_err(|status| tier_mutation_control_status_error(phase, status))?
                             .into_inner()
                     }
                     _ => return Err(Error::other("tier mutation rpc phase is unsupported")),
                 };
                 validate_tier_mutation_response_proof(version, phase, mutation_id, &canonical_payload, &response)?;
                 if !response.success {
-                    return Err(tier_mutation_failed_response_error(version, response.failure_class, response.error_info));
-                }
-                if version == rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION
-                    && response.failure_class != TierMutationFailureClass::Unspecified as i32
-                {
-                    return Err(Error::other("successful peer tier mutation response carried a failure class"));
+                    return Err(Error::other(
+                        response
+                            .error_info
+                            .unwrap_or_else(|| "peer tier mutation failed without an error".to_string()),
+                    ));
                 }
                 let state = decode_tier_mutation_peer_state(response.state)?;
                 Ok(PeerTierMutationOutcome {
@@ -2405,29 +2335,6 @@ mod tests {
         use rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode;
         assert_eq!(ControlPlaneErrorCode::ControlPlaneErrorUnspecified as i32, 0);
         assert_eq!(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32, 1);
-        assert_eq!(ControlPlaneErrorCode::ControlPlaneErrorInvalidArgument as i32, 2);
-    }
-
-    #[test]
-    fn control_plane_failure_preserves_typed_invalid_argument_reason() {
-        use rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode;
-
-        let reason = "durable unresolved-entry recovery requires pool metadata V2 or V3";
-        let err = control_plane_failure(
-            "start_decommission",
-            None,
-            Some(ControlPlaneErrorCode::ControlPlaneErrorInvalidArgument as i32),
-            Some(reason.to_string()),
-        );
-
-        assert!(
-            matches!(
-                err,
-                Error::InvalidArgument(ref scope, ref operation, ref actual_reason)
-                    if scope == "control-plane" && operation == "start_decommission" && actual_reason == reason
-            ),
-            "forwarded validation failures must remain typed and actionable"
-        );
     }
 
     #[test]
@@ -3394,7 +3301,6 @@ mod tests {
         state: i32,
         applied: bool,
         error_info: Option<&'a str>,
-        failure_class: i32,
     }
 
     fn signed_tier_mutation_response(input: TierMutationResponseFixture<'_>) -> TierMutationControlResponse {
@@ -3408,7 +3314,6 @@ mod tests {
                 state: input.state,
                 applied: input.applied,
                 error_info: input.error_info,
-                failure_class: input.failure_class,
             })
             .expect("small tier mutation response should encode");
         let response_proof =
@@ -3419,7 +3324,6 @@ mod tests {
             applied: input.applied,
             error_info: input.error_info.map(str::to_string),
             response_proof: response_proof.into(),
-            failure_class: input.failure_class,
         }
     }
 
@@ -3437,7 +3341,6 @@ mod tests {
             state: TierMutationPeerState::Prepared as i32,
             applied: true,
             error_info: None,
-            failure_class: TierMutationFailureClass::Unspecified as i32,
         });
         validate_tier_mutation_response_proof(
             rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -3462,10 +3365,6 @@ mod tests {
                 applied: false,
                 ..response.clone()
             },
-            TierMutationControlResponse {
-                failure_class: TierMutationFailureClass::Ambiguous as i32,
-                ..response.clone()
-            },
         ] {
             let err = validate_tier_mutation_response_proof(
                 rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
@@ -3489,44 +3388,6 @@ mod tests {
         assert!(err.to_string().contains("invalid tier mutation response proof"));
     }
 
-    #[test]
-    fn tier_mutation_response_rejects_oversized_proof_and_error_before_verification() {
-        let mutation_id = Uuid::new_v4();
-        let payload = b"tier-mutation-prepare";
-        let oversized_proof = TierMutationControlResponse {
-            success: false,
-            state: TierMutationPeerState::Unspecified as i32,
-            applied: false,
-            error_info: None,
-            response_proof: vec![0; rustfs_protos::TIER_MUTATION_RPC_MAX_RESPONSE_PROOF_SIZE + 1].into(),
-            failure_class: TierMutationFailureClass::Ambiguous as i32,
-        };
-        let err = validate_tier_mutation_response_proof(
-            rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
-            TierMutationRpcPhase::Prepare,
-            mutation_id,
-            payload,
-            &oversized_proof,
-        )
-        .expect_err("oversized proof must fail before cryptographic verification");
-        assert!(err.to_string().contains("response proof exceeds size limit"));
-
-        let oversized_error = TierMutationControlResponse {
-            response_proof: Bytes::new(),
-            error_info: Some("e".repeat(rustfs_protos::TIER_MUTATION_RPC_MAX_ERROR_INFO_SIZE + 1)),
-            ..oversized_proof
-        };
-        let err = validate_tier_mutation_response_proof(
-            rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION,
-            TierMutationRpcPhase::Prepare,
-            mutation_id,
-            payload,
-            &oversized_error,
-        )
-        .expect_err("oversized error detail must fail before proof construction");
-        assert!(err.to_string().contains("error response exceeds size limit"));
-    }
-
     #[test]
     fn tier_mutation_peer_state_decode_fails_closed() {
         assert_eq!(
@@ -3571,17 +3432,8 @@ mod tests {
             )
             .is_err()
         );
-        assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).is_err());
-        validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).expect("non-empty abort payload should fit");
-        validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE)
-            .expect("max abort payload should fit");
-        assert!(
-            validate_tier_mutation_payload_len(
-                TierMutationRpcPhase::Abort,
-                rustfs_protos::TIER_MUTATION_RPC_MAX_ABORT_PAYLOAD_SIZE + 1,
-            )
-            .is_err()
-        );
+        validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 0).expect("empty abort payload should fit");
+        assert!(validate_tier_mutation_payload_len(TierMutationRpcPhase::Abort, 1).is_err());
     }
 
     #[test]
@@ -3596,7 +3448,7 @@ mod tests {
                 tonic::Status::deadline_exceeded("peer tier mutation control timed out"),
                 tonic::Status::unavailable("peer tier mutation control unavailable"),
             ] {
-                let err = tier_mutation_control_status_error(phase, rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION, status);
+                let err = tier_mutation_control_status_error(phase, status);
                 let rendered = err.to_string();
                 assert!(rendered.contains(&format!("peer tier mutation {label} RPC failed")), "{rendered}");
                 assert!(
@@ -3610,61 +3462,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn tier_mutation_v4_to_v3_rejection_classification_requires_exact_status_and_message() {
-        let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION;
-        let exact = format!("unsupported tier mutation peer protocol version: {version}");
-        let rejected = tier_mutation_control_status_error(
-            TierMutationRpcPhase::Prepare,
-            version,
-            tonic::Status::failed_precondition(exact.clone()),
-        );
-        assert!(tier_mutation_error_is_definitely_rejected(&rejected));
-
-        for status in [
-            tonic::Status::failed_precondition(format!("{exact}.")),
-            tonic::Status::failed_precondition(format!("unsupported tier mutation peer protocol version: {}", version - 1)),
-            tonic::Status::invalid_argument(exact.clone()),
-            tonic::Status::unimplemented(exact),
-        ] {
-            let ambiguous = tier_mutation_control_status_error(TierMutationRpcPhase::Prepare, version, status);
-            assert!(
-                !tier_mutation_error_is_definitely_rejected(&ambiguous),
-                "near-text, wrong-code, and Unimplemented failures must remain ambiguous"
-            );
-        }
-    }
-
-    #[test]
-    fn tier_mutation_v4_failure_class_is_typed_and_fails_closed() {
-        let version = rustfs_protos::TIER_MUTATION_RPC_PROTOCOL_VERSION;
-        let rejected = tier_mutation_failed_response_error(
-            version,
-            TierMutationFailureClass::PreDispatchRejected as i32,
-            Some("rejected".to_string()),
-        );
-        assert!(tier_mutation_error_is_definitely_rejected(&rejected));
-
-        for failure_class in [
-            TierMutationFailureClass::Unspecified as i32,
-            TierMutationFailureClass::Ambiguous as i32,
-            99,
-        ] {
-            let ambiguous = tier_mutation_failed_response_error(version, failure_class, None);
-            assert!(
-                !tier_mutation_error_is_definitely_rejected(&ambiguous),
-                "missing, unknown, and explicit ambiguous classes must trigger Abort fanout"
-            );
-        }
-
-        let v3_ignores_v4_class = tier_mutation_failed_response_error(
-            rustfs_protos::TIER_MUTATION_RPC_PREVIOUS_PROTOCOL_VERSION,
-            TierMutationFailureClass::PreDispatchRejected as i32,
-            Some("legacy failure".to_string()),
-        );
-        assert!(!tier_mutation_error_is_definitely_rejected(&v3_ignores_v4_class));
-    }
-
     #[tokio::test]
     async fn peer_rest_client_rejects_oversized_tier_prepare_before_dialing() {
         let client = test_peer_client();

crates/ecstore/src/cluster/rpc/peer_s3_client.rs

@@ -190,7 +190,7 @@ fn install_heal_bucket_pre_mutation_barrier() -> Arc<DeleteBucketEmptyScanBarrie
 }
 
 #[cfg(test)]
-pub(crate) async fn pause_after_delete_bucket_empty_scan() {
+async fn pause_after_delete_bucket_empty_scan() {
     let barrier = DELETE_BUCKET_EMPTY_SCAN_BARRIER
         .lock()
         .expect("empty scan barrier lock should not be poisoned")

crates/ecstore/src/config/com.rs

@@ -800,22 +800,11 @@ where
             if log_error {
                 error!("save_config_with_opts: err: {:?}, file: {}", err, file);
             }
-            Err(map_system_metadata_write_error(err, file))
+            Err(err)
         }
     }
 }
 
-/// A system metadata volume outage must remain retryable instead of being
-/// exposed as the user-facing bucket-not-found response.
-pub(crate) fn map_system_metadata_write_error(err: Error, file: &str) -> Error {
-    match err {
-        Error::BucketNotFound(_) | Error::VolumeNotFound => {
-            Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), file.to_string())
-        }
-        other => other,
-    }
-}
-
 fn new_server_config() -> Config {
     Config::new()
 }
@@ -2807,14 +2796,14 @@ mod tests {
     use super::{
         SERVER_CONFIG_LOCK, ServerConfigSnapshot, apply_dynamic_config_for_sub_sys_with, build_scalar_config_object,
         config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
-        heal_config_descriptor, is_standard_object_server_config, lookup_configs, map_system_metadata_write_error,
-        new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty,
-        read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot, save_config_with_opts_inner,
-        save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation,
-        server_config_transaction_lock_path, should_warn_ignored_scalar_section, storage_class_kvs_mut,
+        heal_config_descriptor, is_standard_object_server_config, lookup_configs, new_and_save_server_config, read_config,
+        read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty, read_config_with_metadata,
+        read_config_without_migrate, read_server_config_snapshot, save_server_config, save_server_config_snapshot,
+        save_server_config_snapshot_with_generation, server_config_transaction_lock_path, should_warn_ignored_scalar_section,
+        storage_class_kvs_mut,
     };
     use crate::config::{audit, heal, notify, oidc, scanner};
-    use crate::disk::{RUSTFS_META_BUCKET, endpoint::Endpoint};
+    use crate::disk::endpoint::Endpoint;
     use crate::error::{Error, Result};
     use crate::layout::endpoints::SetupType;
     use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
@@ -2846,72 +2835,6 @@ mod tests {
         assert!(rendered.contains("panicked"));
         assert!(!rendered.contains("do-not-expose-payload"));
     }
-
-    #[test]
-    fn system_metadata_volume_failures_map_to_retryable_write_errors() {
-        for error in [Error::VolumeNotFound, Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())] {
-            assert_eq!(
-                map_system_metadata_write_error(error, "buckets/example/.metadata.bin"),
-                Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), "buckets/example/.metadata.bin".to_string())
-            );
-        }
-
-        let other = Error::other("metadata encoding failed");
-        assert_eq!(map_system_metadata_write_error(other.clone(), "buckets/example/.metadata.bin"), other);
-    }
-
-    #[derive(Debug, Default)]
-    struct MetadataWriteStore {
-        error: Option<Error>,
-    }
-
-    #[async_trait::async_trait]
-    impl crate::storage_api_contracts::object::ObjectIO for MetadataWriteStore {
-        type Error = Error;
-        type RangeSpec = HTTPRangeSpec;
-        type HeaderMap = HeaderMap;
-        type ObjectOptions = ObjectOptions;
-        type ObjectInfo = ObjectInfo;
-        type GetObjectReader = GetObjectReader;
-        type PutObjectReader = PutObjReader;
-
-        async fn get_object_reader(
-            &self,
-            _bucket: &str,
-            _object: &str,
-            _range: Option<Self::RangeSpec>,
-            _headers: Self::HeaderMap,
-            _opts: &Self::ObjectOptions,
-        ) -> core::result::Result<Self::GetObjectReader, Self::Error> {
-            Err(Error::FileNotFound)
-        }
-
-        async fn put_object(
-            &self,
-            _bucket: &str,
-            _object: &str,
-            _data: &mut Self::PutObjectReader,
-            _opts: &Self::ObjectOptions,
-        ) -> core::result::Result<Self::ObjectInfo, Self::Error> {
-            Err(self.error.clone().expect("test store error should be configured"))
-        }
-    }
-
-    #[tokio::test]
-    async fn save_config_preserves_retryable_system_volume_errors() {
-        let store = Arc::new(MetadataWriteStore {
-            error: Some(Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())),
-        });
-        let error =
-            save_config_with_opts_inner(store, "buckets/example/.metadata.bin", Vec::new(), &ObjectOptions::default(), false)
-                .await
-                .expect_err("missing metadata volume must fail");
-
-        assert_eq!(
-            error,
-            Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), "buckets/example/.metadata.bin".to_string())
-        );
-    }
     use rustfs_lock::client::LockClient;
     use rustfs_lock::client::local::LocalClient;
     use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};

crates/ecstore/src/core/sets.rs

@@ -37,9 +37,8 @@ use crate::{
     runtime::instance::{InstanceContext, bootstrap_ctx},
     runtime::sources as runtime_sources,
     set_disk::{PreparedGetObjectMetadata, SetDisks},
-    store::{
-        RemoteTuplePublicationFence,
-        init_format::{check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum},
+    store::init_format::{
+        check_format_erasure_values, load_format_erasure_all, save_format_file, select_format_erasure_in_quorum,
     },
 };
 use futures::{
@@ -626,19 +625,6 @@ impl Sets {
             .put_object_with_old_current_size(bucket, object, data, opts)
             .await
     }
-
-    pub(crate) async fn put_object_with_old_current_size_for_data_movement(
-        &self,
-        bucket: &str,
-        object: &str,
-        data: &mut PutObjReader,
-        opts: &ObjectOptions,
-        publication_fence: RemoteTuplePublicationFence,
-    ) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
-        self.get_disks_by_key(object)
-            .put_object_with_old_current_size_for_data_movement(bucket, object, data, opts, publication_fence)
-            .await
-    }
 }
 
 #[async_trait::async_trait]
@@ -1365,20 +1351,11 @@ pub(crate) async fn make_local_two_set_sets_with_ctx(ctx: Arc<InstanceContext>)
 pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
     ctx: Arc<InstanceContext>,
     pool_idx: usize,
-) -> (Vec<tempfile::TempDir>, Arc<Sets>) {
-    make_local_two_set_sets_for_pool_with_drive_count_and_ctx(ctx, pool_idx, 2).await
-}
-
-#[cfg(any(test, feature = "test-util"))]
-pub(crate) async fn make_local_two_set_sets_for_pool_with_drive_count_and_ctx(
-    ctx: Arc<InstanceContext>,
-    pool_idx: usize,
-    set_drive_count: usize,
 ) -> (Vec<tempfile::TempDir>, Arc<Sets>) {
     use crate::layout::endpoint::Endpoint;
     use rustfs_lock::client::local::LocalClient;
 
-    let format = FormatV3::new(2, set_drive_count);
+    let format = FormatV3::new(2, 2);
     let mut temp_dirs = Vec::new();
     let mut all_endpoints = Vec::new();
     let mut disk_sets = Vec::new();
@@ -1386,7 +1363,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_drive_count_and_ctx(
     for set_index in 0..2 {
         let mut endpoints = Vec::new();
         let mut disks = Vec::new();
-        for disk_index in 0..set_drive_count {
+        for disk_index in 0..2 {
             let temp_dir = tempfile::tempdir().expect("tempdir should be created");
             let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("tempdir path should be utf8"))
                 .expect("endpoint should parse");
@@ -1412,7 +1389,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_drive_count_and_ctx(
             endpoints.push(endpoint);
             disks.push(Some(disk));
         }
-        let lockers = (0..set_drive_count)
+        let lockers = (0..2)
             .map(|_| {
                 Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
                     rustfs_lock::FastObjectLockManager::new(),
@@ -1423,7 +1400,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_drive_count_and_ctx(
             SetDisks::new_with_instance_ctx(
                 "test-owner".to_string(),
                 Arc::new(RwLock::new(disks)),
-                set_drive_count,
+                2,
                 1,
                 set_index,
                 pool_idx,
@@ -1443,7 +1420,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_drive_count_and_ctx(
         endpoints: PoolEndpoints {
             legacy: false,
             set_count: 2,
-            drives_per_set: set_drive_count,
+            drives_per_set: 2,
             endpoints: Endpoints::from(all_endpoints),
             cmd_line: String::new(),
             platform: String::new(),
@@ -1451,7 +1428,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_drive_count_and_ctx(
         format,
         parity_count: 1,
         set_count: 2,
-        set_drive_count,
+        set_drive_count: 2,
         default_parity_count: 1,
         distribution_algo: DistributionAlgoVersion::V1,
         exit_signal: None,

crates/ecstore/src/data_movement/mod.rs

@@ -16,18 +16,17 @@
 
 pub(crate) mod backpressure;
 
-use crate::core::pools::{DecommissionCapacityOwner, decommission_capacity_mutation_id};
 use crate::error::{
     Error, Result, is_err_data_movement_overwrite, is_err_invalid_upload_id, is_err_object_not_found, is_err_version_not_found,
 };
 use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
 use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
 use crate::storage_api_contracts::{
-    multipart::CompletePart,
+    multipart::{CompletePart, MultipartOperations as _},
     namespace::NamespaceLocking as _,
     object::{HTTPPreconditions, ObjectOperations as _},
 };
-use crate::store::{DecommissionFixedReadAnchor, ECStore, SourceCleanupMutationFence};
+use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence};
 use bytes::Bytes;
 use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
 use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
@@ -161,99 +160,6 @@ pub fn mark_multipart_upload_completed(flag: &Arc<AtomicBool>) {
     flag.store(false, Ordering::Relaxed);
 }
 
-#[cfg(all(test, feature = "test-util"))]
-struct DataMovementMultipartAbortBarrierState {
-    bucket: String,
-    object: String,
-    arrived: tokio::sync::Notify,
-    release: tokio::sync::Notify,
-}
-
-#[cfg(all(test, feature = "test-util"))]
-pub(crate) struct DataMovementMultipartAbortBarrier {
-    state: Arc<DataMovementMultipartAbortBarrierState>,
-}
-
-#[cfg(all(test, feature = "test-util"))]
-static DATA_MOVEMENT_MULTIPART_ABORT_BARRIER: std::sync::OnceLock<
-    std::sync::Mutex<Option<Arc<DataMovementMultipartAbortBarrierState>>>,
-> = std::sync::OnceLock::new();
-
-#[cfg(all(test, feature = "test-util"))]
-impl DataMovementMultipartAbortBarrier {
-    pub(crate) fn install(bucket: &str, object: &str) -> Self {
-        let state = Arc::new(DataMovementMultipartAbortBarrierState {
-            bucket: bucket.to_string(),
-            object: object.to_string(),
-            arrived: tokio::sync::Notify::new(),
-            release: tokio::sync::Notify::new(),
-        });
-        let mut slot = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
-            .get_or_init(|| std::sync::Mutex::new(None))
-            .lock()
-            .expect("data movement multipart abort barrier mutex should not poison");
-        assert!(slot.is_none(), "data movement multipart abort barrier must be unique");
-        *slot = Some(Arc::clone(&state));
-        Self { state }
-    }
-
-    pub(crate) async fn wait_until_paused(&self) {
-        tokio::time::timeout(StdDuration::from_secs(30), self.state.arrived.notified())
-            .await
-            .expect("data movement multipart failure should reach abort cleanup");
-    }
-}
-
-#[cfg(all(test, feature = "test-util"))]
-impl Drop for DataMovementMultipartAbortBarrier {
-    fn drop(&mut self) {
-        self.state.release.notify_one();
-        let mut slot = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
-            .get_or_init(|| std::sync::Mutex::new(None))
-            .lock()
-            .expect("data movement multipart abort barrier mutex should not poison");
-        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
-            *slot = None;
-        }
-    }
-}
-
-#[cfg(all(test, feature = "test-util"))]
-async fn pause_data_movement_multipart_before_abort(bucket: &str, object: &str) {
-    let barrier = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
-        .get_or_init(|| std::sync::Mutex::new(None))
-        .lock()
-        .expect("data movement multipart abort barrier mutex should not poison")
-        .as_ref()
-        .filter(|barrier| barrier.bucket == bucket && barrier.object == object)
-        .cloned();
-    if let Some(barrier) = barrier {
-        barrier.arrived.notify_one();
-        barrier.release.notified().await;
-    }
-}
-
-fn data_movement_abort_opts(
-    src_pool_idx: usize,
-    expected_bucket_incarnation_id: Option<uuid::Uuid>,
-    lock_lost_signal: Option<&Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
-    capacity_owner: Option<DecommissionCapacityOwner>,
-) -> ObjectOptions {
-    let mut opts = ObjectOptions {
-        data_movement: true,
-        src_pool_idx,
-        expected_bucket_incarnation_id,
-        ..Default::default()
-    };
-    if let Some(capacity_owner) = capacity_owner {
-        capacity_owner.apply_to(&mut opts);
-    }
-    if let Some(signal) = lock_lost_signal {
-        opts.add_namespace_lock_lost_signal(Arc::clone(signal));
-    }
-    opts
-}
-
 fn insert_data_movement_checksum(user_defined: &mut HashMap<String, String>, object_info: &ObjectInfo) {
     rustfs_utils::http::remove_header_map(user_defined, rustfs_utils::http::SUFFIX_REPLICATION_SSEC_CRC);
     if let Some(checksum) = object_info.checksum.as_ref().filter(|checksum| !checksum.is_empty()) {
@@ -286,7 +192,7 @@ fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usiz
         preserve_etag: object_info.etag.clone(),
         src_pool_idx,
         data_movement: true,
-        ..ObjectOptions::with_capacity_expected_data_bytes(usize::try_from(object_info.size).ok())
+        ..Default::default()
     }
 }
 
@@ -457,7 +363,7 @@ fn data_movement_complete_multipart_opts(
         preserve_etag: object_info.etag.clone(),
         user_defined,
         src_pool_idx,
-        ..ObjectOptions::with_capacity_expected_data_bytes(usize::try_from(object_info.size).ok())
+        ..Default::default()
     })
 }
 
@@ -627,7 +533,6 @@ fn schedule_data_movement_multipart_abort_cleanup(
     bucket: String,
     object: String,
     upload_id: String,
-    opts: ObjectOptions,
     op_label: &str,
 ) {
     let op_label = op_label.to_string();
@@ -635,32 +540,23 @@ fn schedule_data_movement_multipart_abort_cleanup(
         for attempt in 1..=DATA_MOVEMENT_MULTIPART_ABORT_RETRY_ATTEMPTS {
             tokio::time::sleep(StdDuration::from_secs(DATA_MOVEMENT_MULTIPART_ABORT_RETRY_DELAY_SECS)).await;
 
-            if store.pools.get(target_pool_idx).is_none() {
+            let Some(pool) = store.pools.get(target_pool_idx).cloned() else {
                 error!(
                     "{op_label}: background abort_multipart_upload cleanup skipped for {bucket}/{object} upload {upload_id}: target pool {target_pool_idx} is out of range"
                 );
                 return;
-            }
-
-            let mut cleanup_opts = opts.clone();
-            let _multipart_mutation_fence = match DecommissionCapacityOwner::from_options(&cleanup_opts) {
-                Some(owner) => match store.acquire_decommission_multipart_mutation_fence(owner).await {
-                    Ok(fence) => {
-                        fence.add_namespace_lock_fence(&mut cleanup_opts);
-                        Some(fence)
-                    }
-                    Err(err) => {
-                        error!(
-                            "{op_label}: background abort_multipart_upload cleanup could not fence {bucket}/{object} upload {upload_id} on attempt {attempt}: {err:?}"
-                        );
-                        continue;
-                    }
-                },
-                None => None,
             };
 
-            match store
-                .abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object, &upload_id, &cleanup_opts)
+            match pool
+                .abort_multipart_upload(
+                    &bucket,
+                    &object,
+                    &upload_id,
+                    &ObjectOptions {
+                        data_movement: true,
+                        ..Default::default()
+                    },
+                )
                 .await
             {
                 Ok(()) => {
@@ -717,8 +613,8 @@ fn is_equivalent_data_movement_part(source: &ObjectPartInfo, target: &ObjectPart
                 == target.checksums.as_ref().filter(|checksums| !checksums.is_empty()))
 }
 
-pub(crate) fn data_movement_parts_by_number(parts: &[ObjectPartInfo]) -> Option<HashMap<usize, &ObjectPartInfo>> {
-    let mut parts_by_number = HashMap::with_capacity(parts.len());
+fn data_movement_parts_by_number(parts: &[ObjectPartInfo]) -> Option<BTreeMap<usize, &ObjectPartInfo>> {
+    let mut parts_by_number = BTreeMap::new();
     for part in parts {
         if parts_by_number.insert(part.number, part).is_some() {
             return None;
@@ -1438,43 +1334,27 @@ fn resolve_data_movement_overwrite_resume_result_for(
     Ok(matches!(err, Error::PreconditionFailed) && is_superseding_unversioned_data_movement_object(source, &target))
 }
 
-#[derive(Clone, Copy)]
-struct DataMovementOverwriteCapacity {
-    owner: Option<DecommissionCapacityOwner>,
-    expected_data_bytes: Option<usize>,
-}
-
 async fn should_treat_data_movement_overwrite_as_complete(
     store: &ECStore,
-    pool_indices: (usize, usize),
+    src_pool_idx: usize,
+    target_pool_idx: usize,
     bucket: &str,
     object_info: &ObjectInfo,
     err: &Error,
     compare_part_checksums: bool,
-    capacity: DataMovementOverwriteCapacity,
 ) -> Result<bool> {
     if !should_check_data_movement_overwrite_resume(err) {
         return Ok(false);
     }
-    let (src_pool_idx, target_pool_idx) = pool_indices;
 
-    let equivalent = resolve_data_movement_overwrite_resume_result_for(
+    resolve_data_movement_overwrite_resume_result_for(
         err,
         find_data_movement_target_info(store, target_pool_idx, bucket, object_info).await,
         object_info,
         src_pool_idx,
         target_pool_idx,
         compare_part_checksums,
-    )?;
-    if equivalent && let Some(owner) = capacity.owner {
-        let expected_data_bytes = capacity
-            .expected_data_bytes
-            .ok_or_else(|| Error::other("equivalent data-movement target cannot reconcile unknown committed data size"))?;
-        store
-            .reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, expected_data_bytes)
-            .await?;
-    }
-    Ok(equivalent)
+    )
 }
 
 fn data_movement_part_stage_error(
@@ -1515,10 +1395,9 @@ pub(crate) async fn migrate_decommission_object(
     rd: GetObjectReader,
     source_bucket_incarnation_id: Option<uuid::Uuid>,
     op_label: &str,
-    capacity_owner: Option<DecommissionCapacityOwner>,
 ) -> Result<()> {
     let source = rd.object_info.clone();
-    let mutation_fence = store
+    let _mutation_fence = store
         .acquire_decommission_object_mutation_fence(&bucket, &source.name)
         .await?;
     let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source)
@@ -1536,8 +1415,7 @@ pub(crate) async fn migrate_decommission_object(
         source_bucket_incarnation_id,
         op_label,
         None,
-        capacity_owner,
-        Some(mutation_fence),
+        Some(&_mutation_fence),
     )
     .await
 }
@@ -1573,7 +1451,6 @@ pub(crate) async fn migrate_object_with_lock_lost_signal(
         op_label,
         lock_lost_signal,
         None,
-        None,
     )
     .await
 }
@@ -1587,117 +1464,24 @@ async fn migrate_object_inner(
     source_bucket_incarnation_id: Option<uuid::Uuid>,
     op_label: &str,
     lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
-    capacity_owner: Option<DecommissionCapacityOwner>,
-    mutation_fence: Option<DecommissionFixedReadAnchor>,
+    mutation_fence: Option<&ObjectLockDiagGuard>,
 ) -> Result<()> {
-    let mut mutation_fence = mutation_fence;
     let object_info = rd.object_info.clone();
-    let capacity_owner = capacity_owner.map(|owner| {
-        let version_id = object_info.version_id.map(|version_id| version_id.to_string());
-        let mutation_id = owner.mutation_id.unwrap_or_else(|| {
-            decommission_capacity_mutation_id(
-                owner,
-                &bucket,
-                &object_info.name,
-                version_id.as_deref(),
-                object_info.delete_marker,
-                object_info.mod_time,
-            )
-        });
-        owner.with_mutation_id(mutation_id)
-    });
-    // Capture the exact source/tier identity before any client-paced read, but
-    // defer both the tier lease and source/target write locks to the final
-    // publication. Decommission already owns main's fixed-domain mutation
-    // fence, so reacquiring that domain as a write lock would self-deadlock.
-    let remote_tuple_publication_fence = store
-        .acquire_remote_tuple_publication_fence(&bucket, pool_idx, &object_info, false)
-        .await?;
     let has_part_checksums = object_info
         .parts
         .iter()
         .any(|part| part.checksums.as_ref().is_some_and(|checksums| !checksums.is_empty()));
 
     let preserve_part_checksums = data_movement_part_checksum_writer_enabled();
-    let capacity_expected_data_bytes = usize::try_from(object_info.size).ok();
 
     if should_use_multipart_data_movement(&object_info, has_part_checksums) {
-        // The decommission object fence already covers the source/target
-        // namespace for this migration. Acquiring the synthetic multipart
-        // fence while holding that read lock deadlocks local lock domains;
-        // retain the extra fence only for callers without the outer fence.
-        let multipart_mutation_fence = match (capacity_owner, mutation_fence.is_some()) {
-            (Some(owner), false) => Some(store.acquire_decommission_multipart_mutation_fence(owner).await?),
-            _ => None,
-        };
         let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx);
-        if let Some(capacity_owner) = capacity_owner {
-            capacity_owner.apply_to(&mut new_multipart_opts);
-        }
         new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
         if let Some(signal) = lock_lost_signal.as_ref() {
             new_multipart_opts.add_namespace_lock_lost_signal(Arc::clone(signal));
         }
-        if let Some(fence) = multipart_mutation_fence.as_ref() {
-            fence.add_namespace_lock_fence(&mut new_multipart_opts);
-        }
-        if let Some(owner) = capacity_owner {
-            let existing_target_pool_idx = store
-                .select_data_movement_pool_idx(&bucket, &object_info.name, -1, &new_multipart_opts, false)
-                .await?;
-            if existing_target_pool_idx != pool_idx
-                && let Some(target) =
-                    find_data_movement_target_info(store.as_ref(), existing_target_pool_idx, &bucket, &object_info).await?
-                && is_equivalent_data_movement_object_identity(&object_info, &target, true, preserve_part_checksums)
-            {
-                let expected_data_bytes = capacity_expected_data_bytes
-                    .ok_or_else(|| Error::other("equivalent multipart target cannot reconcile unknown committed data size"))?;
-                store
-                    .reconcile_decommission_capacity_after_equivalent_target(owner, existing_target_pool_idx, expected_data_bytes)
-                    .await?;
-                info!(
-                    "{op_label}: multipart upload restart reconciled equivalent target for {}/{}",
-                    bucket.as_str(),
-                    object_info.name.as_str()
-                );
-                return Ok(());
-            }
-            let mut cleanup_opts =
-                data_movement_abort_opts(pool_idx, source_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
-            if let Some(anchor) = mutation_fence.as_ref() {
-                anchor.guard().add_namespace_lock_fence(&mut cleanup_opts);
-            }
-            if let Some(fence) = multipart_mutation_fence.as_ref() {
-                fence.add_namespace_lock_fence(&mut cleanup_opts);
-            }
-            for target_pool_idx in store.decommission_capacity_cleanup_target_indices(owner).await? {
-                store
-                    .reconcile_multipart_uploads_for_data_movement(
-                        target_pool_idx,
-                        &bucket,
-                        &object_info.name,
-                        &data_movement_upload_identity(&object_info),
-                        &cleanup_opts,
-                    )
-                    .await
-                    .map_err(|err| {
-                        data_movement_stage_error(
-                            op_label,
-                            "reconcile_multipart_upload",
-                            bucket.as_str(),
-                            object_info.name.as_str(),
-                            err,
-                        )
-                    })?;
-            }
-        }
         let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
-            .handle_new_multipart_upload_with_pool_idx(
-                &bucket,
-                &object_info.name,
-                &new_multipart_opts,
-                mutation_fence.as_ref().map(DecommissionFixedReadAnchor::guard),
-            )
+            .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
             .await
         {
             Ok(res) => res,
@@ -1748,15 +1532,9 @@ async fn migrate_object_inner(
                     expected_bucket_incarnation_id,
                     ..Default::default()
                 };
-                if let Some(capacity_owner) = capacity_owner {
-                    capacity_owner.apply_to(&mut part_opts);
-                }
                 if let Some(signal) = lock_lost_signal.as_ref() {
                     part_opts.add_namespace_lock_lost_signal(Arc::clone(signal));
                 }
-                if let Some(fence) = multipart_mutation_fence.as_ref() {
-                    fence.add_namespace_lock_fence(&mut part_opts);
-                }
                 let pi = match store
                     .put_object_part_for_data_movement(
                         target_pool_idx,
@@ -1800,44 +1578,30 @@ async fn migrate_object_inner(
                         err,
                     )
                 })?;
-            if let Some(capacity_owner) = capacity_owner {
-                capacity_owner.apply_to(&mut complete_multipart_opts);
-            }
             complete_multipart_opts.expected_bucket_incarnation_id = expected_bucket_incarnation_id;
             if let Some(signal) = lock_lost_signal.as_ref() {
                 complete_multipart_opts.add_namespace_lock_lost_signal(Arc::clone(signal));
             }
-            if let Some(fence) = multipart_mutation_fence.as_ref() {
-                fence.add_namespace_lock_fence(&mut complete_multipart_opts);
-            }
-            let remote_tuple_publication_fence = match mutation_fence.take() {
-                Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?,
-                None => remote_tuple_publication_fence,
-            };
             if let Err(err) = store
                 .clone()
-                .complete_multipart_upload_for_data_movement_with_publication_fence(
-                    target_pool_idx,
+                .complete_multipart_upload_for_data_movement(
+                    (target_pool_idx, mutation_fence),
                     &bucket,
                     &object_info.name,
                     &res.upload_id,
                     parts,
                     &complete_multipart_opts,
-                    remote_tuple_publication_fence,
                 )
                 .await
             {
                 if should_treat_data_movement_overwrite_as_complete(
                     store.as_ref(),
-                    (pool_idx, target_pool_idx),
+                    pool_idx,
+                    target_pool_idx,
                     bucket.as_str(),
                     &object_info,
                     &err,
                     preserve_part_checksums,
-                    DataMovementOverwriteCapacity {
-                        owner: capacity_owner,
-                        expected_data_bytes: capacity_expected_data_bytes,
-                    },
                 )
                 .await?
                 {
@@ -1865,37 +1629,31 @@ async fn migrate_object_inner(
         .await;
 
         if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) {
-            let mut abort_opts =
-                data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
-            if let Some(anchor) = mutation_fence.as_ref() {
-                anchor.guard().add_namespace_lock_fence(&mut abort_opts);
-            }
-            if let Some(fence) = multipart_mutation_fence.as_ref() {
-                fence.add_namespace_lock_fence(&mut abort_opts);
-            }
             let abort_result = store
-                .abort_multipart_upload_for_data_movement(
-                    target_pool_idx,
-                    &bucket,
-                    &object_info.name,
-                    &res.upload_id,
-                    &abort_opts,
-                )
+                .abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object_info.name, &res.upload_id, &{
+                    let mut opts = ObjectOptions {
+                        data_movement: true,
+                        src_pool_idx: pool_idx,
+                        expected_bucket_incarnation_id,
+                        ..Default::default()
+                    };
+                    if let Some(signal) = lock_lost_signal.as_ref() {
+                        opts.add_namespace_lock_lost_signal(Arc::clone(signal));
+                    }
+                    opts
+                })
                 .await;
             match abort_result {
                 Ok(()) => return Ok(()),
                 Err(abort_err) if is_err_invalid_upload_id(&abort_err) => {
                     if should_treat_data_movement_overwrite_as_complete(
                         store.as_ref(),
-                        (pool_idx, target_pool_idx),
+                        pool_idx,
+                        target_pool_idx,
                         bucket.as_str(),
                         &object_info,
                         &abort_err,
                         preserve_part_checksums,
-                        DataMovementOverwriteCapacity {
-                            owner: capacity_owner,
-                            expected_data_bytes: capacity_expected_data_bytes,
-                        },
                     )
                     .await?
                     {
@@ -1925,7 +1683,6 @@ async fn migrate_object_inner(
                         bucket.clone(),
                         object_info.name.clone(),
                         res.upload_id.clone(),
-                        abort_opts,
                         op_label,
                     );
                     return Err(data_movement_stage_error(
@@ -1941,24 +1698,19 @@ async fn migrate_object_inner(
 
         if let Err(primary_err) = multipart_result {
             if should_abort_multipart_upload(&abort_multipart_flag) {
-                #[cfg(all(test, feature = "test-util"))]
-                pause_data_movement_multipart_before_abort(&bucket, &object_info.name).await;
-                let mut abort_opts =
-                    data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
-                if let Some(anchor) = mutation_fence.as_ref() {
-                    anchor.guard().add_namespace_lock_fence(&mut abort_opts);
-                }
-                if let Some(fence) = multipart_mutation_fence.as_ref() {
-                    fence.add_namespace_lock_fence(&mut abort_opts);
-                }
                 return match store
-                    .abort_multipart_upload_for_data_movement(
-                        target_pool_idx,
-                        &bucket,
-                        &object_info.name,
-                        &res.upload_id,
-                        &abort_opts,
-                    )
+                    .abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object_info.name, &res.upload_id, &{
+                        let mut opts = ObjectOptions {
+                            data_movement: true,
+                            src_pool_idx: pool_idx,
+                            expected_bucket_incarnation_id,
+                            ..Default::default()
+                        };
+                        if let Some(signal) = lock_lost_signal.as_ref() {
+                            opts.add_namespace_lock_lost_signal(Arc::clone(signal));
+                        }
+                        opts
+                    })
                     .await
                 {
                     Ok(()) => Err(primary_err),
@@ -1970,7 +1722,6 @@ async fn migrate_object_inner(
                             bucket.clone(),
                             object_info.name.clone(),
                             res.upload_id.clone(),
-                            abort_opts,
                             op_label,
                         );
                         Err(resolve_data_movement_abort_result(
@@ -1993,39 +1744,23 @@ async fn migrate_object_inner(
     let mut data = data_movement_put_object_reader(bucket.as_str(), &object_info, rd, op_label)?;
 
     let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx);
-    if let Some(capacity_owner) = capacity_owner {
-        capacity_owner.apply_to(&mut put_opts);
-    }
     put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
     if let Some(signal) = lock_lost_signal {
         put_opts.add_namespace_lock_lost_signal(signal);
     }
-    let remote_tuple_publication_fence = match mutation_fence.take() {
-        Some(anchor) => remote_tuple_publication_fence.under_fixed_read_anchor(anchor)?,
-        None => remote_tuple_publication_fence,
-    };
     let (target_pool_idx, put_result) = store
-        .put_object_for_data_movement_with_publication_fence(
-            &bucket,
-            &object_info.name,
-            &mut data,
-            &put_opts,
-            remote_tuple_publication_fence,
-        )
+        .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
         .await
         .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?;
     if let Err(err) = put_result {
         if should_treat_data_movement_overwrite_as_complete(
             store.as_ref(),
-            (pool_idx, target_pool_idx),
+            pool_idx,
+            target_pool_idx,
             bucket.as_str(),
             &object_info,
             &err,
             preserve_part_checksums,
-            DataMovementOverwriteCapacity {
-                owner: capacity_owner,
-                expected_data_bytes: capacity_expected_data_bytes,
-            },
         )
         .await?
         {

crates/ecstore/src/data_usage/mod.rs

@@ -109,10 +109,7 @@ static USAGE_MEMORY_GENERATION: AtomicU64 = AtomicU64::new(0);
 /// strictly tighter than beta.11 (usage treated as 0) and strictly more
 /// available than a blanket 503. The fallback applies to any window without
 /// authoritative usage, not only pre-v2 upgrades; the values always come from
-/// the last persisted scanner output — pre-discard sizes of the
-/// authoritative snapshot first, backfilled per bucket from the observed
-/// (nonconverged) snapshot for buckets no authoritative cycle has covered
-/// yet (issue #6852). Loads go through the TTL-bounded
+/// the last persisted scanner output. Loads go through the TTL-bounded
 /// snapshot cache, so the quota path adds at most one backend read per
 /// [`DATA_USAGE_CACHE_TTL_SECS`] window. Returns `None` for buckets absent
 /// from every persisted snapshot — those still fail closed.
@@ -171,7 +168,7 @@ fn fresh_cached_data_usage_snapshot(
 
 fn cache_data_usage_snapshot_result(
     cache: &mut Option<CachedDataUsageSnapshot>,
-    result: Result<LoadedUsageBaseline, Error>,
+    result: Result<(DataUsageInfo, HashMap<String, u64>), Error>,
     loaded_at: tokio::time::Instant,
     refresh_generation: u64,
     current_generation: u64,
@@ -181,19 +178,7 @@ fn cache_data_usage_snapshot_result(
     }
 
     Some(match result {
-        Ok(LoadedUsageBaseline {
-            info,
-            mut degraded_baseline,
-            observed_unavailable,
-        }) => {
-            // A flaky observed read must not shrink quota coverage for a TTL
-            // window: carry the previous refresh's baseline entries forward,
-            // letting the fresh (authoritative) values win where they exist.
-            if observed_unavailable && let Some(previous) = cache.as_ref() {
-                for (bucket, size) in &previous.degraded_baseline {
-                    degraded_baseline.entry(bucket.clone()).or_insert(*size);
-                }
-            }
+        Ok((info, degraded_baseline)) => {
             *cache = Some(CachedDataUsageSnapshot {
                 info: Some(info.clone()),
                 loaded_at,
@@ -437,7 +422,9 @@ async fn save_data_usage_in_backend(
     if publication_epoch != expected_publication_epoch {
         return Err(Error::other("data usage publication epoch changed before save"));
     }
-    crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data).await?;
+    crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
+        .await
+        .map_err(Error::other)?;
     drop(publication_guard);
 
     cleanup_observed_data_usage_after_authoritative_save_with_publication(store.as_ref(), &data_usage_info, Some(store.as_ref()))
@@ -654,7 +641,7 @@ where
     {
         Ok(reader) => reader,
         Err(Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::ConfigNotFound) => return Ok(None),
-        Err(err) => return Err(map_data_usage_metadata_read_error(err, object)),
+        Err(err) => return Err(err),
     };
     let revision = reader
         .object_info
@@ -669,18 +656,6 @@ where
     Ok(Some((data_usage_info, revision)))
 }
 
-/// A missing usage object is harmless during bucket creation, but a missing
-/// system metadata volume is a storage outage. Keep the latter retryable and
-/// distinguishable from the user bucket not existing.
-fn map_data_usage_metadata_read_error(err: Error, object: &str) -> Error {
-    match err {
-        Error::BucketNotFound(_) | Error::VolumeNotFound => {
-            Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), object.to_string())
-        }
-        other => other,
-    }
-}
-
 fn data_usage_contains_bucket(data_usage_info: &DataUsageInfo, bucket: &str) -> bool {
     data_usage_info.buckets_usage.contains_key(bucket) || data_usage_info.bucket_sizes.contains_key(bucket)
 }
@@ -937,7 +912,7 @@ where
             )
             .await;
         drop(publication_guard);
-        match save_result.map_err(|err| crate::config::com::map_system_metadata_write_error(err, object)) {
+        match save_result {
             Ok(_) => return Ok(()),
             Err(err) => {
                 if let Some((observed, observed_revision)) = load_data_usage_for_bucket_removal(store, object).await? {
@@ -1128,78 +1103,24 @@ async fn load_data_usage_snapshot(store: Arc<ECStore>) -> Result<(DataUsageInfo,
 /// Load data usage info from backend storage
 #[instrument(skip(store))]
 pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
-    Ok(load_data_usage_from_backend_with_baseline(store).await?.info)
-}
-
-/// One refresh of the persisted usage snapshot plus the quota-admission
-/// baseline derived from it.
-struct LoadedUsageBaseline {
-    info: DataUsageInfo,
-    degraded_baseline: HashMap<String, u64>,
-    /// True when the observed snapshot could not be read (a transport error,
-    /// not absence): the cached loader then carries the previous refresh's
-    /// baseline entries forward instead of shrinking quota coverage for a
-    /// whole TTL window over one flaky read.
-    observed_unavailable: bool,
+    Ok(load_data_usage_from_backend_with_baseline(store).await?.0)
 }
 
 /// Like [`load_data_usage_from_backend`], but also returns the pre-discard
 /// per-bucket sizes so the cached loader can retain them as the degraded
 /// quota-admission baseline (issue #5716).
-async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Result<LoadedUsageBaseline, Error> {
-    let (loaded_snapshot, source) = load_data_usage_snapshot(store.clone()).await?;
-    // The observed-newness gate below compares against the snapshot as
-    // persisted, before normalization demotes or discards anything.
-    let authoritative_as_persisted = loaded_snapshot.clone();
-    let (info, mut degraded_baseline) = normalize_loaded_data_usage(loaded_snapshot, source.is_authoritative()).await;
-
-    // A bucket without a converged scanner cycle behind it — a freshly joined
-    // replica whose every cycle is superseded by the sustained replication
-    // write stream, or a bucket created after the last converged cycle on a
-    // busy site (#6852) — has no authoritative size, and quota admission
-    // fails its writes closed indefinitely. The observed (nonconverged)
-    // snapshot those superseded cycles still publish is the only grounded
-    // usage in that window, so it backfills buckets the loaded baseline does
-    // not cover; a value already in the baseline always wins. The newness
-    // gate ties the observation to this exact authoritative snapshot, so a
-    // stale observed object left behind by an earlier incarnation (e.g. a
-    // deleted and recreated bucket) cannot inject ghost usage. Loads sit
-    // behind the same TTL cache as the snapshot itself, so this adds at most
-    // one backend read per TTL window.
-    let mut observed_unavailable = false;
-    match load_observed_data_usage_snapshot(store).await {
-        Ok(Some(observed)) if observed_data_usage_is_newer(&observed, &authoritative_as_persisted) => {
-            backfill_degraded_baseline_from_observed(&mut degraded_baseline, &observed);
-        }
-        Ok(_) => {}
-        Err(_) => observed_unavailable = true,
-    }
-
-    Ok(LoadedUsageBaseline {
-        info,
-        degraded_baseline,
-        observed_unavailable,
-    })
+async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Result<(DataUsageInfo, HashMap<String, u64>), Error> {
+    let (data_usage_info, source) = load_data_usage_snapshot(store).await?;
+    Ok(normalize_loaded_data_usage(data_usage_info, source.is_authoritative()).await)
 }
 
-/// Fill quota-baseline gaps from an observed (nonconverged) snapshot without
-/// overriding any bucket the authoritative baseline already covers.
-fn backfill_degraded_baseline_from_observed(degraded_baseline: &mut HashMap<String, u64>, observed: &DataUsageInfo) {
-    for (bucket, usage) in &observed.buckets_usage {
-        degraded_baseline.entry(bucket.clone()).or_insert(usage.size);
-    }
-}
-
-/// `Ok(None)` means the observed snapshot is absent or invalid (a settled
-/// answer); `Err` means it could not be read at all, so the caller may keep
-/// using what it learned from a previous read.
-async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Result<Option<DataUsageInfo>, Error> {
+async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUsageInfo> {
     let data = match read_config_preserve_empty(store, &DATA_USAGE_OBSERVED_OBJ_NAME_PATH).await {
         Ok(data) => data,
-        Err(Error::ConfigNotFound) => return Ok(None),
+        Err(Error::ConfigNotFound) => return None,
         Err(err) => {
             record_usage_snapshot_failure("read_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
-            return Err(err);
+            return None;
         }
     };
 
@@ -1208,7 +1129,7 @@ async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Result<Option
             if info.usage_snapshot_converged == Some(false)
                 && (info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()) =>
         {
-            Ok(Some(info))
+            Some(info)
         }
         Ok(_) => {
             error!(
@@ -1219,11 +1140,11 @@ async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Result<Option
                 object = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
                 "observed data usage snapshot was not a structurally complete nonconverged view"
             );
-            Ok(None)
+            None
         }
         Err(err) => {
             record_usage_snapshot_decode_failure("parse_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
-            Ok(None)
+            None
         }
     }
 }
@@ -1281,9 +1202,7 @@ fn merge_partial_observation_for_admin(mut authoritative: DataUsageInfo, observe
 
 async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
     let (authoritative, source) = load_data_usage_snapshot(store.clone()).await?;
-    // For the one-shot admin view a failed observed read degrades to "no
-    // observation", same as before the read was fallible.
-    let observed = load_observed_data_usage_snapshot(store).await.ok().flatten();
+    let observed = load_observed_data_usage_snapshot(store).await;
     let (selected, selected_is_current_format) =
         select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
     Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0)
@@ -1446,11 +1365,7 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
         let refresh_generation = admin_data_usage_snapshot_generation();
         let result = load_admin_data_usage_from_backend(store.clone())
             .await
-            .map(|info| LoadedUsageBaseline {
-                info,
-                degraded_baseline: HashMap::new(),
-                observed_unavailable: false,
-            });
+            .map(|info| (info, HashMap::new()));
         let loaded_at = tokio::time::Instant::now();
         let mut cache = admin_data_usage_snapshot_cache().write().await;
         if let Some(result) = cache_data_usage_snapshot_result(
@@ -2601,37 +2516,6 @@ mod tests {
     use std::sync::Arc;
     use tokio::{io::AsyncReadExt, sync::Mutex};
 
-    #[test]
-    fn observed_snapshot_only_backfills_baseline_gaps() {
-        let mut baseline = HashMap::from([("covered".to_string(), 111_u64)]);
-        let observed = DataUsageInfo {
-            buckets_usage: HashMap::from([
-                (
-                    "covered".to_string(),
-                    BucketUsageInfo {
-                        size: 999,
-                        ..Default::default()
-                    },
-                ),
-                (
-                    "replica-only".to_string(),
-                    BucketUsageInfo {
-                        size: 42,
-                        ..Default::default()
-                    },
-                ),
-            ]),
-            ..Default::default()
-        };
-
-        backfill_degraded_baseline_from_observed(&mut baseline, &observed);
-
-        // The authoritative value must win; only the uncovered bucket (#6852:
-        // a replica that never landed a converged cycle) is filled in.
-        assert_eq!(baseline.get("covered"), Some(&111));
-        assert_eq!(baseline.get("replica-only"), Some(&42));
-    }
-
     #[derive(Debug, Default)]
     struct UsageCasState {
         object: Option<(Vec<u8>, u64)>,
@@ -2875,7 +2759,6 @@ mod tests {
     struct UsageCacheReadStore {
         transient_failures: Mutex<usize>,
         reads: Mutex<Vec<String>>,
-        terminal_error: Mutex<Option<Error>>,
     }
 
     impl UsageCacheReadStore {
@@ -2883,15 +2766,6 @@ mod tests {
             Self {
                 transient_failures: Mutex::new(n),
                 reads: Mutex::new(Vec::new()),
-                terminal_error: Mutex::new(None),
-            }
-        }
-
-        fn with_terminal_error(error: Error) -> Self {
-            Self {
-                transient_failures: Mutex::new(0),
-                reads: Mutex::new(Vec::new()),
-                terminal_error: Mutex::new(Some(error)),
             }
         }
 
@@ -2919,9 +2793,6 @@ mod tests {
             _opts: &Self::ObjectOptions,
         ) -> Result<Self::GetObjectReader, Self::Error> {
             self.reads.lock().await.push(object.to_string());
-            if let Some(error) = self.terminal_error.lock().await.clone() {
-                return Err(error);
-            }
             let mut remaining = self.transient_failures.lock().await;
             if *remaining > 0 {
                 *remaining -= 1;
@@ -2964,7 +2835,6 @@ mod tests {
             decommission_cancelers: RwLock::new(Vec::new()),
             start_gate: TokioMutex::new(()),
             pool_meta_save_gate: TokioMutex::default(),
-            decommission_capacity_entry_gate: TokioMutex::default(),
             ctx,
             bucket_fence_registry: Arc::default(),
         })
@@ -2987,22 +2857,6 @@ mod tests {
         assert!(!is_data_usage_cache_absent(&Error::DiskNotFound));
     }
 
-    #[test]
-    fn data_usage_removal_maps_missing_system_volume_to_read_quorum() {
-        for error in [Error::VolumeNotFound, Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())] {
-            assert_eq!(
-                map_data_usage_metadata_read_error(error, "bucket-metadata/.usage.json"),
-                Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string())
-            );
-        }
-
-        let missing_object = Error::ObjectNotFound(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string());
-        assert_eq!(
-            map_data_usage_metadata_read_error(missing_object.clone(), "bucket-metadata/.usage.json"),
-            missing_object
-        );
-    }
-
     #[tokio::test]
     async fn load_data_usage_cache_treats_absence_as_an_empty_cache_without_retrying() {
         let name = "usage-cache";
@@ -3018,22 +2872,6 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    async fn data_usage_removal_surfaces_missing_system_volume_as_read_quorum() {
-        for cause in [Error::BucketNotFound(RUSTFS_META_BUCKET.to_string()), Error::VolumeNotFound] {
-            let store = UsageCacheReadStore::with_terminal_error(cause);
-
-            let error = load_data_usage_for_bucket_removal(&store, "bucket-metadata/.usage.json")
-                .await
-                .expect_err("missing system metadata volume must not be treated as an absent usage object");
-
-            assert_eq!(
-                error,
-                Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string())
-            );
-        }
-    }
-
     #[tokio::test]
     async fn load_data_usage_cache_retries_a_transient_failure() {
         let name = "usage-cache";
@@ -3586,11 +3424,7 @@ mod tests {
 
         let first = cache_data_usage_snapshot_result(
             &mut cache,
-            Ok(LoadedUsageBaseline {
-                info: expected,
-                degraded_baseline: HashMap::new(),
-                observed_unavailable: false,
-            }),
+            Ok((expected, HashMap::new())),
             loaded_at,
             refresh_generation,
             data_usage_snapshot_generation(),
@@ -3605,38 +3439,6 @@ mod tests {
         assert_snapshot_bucket(&cached, "bucket");
     }
 
-    #[test]
-    #[serial]
-    fn unavailable_observed_read_keeps_previous_baseline_coverage() {
-        let loaded_at = tokio::time::Instant::now();
-        let refresh_generation = data_usage_snapshot_generation();
-        let mut cache = Some(CachedDataUsageSnapshot {
-            info: Some(data_usage_info_for_test("bucket", 1, 42, SystemTime::UNIX_EPOCH)),
-            loaded_at,
-            degraded_baseline: HashMap::from([("observed-only".to_string(), 7_u64), ("covered".to_string(), 1)]),
-        });
-
-        cache_data_usage_snapshot_result(
-            &mut cache,
-            Ok(LoadedUsageBaseline {
-                info: data_usage_info_for_test("bucket", 1, 42, SystemTime::UNIX_EPOCH),
-                degraded_baseline: HashMap::from([("covered".to_string(), 2_u64)]),
-                observed_unavailable: true,
-            }),
-            loaded_at,
-            refresh_generation,
-            data_usage_snapshot_generation(),
-        )
-        .expect("an uninterrupted refresh should populate the cache")
-        .expect("successful load must be returned");
-
-        let baseline = &cache.as_ref().expect("cache must be populated").degraded_baseline;
-        // The bucket only the (now unreadable) observed snapshot covered must
-        // survive the refresh; the freshly loaded value wins where it exists.
-        assert_eq!(baseline.get("observed-only"), Some(&7));
-        assert_eq!(baseline.get("covered"), Some(&2));
-    }
-
     #[test]
     #[serial]
     fn cache_invalidation_during_refresh_prevents_stale_snapshot_resurrection() {
@@ -3651,11 +3453,7 @@ mod tests {
 
         let stale_result = cache_data_usage_snapshot_result(
             &mut cache,
-            Ok(LoadedUsageBaseline {
-                info: data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH),
-                degraded_baseline: HashMap::new(),
-                observed_unavailable: false,
-            }),
+            Ok((data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH), HashMap::new())),
             loaded_at,
             refresh_generation,
             data_usage_snapshot_generation(),

crates/ecstore/src/disk/disk_store.rs

@@ -275,15 +275,12 @@ where
     F: FnOnce() -> Fut + Send + 'static,
     Fut: std::future::Future<Output = Result<T>> + Send + 'static,
 {
-    if external_guard.is_none() {
-        return operation().await;
-    }
     tokio::spawn(async move {
         let _external_guard = external_guard;
         operation().await
     })
     .await
-    .map_err(|_| Error::other("owned mutation task failed"))?
+    .map_err(|err| Error::other(format!("owned mutation task failed: {err}")))?
 }
 
 impl DiskStoreRenameDataExt for LocalDiskWrapper {
@@ -755,20 +752,17 @@ impl DiskOperationMetrics {
         let elapsed_nanos = u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX);
         let slot = &self.last_minute[(now_sec % 60) as usize];
         loop {
-            // The successful CAS below is AcqRel, so it is the publication
-            // fence for the writer that owns this slot. The initial parity
-            // check does not need to acquire the slot payload.
-            let version = slot.version.load(Ordering::Relaxed);
+            let version = slot.version.load(Ordering::Acquire);
             if !version.is_multiple_of(2) {
                 std::hint::spin_loop();
                 continue;
             }
             if slot
                 .version
-                .compare_exchange(version, version.wrapping_add(1), Ordering::AcqRel, Ordering::Relaxed)
+                .compare_exchange(version, version.wrapping_add(1), Ordering::AcqRel, Ordering::Acquire)
                 .is_ok()
             {
-                if slot.unix_sec.load(Ordering::Relaxed) != now_sec {
+                if slot.unix_sec.load(Ordering::Acquire) != now_sec {
                     slot.count.store(0, Ordering::Relaxed);
                     slot.acc_time.store(0, Ordering::Relaxed);
                     slot.unix_sec.store(now_sec, Ordering::Release);
@@ -784,10 +778,14 @@ impl DiskOperationMetrics {
     fn last_minute_snapshot(&self, now_sec: u64) -> TimedAction {
         let mut snapshot = TimedAction::default();
         for slot in &self.last_minute {
-            let Some((slot_sec, count, acc_time)) = slot.snapshot() else {
+            let version = slot.version.load(Ordering::Acquire);
+            if !version.is_multiple_of(2) {
                 continue;
-            };
-            if slot_sec <= now_sec && now_sec.saturating_sub(slot_sec) < 60 {
+            }
+            let slot_sec = slot.unix_sec.load(Ordering::Acquire);
+            let count = slot.count.load(Ordering::Acquire);
+            let acc_time = slot.acc_time.load(Ordering::Acquire);
+            if slot.version.load(Ordering::Acquire) == version && slot_sec <= now_sec && now_sec.saturating_sub(slot_sec) < 60 {
                 snapshot.count = snapshot.count.saturating_add(count);
                 snapshot.acc_time = snapshot.acc_time.saturating_add(acc_time);
             }
@@ -796,23 +794,6 @@ impl DiskOperationMetrics {
     }
 }
 
-impl TimedActionSlot {
-    fn snapshot(&self) -> Option<(u64, u64, u64)> {
-        let version = self.version.load(Ordering::Acquire);
-        if !version.is_multiple_of(2) {
-            return None;
-        }
-
-        // The first Acquire load publishes the payload written before the
-        // matching Release store. Relaxed payload loads are sufficient while
-        // the final Acquire version load validates that no writer intervened.
-        let slot_sec = self.unix_sec.load(Ordering::Relaxed);
-        let count = self.count.load(Ordering::Relaxed);
-        let acc_time = self.acc_time.load(Ordering::Relaxed);
-        (self.version.load(Ordering::Acquire) == version).then_some((slot_sec, count, acc_time))
-    }
-}
-
 pub(crate) struct DiskHealthWaitingGuard<'a> {
     health: &'a DiskHealthTracker,
 }
@@ -2435,25 +2416,6 @@ mod tests {
         assert_eq!(window.acc_time, 18_000);
     }
 
-    #[test]
-    fn timed_action_slot_snapshot_skips_writer_owned_slot() {
-        let slot = TimedActionSlot::default();
-        slot.unix_sec.store(70, Ordering::Relaxed);
-        slot.count.store(2, Ordering::Relaxed);
-        slot.acc_time.store(18_000, Ordering::Relaxed);
-        slot.version.store(2, Ordering::Release);
-        assert_eq!(slot.snapshot(), Some((70, 2, 18_000)));
-
-        assert_eq!(slot.version.compare_exchange(2, 3, Ordering::AcqRel, Ordering::Relaxed), Ok(2));
-        slot.unix_sec.store(71, Ordering::Relaxed);
-        slot.count.store(1, Ordering::Relaxed);
-        slot.acc_time.store(11_000, Ordering::Relaxed);
-        assert_eq!(slot.snapshot(), None);
-
-        slot.version.store(4, Ordering::Release);
-        assert_eq!(slot.snapshot(), Some((71, 1, 11_000)));
-    }
-
     #[test]
     fn disk_health_metrics_snapshot_exports_waiting_errors_and_operation_windows() {
         let metrics = DiskHealthMetricEpoch::default();

crates/ecstore/src/disk/error.rs

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM;
 use rustfs_rio::{InternodeHttpError, InternodeHttpErrorKind};
 use std::error::Error as StdError;
 use std::hash::{Hash, Hasher};
@@ -230,19 +229,6 @@ fn classify_internode_missing_error(error: &InternodeHttpError) -> Option<DiskEr
     None
 }
 
-fn internode_write_error_is_retryable(error: &InternodeHttpError) -> bool {
-    error.kind().is_retryable()
-        || (matches!(error.kind(), InternodeHttpErrorKind::HttpStatus(status) if status.as_u16() == 409)
-            && error.context().operation() == Some(INTERNODE_OPERATION_PUT_FILE_STREAM))
-}
-
-fn io_error_contains_retryable_internode_write(error: &io::Error) -> bool {
-    error
-        .get_ref()
-        .and_then(|source| source.downcast_ref::<InternodeHttpError>())
-        .is_some_and(internode_write_error_is_retryable)
-}
-
 /// Wrap a terminal shard-read failure without changing its typed
 /// classification.  Timeout-like disk errors retain `TimedOut`; other errors
 /// retain their inner I/O kind or use `Other` when no more specific kind exists.
@@ -350,7 +336,10 @@ impl DiskError {
 
     pub fn is_retryable_internode_write_failure(&self) -> bool {
         match self {
-            DiskError::Io(io_error) => io_error_contains_retryable_internode_write(io_error),
+            DiskError::Io(io_error) => io_error
+                .get_ref()
+                .and_then(|source| source.downcast_ref::<InternodeHttpError>())
+                .is_some_and(|err| err.kind().is_retryable()),
             _ => false,
         }
     }
@@ -1251,68 +1240,6 @@ mod tests {
         assert!(!DiskError::FileNotFound.is_internode_http_status(429));
     }
 
-    #[test]
-    fn test_put_file_server_epoch_conflict_is_retryable_write_failure() {
-        let conflict = DiskError::from(rustfs_rio::new_test_internode_http_io_error(
-            rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::CONFLICT),
-        ));
-        let bad_request = DiskError::from(rustfs_rio::new_test_internode_http_io_error(
-            rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::BAD_REQUEST),
-        ));
-
-        assert!(conflict.is_retryable_internode_write_failure());
-        assert!(!bad_request.is_retryable_internode_write_failure());
-    }
-
-    #[tokio::test]
-    async fn read_stream_conflict_is_not_a_retryable_put_file_failure() {
-        use tokio::io::{AsyncReadExt, AsyncWriteExt};
-
-        tokio::time::timeout(std::time::Duration::from_secs(5), async {
-            let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
-                .await
-                .expect("bind isolated HTTP fixture");
-            let address = listener.local_addr().expect("fixture address");
-            let server = tokio::spawn(async move {
-                let (mut stream, _) = listener.accept().await.expect("accept read request");
-                let mut request = [0_u8; 4096];
-                let mut read = 0;
-                loop {
-                    let count = stream.read(&mut request[read..]).await.expect("read HTTP request");
-                    assert!(count > 0, "request ended before its complete headers");
-                    read += count;
-                    if request[..read].windows(4).any(|bytes| bytes == b"\r\n\r\n") {
-                        break;
-                    }
-                    assert!(read < request.len(), "fixture request headers exceed their budget");
-                }
-                stream
-                    .write_all(b"HTTP/1.1 409 Conflict\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
-                    .await
-                    .expect("send typed conflict response");
-            });
-            let error = match rustfs_rio::HttpReader::new(
-                format!("http://{address}/rustfs/rpc/read_file_stream"),
-                http::Method::GET,
-                http::HeaderMap::new(),
-                None,
-            )
-            .await
-            {
-                Ok(_) => panic!("HTTP 409 must fail the read"),
-                Err(error) => DiskError::from(error),
-            };
-            server.await.expect("fixture task should complete");
-            assert!(error.is_internode_http_status(409));
-            assert!(
-                !error.is_retryable_internode_write_failure(),
-                "read-operation 409 must not trigger put-file retry"
-            );
-        })
-        .await
-        .expect("isolated read-conflict test must finish within its budget");
-    }
-
     #[test]
     fn test_internode_missing_errors_preserve_disk_error_types() {
         let file_missing = DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error());

crates/ecstore/src/disk/local.rs

@@ -3148,10 +3148,9 @@ impl LocalIoBackend for StdBackend {
                 direct_read_copy_fault_delta: MmapPageFaultDelta,
                 blocking_task_duration: StdDuration,
                 used_direct_io: bool,
-                /// The descriptor and size snapshot opened by THIS call (None on a
-                /// cache hit), handed back so the async caller can index it in the
-                /// fd cache.
-                opened_fd: Option<Arc<FdCacheEntry>>,
+                /// The descriptor opened by THIS call (None on a cache hit), handed
+                /// back so the async caller can index it in the fd cache.
+                opened_fd: Option<Arc<std::fs::File>>,
             }
 
             enum MmapCopyReadError {
@@ -3198,12 +3197,12 @@ impl LocalIoBackend for StdBackend {
                 (cache, key, gen_at_open)
             });
             #[cfg(target_os = "linux")]
-            let cached_fd: Option<Arc<FdCacheEntry>> = match &fd_lookup {
+            let cached_fd: Option<Arc<std::fs::File>> = match &fd_lookup {
                 Some((cache, key, _)) => cache.get(key).await,
                 None => None,
             };
             #[cfg(not(target_os = "linux"))]
-            let cached_fd: Option<Arc<FdCacheEntry>> = None;
+            let cached_fd: Option<Arc<std::fs::File>> = None;
 
             let blocking_wait_start = metrics_enabled.then(std::time::Instant::now);
             let read_result = tokio::task::spawn_blocking(move || {
@@ -3225,15 +3224,8 @@ impl LocalIoBackend for StdBackend {
                 // the read below is positioned (mmap offset argument / `read_exact_at`)
                 // and never depends on the descriptor's current offset. `cached_fd` being
                 // None also marks this call as a miss for the cache-insert side-channel.
-                // The cached length is the metadata snapshot captured at open time;
-                // all in-place/replacement writers invalidate this entry before
-                // publishing a mutation, so cache hits avoid a redundant fstat.
-                let (file, cached_len, access_check_duration) = if let Some(cached) = cached_fd.as_ref() {
-                    (
-                        cached.file.as_ref().try_clone().map_err(DiskError::from)?,
-                        Some(cached.len),
-                        StdDuration::ZERO,
-                    )
+                let (file, access_check_duration) = if let Some(cached) = cached_fd.as_ref() {
+                    (cached.as_ref().try_clone().map_err(DiskError::from)?, StdDuration::ZERO)
                 } else {
                     // Measure the volume access probe only — the part-path resolution
                     // above is accounted in `path_resolve_duration` (rustfs/backlog#1801).
@@ -3244,27 +3236,20 @@ impl LocalIoBackend for StdBackend {
                             .map_err(|e| DiskError::from(to_access_error(e, DiskError::VolumeAccessDenied)))?;
                     }
                     let access_check_duration = access_check_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
-                    (std::fs::File::open(&file_path).map_err(DiskError::from)?, None, access_check_duration)
+                    (std::fs::File::open(&file_path).map_err(DiskError::from)?, access_check_duration)
                 };
                 let file_open_duration = file_open_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
 
-                let (metadata_len, metadata_lookup_duration) = if let Some(len) = cached_len {
-                    // Reuse the open-time metadata snapshot on a cache hit. The
-                    // generation fence and mutation invalidation keep this value
-                    // tied to the inode held by `file`.
-                    (len, StdDuration::ZERO)
-                } else {
-                    let metadata_lookup_start = metrics_enabled.then(StdInstant::now);
-                    let meta = file.metadata().map_err(DiskError::from)?;
-                    let duration = metadata_lookup_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
-                    (meta.len(), duration)
-                };
+                let metadata_lookup_start = metrics_enabled.then(StdInstant::now);
+                // On a cache hit this fstats the cached descriptor — the inode it was
+                // opened against, which invalidation keeps current for live entries. EC
+                // shards are fixed-length, so a still-cached pre-heal length is benign.
+                let meta = file.metadata().map_err(DiskError::from)?;
+                let metadata_lookup_duration = metadata_lookup_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
 
                 let metadata_validate_start = metrics_enabled.then(StdInstant::now);
-                if metadata_len < end_offset_u64 {
-                    return Err(MmapCopyReadError::OutOfBounds {
-                        actual_size: metadata_len,
-                    });
+                if meta.len() < end_offset_u64 {
+                    return Err(MmapCopyReadError::OutOfBounds { actual_size: meta.len() });
                 }
                 let metadata_validate_duration =
                     metadata_validate_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
@@ -3410,14 +3395,9 @@ impl LocalIoBackend for StdBackend {
                 // Arc; `cached_fd.is_none()` is true exactly when this call did the open.
                 // Non-Linux has no fd cache, so skip the Arc allocation there.
                 #[cfg(target_os = "linux")]
-                let opened_fd: Option<Arc<FdCacheEntry>> = cached_fd.is_none().then(|| {
-                    Arc::new(FdCacheEntry {
-                        file: Arc::new(file),
-                        len: metadata_len,
-                    })
-                });
+                let opened_fd: Option<Arc<std::fs::File>> = cached_fd.is_none().then(|| Arc::new(file));
                 #[cfg(not(target_os = "linux"))]
-                let opened_fd: Option<Arc<FdCacheEntry>> = None;
+                let opened_fd: Option<Arc<std::fs::File>> = None;
 
                 Ok::<MmapCopyReadResult, MmapCopyReadError>(MmapCopyReadResult {
                     bytes,
@@ -3540,7 +3520,7 @@ impl LocalIoBackend for StdBackend {
                     }
                 }
             }
-            // Index the freshly opened descriptor and metadata snapshot for future cache hits
+            // Index the freshly opened descriptor for future cache hits
             // (rustfs/backlog#1801). `insert_if_fresh` refuses to cache if an
             // invalidation (heal/delete/rename) bumped the generation between the
             // open snapshot and now, so a stale pre-mutation inode is never served
@@ -3892,18 +3872,6 @@ struct FdKey {
     direct: bool,
 }
 
-/// Descriptor and immutable size snapshot retained for one cached shard inode.
-///
-/// The generation fence and explicit mutation invalidation keep the snapshot
-/// tied to the inode held by `file`, allowing cache hits to avoid a repeated
-/// metadata syscall without weakening replacement/heal semantics.
-struct FdCacheEntry {
-    /// An independently cloneable descriptor for the immutable shard inode.
-    file: Arc<std::fs::File>,
-    /// File length captured together with the descriptor.
-    len: u64,
-}
-
 /// Per-disk cache of open descriptors for io_uring reads (backlog#1145).
 ///
 /// Why this exists: `pread_uring` opened the file on the blocking pool for every
@@ -3933,7 +3901,7 @@ struct FdCacheEntry {
 /// the descriptor once no in-flight read still holds it.
 #[cfg(target_os = "linux")]
 struct FdCache {
-    cache: moka::future::Cache<FdKey, Arc<FdCacheEntry>>,
+    cache: moka::future::Cache<FdKey, Arc<std::fs::File>>,
     /// Bumped by every invalidation. A miss-path open snapshots this before it
     /// opens and refuses to insert if it moved, so an fd opened before a
     /// heal/delete commit can never be resurrected into the cache after the
@@ -3963,7 +3931,7 @@ impl FdCache {
         }
     }
 
-    async fn get(&self, key: &FdKey) -> Option<Arc<FdCacheEntry>> {
+    async fn get(&self, key: &FdKey) -> Option<Arc<std::fs::File>> {
         self.cache.get(key).await
     }
 
@@ -3978,11 +3946,11 @@ impl FdCache {
     /// open bumped the generation, so a stale pre-heal/pre-delete inode is never
     /// cached. The post-insert re-check closes the tiny window where an
     /// invalidate races the insert itself, by removing the entry we just added.
-    async fn insert_if_fresh(&self, key: FdKey, entry: Arc<FdCacheEntry>, gen_at_open: u64) {
+    async fn insert_if_fresh(&self, key: FdKey, file: Arc<std::fs::File>, gen_at_open: u64) {
         if self.generation.load(Ordering::Acquire) != gen_at_open {
             return;
         }
-        self.cache.insert(key.clone(), entry).await;
+        self.cache.insert(key.clone(), file).await;
         if self.generation.load(Ordering::Acquire) != gen_at_open {
             self.cache.invalidate(&key).await;
         }
@@ -4018,7 +3986,7 @@ impl FdCache {
         self.generation.fetch_add(1, Ordering::AcqRel);
         let volume = volume.to_owned();
         let prefix = prefix.trim_end_matches('/').to_owned();
-        let matches = move |k: &FdKey, _: &Arc<FdCacheEntry>| {
+        let matches = move |k: &FdKey, _: &Arc<std::fs::File>| {
             k.volume == volume && (k.path == prefix || k.path.strip_prefix(&prefix).is_some_and(|r| r.starts_with('/')))
         };
         if self.cache.invalidate_entries_if(matches).is_err() {
@@ -4034,7 +4002,7 @@ impl FdCache {
     fn invalidate_volume(&self, volume: &str) {
         self.generation.fetch_add(1, Ordering::AcqRel);
         let volume = volume.to_owned();
-        let matches = move |k: &FdKey, _: &Arc<FdCacheEntry>| k.volume == volume;
+        let matches = move |k: &FdKey, _: &Arc<std::fs::File>| k.volume == volume;
         if self.cache.invalidate_entries_if(matches).is_err() {
             self.cache.invalidate_all();
         }
@@ -4052,8 +4020,7 @@ impl FdCache {
     /// tests that drive the cache directly.
     #[cfg(test)]
     async fn insert(&self, key: FdKey, file: Arc<std::fs::File>) {
-        let len = file.metadata().map(|metadata| metadata.len()).unwrap_or_default();
-        self.cache.insert(key, Arc::new(FdCacheEntry { file, len })).await;
+        self.cache.insert(key, file).await;
     }
 
     #[cfg(test)]
@@ -4432,12 +4399,7 @@ impl UringBackend {
         };
 
         let file = match cached {
-            Some(entry) => {
-                if entry.len < u64::try_from(end_offset).map_err(|_| DiskError::FileCorrupt)? {
-                    return Err(DiskError::FileCorrupt);
-                }
-                Arc::clone(&entry.file)
-            }
+            Some(file) => file,
             None => {
                 // Snapshot the cache generation BEFORE opening (rustfs/backlog#1176):
                 // if a heal/delete invalidation runs while this open is in flight,
@@ -4447,7 +4409,7 @@ impl UringBackend {
                 let root = self.root.clone();
                 let volume_owned = volume.to_owned();
                 let path_owned = path.to_owned();
-                let (file, len) = tokio::task::spawn_blocking(move || -> Result<(std::fs::File, u64)> {
+                let file = tokio::task::spawn_blocking(move || -> Result<std::fs::File> {
                     let file_path = resolve_uring_object_path(&root, &volume_owned, &path_owned)?;
                     let file = std::fs::File::open(&file_path).map_err(DiskError::from)?;
                     let meta = file.metadata().map_err(DiskError::from)?;
@@ -4455,22 +4417,30 @@ impl UringBackend {
                     if meta.len() < end_offset_u64 {
                         return Err(DiskError::FileCorrupt);
                     }
-                    Ok((file, meta.len()))
+                    Ok(file)
                 })
                 .await
                 .map_err(|e| DiskError::other(format!("uring pread join error: {e}")))??;
-                let file = Arc::new(FdCacheEntry {
-                    file: Arc::new(file),
-                    len,
-                });
+                let file = Arc::new(file);
                 if let (Some((cache, key)), Some(gen_at_open)) = (cache_entry, gen_at_open) {
                     cache.insert_if_fresh(key, Arc::clone(&file), gen_at_open).await;
                 }
-                file.file.clone()
+                file
             }
         };
 
         if length == 0 {
+            // Parity with StdBackend and the miss path (rustfs/backlog#1173): a
+            // zero-length read still rejects an offset past EOF. The miss path
+            // validated `meta.len() < end_offset` (end_offset == offset here), but
+            // a cache hit skipped it — so fstat the descriptor and match. This is
+            // a rare path (callers do not issue zero-length reads), so the one
+            // extra fstat is negligible.
+            match file.metadata() {
+                Ok(meta) if offset_u64 > meta.len() => return Err(DiskError::FileCorrupt),
+                Ok(_) => {}
+                Err(e) => return Err(DiskError::from(e)),
+            }
             return Ok(Bytes::new());
         }
 
@@ -6444,9 +6414,6 @@ impl LocalDisk {
                     .abort_reserved_version_delete(object_dir, rollback_dir, volume, path, "delete_versions_commit_intent", err)
                     .await);
             }
-            if should_fail_after_delete_commit(self.root.as_path(), path) {
-                return Err(DiskError::Unexpected);
-            }
             return Ok(());
         }
 
@@ -6517,10 +6484,6 @@ impl LocalDisk {
                 .await);
         }
 
-        if should_fail_after_delete_commit(self.root.as_path(), path) {
-            return Err(DiskError::Unexpected);
-        }
-
         Ok(())
     }
 
@@ -7099,9 +7062,6 @@ impl LocalDisk {
                                 .await?
                         {
                             meta.name.push_str(SLASH_SEPARATOR);
-                            // Conservative listings verify physical prefixes. Never-versioned
-                            // buckets use the bounded fast path and reclaim residue after an
-                            // exact recursive listing proves that prefix empty.
                             if opts.recursive
                                 || opts.incl_deleted
                                 || opts.skip_hidden_prefix_check
@@ -17629,7 +17589,7 @@ mod test {
     }
 
     #[tokio::test]
-    async fn test_scan_dir_nonrecursive_fast_path_preserves_probe_bound() {
+    async fn test_scan_dir_nonrecursive_visible_prefix_probe_cost() {
         use rustfs_filemeta::MetacacheReader;
         use tempfile::tempdir;
 
@@ -17661,10 +17621,6 @@ mod test {
             expected_names.push(format!("{prefix}/"));
         }
 
-        fs::create_dir_all(bucket_dir.join("stale/nested/residue"))
-            .await
-            .expect("stale backing directory should be created");
-
         async fn scan_prefixes(disk: &LocalDisk, bucket: &str, skip_hidden_prefix_check: bool) -> (Vec<String>, usize) {
             let probe_count = Arc::new(AtomicUsize::new(0));
             let (reader, mut writer) = tokio::io::duplex(64 * 1024);
@@ -17707,11 +17663,8 @@ mod test {
         let (fast_path_names, fast_path_probes) = scan_prefixes(&disk, bucket, true).await;
 
         assert_eq!(conservative_names, expected_names);
-        let mut expected_fast_path_names = expected_names.clone();
-        expected_fast_path_names.push("stale/".to_owned());
-        assert_eq!(fast_path_names, expected_fast_path_names);
-        let expected_probes = PREFIX_COUNT * 3 + 3;
-        assert_eq!(conservative_probes, expected_probes);
+        assert_eq!(fast_path_names, expected_names);
+        assert_eq!(conservative_probes, PREFIX_COUNT * 3);
         assert_eq!(fast_path_probes, 0);
     }
 
@@ -21454,10 +21407,11 @@ mod test {
 
     /// Zero-length read bounds parity on the cache-HIT path (backlog#1173/#1180).
     /// A `length == 0` read past EOF must be rejected identically whether the
-    /// descriptor is freshly opened (miss path) or served from the cache. Seeds
-    /// the cache with a normal read so the zero-length reads reuse the same
-    /// open-time size snapshot, then pins that UringBackend and StdBackend agree
-    /// on every case.
+    /// descriptor is freshly opened (miss path) or served from the cache: the
+    /// cache-hit branch fstats the descriptor to reproduce the miss path's
+    /// `offset > len` check instead of returning empty unconditionally. Seeds
+    /// the cache with a normal read so the zero-length reads are hits, then pins
+    /// that UringBackend and StdBackend agree on every case.
     #[cfg(target_os = "linux")]
     #[tokio::test(flavor = "multi_thread")]
     async fn uring_zero_length_read_bounds_match_std_on_cache_hit() {

crates/ecstore/src/disk/mod.rs

@@ -677,6 +677,23 @@ impl DiskAPI for Disk {
 }
 
 impl Disk {
+    pub(crate) async fn delete_with_scanner_publication_lease(
+        &self,
+        volume: &str,
+        path: &str,
+        opts: DeleteOptions,
+        scanner_publication_lease_token: Option<Uuid>,
+    ) -> Result<()> {
+        match self {
+            Disk::Local(local_disk) => local_disk.delete(volume, path, opts).await,
+            Disk::Remote(remote_disk) => {
+                remote_disk
+                    .delete_with_scanner_publication_lease(volume, path, opts, scanner_publication_lease_token)
+                    .await
+            }
+        }
+    }
+
     pub async fn delete_with_scanner_publication_lease_and_guard(
         &self,
         volume: &str,