diff --git a/.config/nextest.toml b/.config/nextest.toml index f4e8bd200..4322f7d28 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -1,5 +1,7 @@ # nextest configuration for RustFS. # +experimental = ["setup-scripts"] + # Serialize the ecstore tests that share the process-wide disk registry or # exercise a multi-disk commit handoff across nextest process boundaries. # @@ -44,7 +46,17 @@ e2e-reliability = { max-threads = 1 } e2e-inline-boundaries = { max-threads = 1 } e2e-cluster-nightly = { max-threads = 1 } +# These exact regression scenarios build deep async storage futures that exceed +# libtest's 2 MiB spawned-thread stack on Linux. Give only their test processes +# the same 32 MiB stack already used by the crate's dedicated large-stack tests. +[scripts.setup.ecstore-large-stack] +command = ['sh', '-c', 'echo RUST_MIN_STACK=33554432 >> "$NEXTEST_ENV"'] + # --- default profile (local): serialize the flaky groups, never retry -------- +[[profile.default.scripts]] +filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)' +setup = 'ecstore-large-stack' + [[profile.default.overrides]] filter = 'package(rustfs-ecstore) & (test(concurrent_resend_same_part_commits_one_generation) | test(concurrent_config_writes_from_separate_nodes_do_not_lose_writes) | test(/^store::bucket::tests::bucket_delete_(mark_delete|purge_removes|default_s3_delete)/))' test-group = 'ecstore-serial-flaky' @@ -134,6 +146,10 @@ fail-fast = false # marker is the observable signal the flake policy is built around. path = "junit.xml" +[[profile.ci.scripts]] +filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)' +setup = 'ecstore-large-stack' + # =========================================================================== # QUARANTINE — flaky tests granted retries = 2 under the ci profile ONLY. # diff --git a/crates/e2e_test/src/quota_test.rs b/crates/e2e_test/src/quota_test.rs index 8e5059e7e..d6f5b6e8e 100644 --- a/crates/e2e_test/src/quota_test.rs +++ b/crates/e2e_test/src/quota_test.rs @@ -155,6 +155,22 @@ impl QuotaTestEnv { Ok(stats.get("current_usage").and_then(|v| v.as_u64()).unwrap_or(0)) } + async fn wait_for_bucket_usage(&self, expected: u64) -> Result> { + let convergence = async { + loop { + let usage = self.get_bucket_usage().await?; + if usage == expected { + return Ok::>(usage); + } + sleep(Duration::from_millis(100)).await; + } + }; + match timeout(Duration::from_secs(30), convergence).await { + Ok(result) => result, + Err(_) => Err(format!("bucket usage did not converge to {expected} bytes within 30 seconds").into()), + } + } + pub async fn set_bucket_quota_for( &self, bucket: &str, @@ -444,8 +460,8 @@ mod integration_tests { .send() .await?; - // Check updated usage - let updated_usage = env.get_bucket_usage().await?; + // A completed scanner generation releases the conservative quota floor after a delete. + let updated_usage = env.wait_for_bucket_usage(256 * 1024).await?; assert_eq!(updated_usage, 256 * 1024); env.cleanup_bucket().await?; diff --git a/rustfs/src/auth.rs b/rustfs/src/auth.rs index 5b019de18..73c869c6c 100644 --- a/rustfs/src/auth.rs +++ b/rustfs/src/auth.rs @@ -430,7 +430,7 @@ pub async fn check_key_valid_with_context( reason = "account_disabled", "Access key validation rejected" ); - return Err(s3_error!(InvalidRequest, "ErrAccessKeyDisabled")); + return Err(s3_error!(InvalidAccessKeyId, "check key failed")); } warn!(