Compare commits

...
Author SHA1 Message Date
housemeandheihutu 59de0de6b3 fix(scanner): retain cleanup scope after cancellation
Co-Authored-By: heihutu <[email protected]>
2026-08-30 03:10:32 +08:00
housemeandheihutu 52961d2726 fix(scanner): fence observed cleanup mutations
Co-Authored-By: heihutu <[email protected]>
2026-08-30 03:04:42 +08:00
houseme 290deb8665 Revert "fix(scanner): fence backup and cleanup mutations"
This reverts commit 2dcc064a5a.
2026-08-30 03:02:03 +08:00
housemeandheihutu 2dcc064a5a fix(scanner): fence backup and cleanup mutations
Carry storage-owned publication scopes through backup CAS writes and observed cleanup deletes so cancellation and absolute lease deadlines retain the same safety boundary.

Co-Authored-By: heihutu <[email protected]>
2026-08-30 03:01:11 +08:00
housemeandheihutu 926db4d3c6 chore(storage): satisfy publication scope lint
Co-Authored-By: heihutu <[email protected]>
2026-08-30 02:54:03 +08:00
housemeandheihutu 6b9976e91c fix(scanner): fence backup publication scope
Co-Authored-By: heihutu <[email protected]>
2026-08-30 02:51:47 +08:00
houseme 727c241a34 fix(storage): retain publication scope through fanout 2026-08-30 02:28:30 +08:00
housemeandheihutu 76d2eddbf9 fix(scanner): own fenced publication mutations
Remove the static lease budget rejection and carry an effective absolute deadline into storage-owned publication scopes. Keep remote grants until a scope reaches a safe terminal state.

Co-Authored-By: heihutu <[email protected]>
2026-08-30 02:02:35 +08:00
housemeandheihutu 2947d22cec fix(storage): retain publication guard through fenced mutations
Co-Authored-By: heihutu <[email protected]>
2026-08-30 01:43:39 +08:00
housemeandheihutu bf39bbc3b4 test(scanner): cover publication deadline and lease fencing
Co-Authored-By: heihutu <[email protected]>
2026-08-30 01:34:13 +08:00
housemeandheihutu af8bbae644 feat(ecstore): add scanner publication commit scope
Co-Authored-By: heihutu <[email protected]>
2026-08-30 01:32:56 +08:00
houseme bb2452789a fix(scanner): use typed activity timeout error 2026-08-30 00:56:03 +08:00
housemeandheihutu 923f86f2aa fix(scanner): fence legacy usage floor fallback
Co-Authored-By: heihutu <[email protected]>
2026-08-30 00:56:03 +08:00
housemeandheihutu 2004001d47 fix(scanner): reuse recovered usage baseline for publication
Co-Authored-By: heihutu <[email protected]>
2026-08-30 00:56:03 +08:00
housemeandheihutu 3a4a6b0161 fix(scanner): preserve publication lease defer reasons
Keep lease expiry and release failures distinct from activity baseline failures so scanner freshness metrics and cycle outcomes identify the publication barrier that blocked progress. Preserve fail-closed behavior.

Co-Authored-By: heihutu <[email protected]>
2026-08-30 00:56:03 +08:00
housemeandheihutu b7074129ee fix(scanner): recover usage floor from fenced backups
Co-Authored-By: heihutu <[email protected]>
2026-08-30 00:56:03 +08:00
housemeandheihutu 29ed499773 fix(scanner): classify publication lease deferrals
Distinguish persistence budget and lease deadline deferrals from unavailable activity baselines, and ensure lease-gate deferrals update usage metrics. Keep the fixed lease gate fail-closed while storage-owned commit scope work remains pending.

Co-Authored-By: heihutu <[email protected]>
2026-08-30 00:56:03 +08:00
houseme 402ac43207 fix(scanner): retain post-scan observations
Preserve a complete scanner walk as a non-converged observation when the final activity probe is unavailable. Advance the cycle as partial without acknowledging dirty usage.\n\nCo-Authored-By: heihutu <[email protected]>
2026-08-30 00:56:03 +08:00
housemeandheihutu f722604f02 fix(scanner): retry transient activity probes
Retry one failed scanner activity probe after a bounded reconnect when the failure is transport-like or timed out. Keep protocol and response validation failures fail-closed.

Co-Authored-By: heihutu <[email protected]>
2026-08-30 00:56:03 +08:00
housemeandheihutu d1b962372a fix(scanner): recover usage floor from fenced backups
Co-Authored-By: heihutu <[email protected]>
2026-08-30 00:56:03 +08:00
housemeandheihutu 887157e560 test(scanner): cover bucket drive guard lifecycle
Co-Authored-By: heihutu <[email protected]>
2026-08-30 00:56:03 +08:00
22 changed files with 2026 additions and 142 deletions
+4 -3
View File
@@ -416,9 +416,10 @@ pub mod object {
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver,
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, StreamConsumer, get_object_body_cache_plaintext_len,
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
ScannerPublicationCommitState, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
unregister_object_mutation_hook,
};
pub use crate::store::{
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
@@ -725,7 +725,7 @@ impl PeerRestClient {
/// never take it offline no matter what its message says. The substring
/// fallback only covers failures that exist purely as text, such as the
/// dial errors `get_client` wraps.
fn is_network_like_error(err: &Error) -> bool {
pub(crate) fn is_network_like_error(err: &Error) -> bool {
if let Error::Io(io_err) = err
&& let Some(status) = embedded_tonic_status(io_err)
{
+143
View File
@@ -250,6 +250,37 @@ pub(crate) trait DiskStoreRenameDataExt {
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp>;
async fn rename_data_borrowed_with_guard(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
external_guard: Option<Arc<dyn Send + Sync>>,
) -> Result<RenameDataResp> {
let _ = external_guard;
self.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
.await
}
}
/// Run a mutation in an owned task when a caller supplied publication guard.
/// RPC cancellation drops only the waiter; the mutation owner keeps the guard
/// until its operation has returned, including any detached blocking syscall.
async fn run_owned_mutation<T, F, Fut>(external_guard: Option<Arc<dyn Send + Sync>>, operation: F) -> Result<T>
where
T: Send + 'static,
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = Result<T>> + Send + 'static,
{
tokio::spawn(async move {
let _external_guard = external_guard;
operation().await
})
.await
.map_err(|err| Error::other(format!("owned mutation task failed: {err}")))?
}
impl DiskStoreRenameDataExt for LocalDiskWrapper {
@@ -273,6 +304,49 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
)
.await
}
async fn rename_data_borrowed_with_guard(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
external_guard: Option<Arc<dyn Send + Sync>>,
) -> Result<RenameDataResp> {
let operation = self.clone();
let src_volume = src_volume.to_owned();
let src_path = src_path.to_owned();
let fi = fi.clone();
let dst_volume = dst_volume.to_owned();
let dst_path = dst_path.to_owned();
let timeout_duration = if external_guard.is_some() {
// A fenced mutation owns the publication guard until the storage
// operation returns. Timing out this waiter would cancel the
// LocalDisk future while a spawn_blocking namespace syscall could
// still be committing, reopening the movement window. The caller
// may drop its waiter; the owned task drains the mutation.
Duration::ZERO
} else {
get_max_timeout_duration()
};
run_owned_mutation(external_guard, move || async move {
operation
.track_disk_health_mutation(
"rename_data",
DiskMetricMutation::Write,
|| async {
operation
.disk
.rename_data_borrowed(&src_volume, &src_path, &fi, &dst_volume, &dst_path)
.await
},
timeout_duration,
)
.await
})
.await
}
}
pub fn get_drive_walkdir_timeout() -> Duration {
@@ -1097,6 +1171,37 @@ impl LocalDiskWrapper {
)
}
/// Run a delete under an owned coordinator task when a publication guard
/// is present. This keeps the guard alive if the RPC waiter is cancelled
/// while the local namespace mutation is still in progress.
pub(crate) async fn delete_with_publication_guard(
&self,
volume: &str,
path: &str,
options: DeleteOptions,
external_guard: Option<Arc<dyn Send + Sync>>,
) -> Result<()> {
let operation = self.clone();
let volume = volume.to_owned();
let path = path.to_owned();
let timeout_duration = if external_guard.is_some() {
Duration::ZERO
} else {
get_max_timeout_duration()
};
run_owned_mutation(external_guard, move || async move {
operation
.track_disk_health_mutation(
"delete",
DiskMetricMutation::Delete,
|| async { operation.disk.delete(&volume, &path, options).await },
timeout_duration,
)
.await
})
.await
}
pub(crate) fn new_with_reconnect_state(
disk: Arc<LocalDisk>,
health_check: bool,
@@ -2247,6 +2352,44 @@ mod tests {
};
use tokio::io::AsyncWrite;
struct DropProbe(Arc<std::sync::atomic::AtomicUsize>);
impl Drop for DropProbe {
fn drop(&mut self) {
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
}
}
#[tokio::test]
async fn owned_mutation_keeps_publication_guard_after_waiter_cancellation() {
let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let guard: Arc<dyn Send + Sync> = Arc::new(DropProbe(Arc::clone(&drops)));
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
let (finished_tx, finished_rx) = tokio::sync::oneshot::channel();
let waiter = tokio::spawn(run_owned_mutation(Some(guard), move || async move {
started_tx.send(()).expect("mutation should signal start");
release_rx.await.expect("mutation should be released");
finished_tx.send(()).expect("mutation should signal completion");
Ok::<_, Error>(())
}));
started_rx.await.expect("mutation owner should start");
waiter.abort();
assert_eq!(drops.load(std::sync::atomic::Ordering::SeqCst), 0);
release_tx.send(()).expect("mutation owner should still be alive");
finished_rx.await.expect("mutation owner should finish");
tokio::time::timeout(Duration::from_secs(1), async {
while drops.load(std::sync::atomic::Ordering::SeqCst) == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("publication guard should be released after mutation completion");
}
struct PendingWriter;
#[test]
+46 -1
View File
@@ -694,6 +694,28 @@ impl Disk {
}
}
pub async fn delete_with_scanner_publication_lease_and_guard(
&self,
volume: &str,
path: &str,
opts: DeleteOptions,
scanner_publication_lease_token: Option<Uuid>,
external_guard: Option<Arc<dyn Send + Sync>>,
) -> Result<()> {
match self {
Disk::Local(local_disk) => {
local_disk
.delete_with_publication_guard(volume, path, opts, external_guard)
.await
}
Disk::Remote(remote_disk) => {
remote_disk
.delete_with_scanner_publication_lease(volume, path, opts, scanner_publication_lease_token)
.await
}
}
}
pub(crate) async fn rename_data_borrowed(
&self,
src_volume: &str,
@@ -714,11 +736,34 @@ impl Disk {
dst_volume: &str,
dst_path: &str,
scanner_publication_lease_token: Option<Uuid>,
) -> Result<RenameDataResp> {
self.rename_data_borrowed_with_fence_and_guard(
src_volume,
src_path,
fi,
dst_volume,
dst_path,
scanner_publication_lease_token,
None,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub async fn rename_data_borrowed_with_fence_and_guard(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
scanner_publication_lease_token: Option<Uuid>,
external_guard: Option<Arc<dyn Send + Sync>>,
) -> Result<RenameDataResp> {
match self {
Disk::Local(local_disk) => {
local_disk
.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
.rename_data_borrowed_with_guard(src_volume, src_path, fi, dst_volume, dst_path, external_guard)
.await
}
Disk::Remote(remote_disk) => {
+323
View File
@@ -19,6 +19,9 @@ use crate::storage_api_contracts::{
HTTPPreconditions, ObjectLockRetentionOptions, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState,
},
};
use std::sync::atomic::{AtomicU8, Ordering};
use tokio::sync::{Mutex, Notify, OwnedRwLockReadGuard};
use tokio_util::sync::CancellationToken;
#[derive(Clone)]
pub struct NamespaceLockFence {
@@ -347,6 +350,320 @@ impl QuotaAdmission {
}
}
const SCANNER_PUBLICATION_SCOPE_ADMITTED: u8 = 0;
const SCANNER_PUBLICATION_SCOPE_IN_FLIGHT: u8 = 1;
const SCANNER_PUBLICATION_SCOPE_COMMITTED: u8 = 2;
const SCANNER_PUBLICATION_SCOPE_ABORTED_BEFORE_COMMIT: u8 = 3;
const SCANNER_PUBLICATION_SCOPE_INDETERMINATE: u8 = 4;
/// The terminal result of a storage-owned scanner publication mutation.
///
/// This state is deliberately not serialized. It is the ownership hand-off
/// between the scanner coordinator and the storage mutation task, so a
/// detached rename/cleanup task can retain the movement permit until it has
/// reported a definitive result.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerPublicationCommitState {
Admitted,
InFlight,
Committed,
AbortedBeforeCommit,
Indeterminate,
}
impl ScannerPublicationCommitState {
fn as_u8(self) -> u8 {
match self {
Self::Admitted => SCANNER_PUBLICATION_SCOPE_ADMITTED,
Self::InFlight => SCANNER_PUBLICATION_SCOPE_IN_FLIGHT,
Self::Committed => SCANNER_PUBLICATION_SCOPE_COMMITTED,
Self::AbortedBeforeCommit => SCANNER_PUBLICATION_SCOPE_ABORTED_BEFORE_COMMIT,
Self::Indeterminate => SCANNER_PUBLICATION_SCOPE_INDETERMINATE,
}
}
fn from_u8(value: u8) -> Self {
match value {
SCANNER_PUBLICATION_SCOPE_IN_FLIGHT => Self::InFlight,
SCANNER_PUBLICATION_SCOPE_COMMITTED => Self::Committed,
SCANNER_PUBLICATION_SCOPE_ABORTED_BEFORE_COMMIT => Self::AbortedBeforeCommit,
SCANNER_PUBLICATION_SCOPE_INDETERMINATE => Self::Indeterminate,
_ => Self::Admitted,
}
}
/// A caller may release its remote lease only after one of these states.
/// `Indeterminate` is intentionally excluded: the mutation may have
/// committed after cancellation or a transport failure.
pub fn permits_lease_release(self) -> bool {
matches!(self, Self::Committed | Self::AbortedBeforeCommit)
}
}
/// Why a storage-owned publication scope could not start its mutation.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerPublicationCommitStartError {
Cancelled,
DeadlineExceeded,
AlreadyStarted,
Terminal,
}
struct ScannerPublicationCommitScopeInner {
expected_movement_epoch: u64,
safe_deadline: tokio::time::Instant,
remote_lease_tokens: Arc<[Uuid]>,
cancellation: CancellationToken,
state: AtomicU8,
completed: Notify,
/// The permit is storage-owned rather than borrowed from the scanner
/// future. A detached mutation task keeps the scope alive and therefore
/// keeps this guard alive until it reports a terminal state.
movement_permit: Mutex<Option<OwnedRwLockReadGuard<()>>>,
lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
}
/// Storage-owned ownership scope for one fenced scanner metadata mutation.
///
/// The scope is an in-memory capability. It is intentionally carried through
/// [`ObjectOptions`] as a hidden field and never participates in serde, object
/// metadata, RPC wire structures, or on-disk formats.
#[derive(Clone)]
pub struct ScannerPublicationCommitScope {
inner: Arc<ScannerPublicationCommitScopeInner>,
}
/// RAII fallback for storage paths that return before their commit closure
/// takes ownership. An in-flight scope is never guessed to be aborted: it is
/// marked indeterminate so remote lease release remains blocked.
pub(crate) struct ScannerPublicationCommitScopeGuard {
scope: Option<ScannerPublicationCommitScope>,
}
impl ScannerPublicationCommitScopeGuard {
pub(crate) fn new(scope: ScannerPublicationCommitScope) -> Self {
Self { scope: Some(scope) }
}
pub(crate) fn disarm(&mut self) {
self.scope = None;
}
}
impl Drop for ScannerPublicationCommitScopeGuard {
fn drop(&mut self) {
let Some(scope) = self.scope.as_ref() else {
return;
};
match scope.state() {
ScannerPublicationCommitState::Admitted => {
let _ = scope.mark_aborted_before_commit();
}
ScannerPublicationCommitState::InFlight => {
let _ = scope.mark_indeterminate();
}
ScannerPublicationCommitState::Committed
| ScannerPublicationCommitState::AbortedBeforeCommit
| ScannerPublicationCommitState::Indeterminate => {}
}
}
}
impl Debug for ScannerPublicationCommitScope {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ScannerPublicationCommitScope")
.field("expected_movement_epoch", &self.expected_movement_epoch())
.field("safe_deadline", &self.safe_deadline())
.field("remote_lease_token_count", &self.remote_lease_tokens().len())
.field("state", &self.state())
.finish()
}
}
impl ScannerPublicationCommitScope {
/// Construct a scope after the storage layer has acquired its movement
/// read permit. Callers must keep the scope attached to the actual
/// mutation owner until [`Self::wait_for_completion`] has resolved.
pub(crate) fn new_storage_owned(
expected_movement_epoch: u64,
safe_deadline: tokio::time::Instant,
remote_lease_tokens: Vec<Uuid>,
movement_permit: OwnedRwLockReadGuard<()>,
) -> Self {
Self::new_storage_owned_with_release_flag(
expected_movement_epoch,
safe_deadline,
remote_lease_tokens,
movement_permit,
Arc::new(std::sync::atomic::AtomicBool::new(true)),
)
}
pub(crate) fn new_storage_owned_with_release_flag(
expected_movement_epoch: u64,
safe_deadline: tokio::time::Instant,
remote_lease_tokens: Vec<Uuid>,
movement_permit: OwnedRwLockReadGuard<()>,
lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
) -> Self {
// Admission itself is not a safe release point. The flag becomes true
// only after the storage mutation owner reports a terminal state.
lease_release_safe.store(false, Ordering::Release);
Self {
inner: Arc::new(ScannerPublicationCommitScopeInner {
expected_movement_epoch,
safe_deadline,
remote_lease_tokens: remote_lease_tokens.into(),
cancellation: CancellationToken::new(),
state: AtomicU8::new(SCANNER_PUBLICATION_SCOPE_ADMITTED),
completed: Notify::new(),
movement_permit: Mutex::new(Some(movement_permit)),
lease_release_safe,
}),
}
}
pub fn expected_movement_epoch(&self) -> u64 {
self.inner.expected_movement_epoch
}
pub fn safe_deadline(&self) -> tokio::time::Instant {
self.inner.safe_deadline
}
pub fn is_expired(&self) -> bool {
tokio::time::Instant::now() >= self.safe_deadline()
}
pub fn remote_lease_tokens(&self) -> &[Uuid] {
&self.inner.remote_lease_tokens
}
pub fn cancellation_token(&self) -> CancellationToken {
self.inner.cancellation.clone()
}
pub fn is_cancelled(&self) -> bool {
self.inner.cancellation.is_cancelled()
}
/// Whether a mutation that has already begun may still enter its durable
/// commit boundary. The storage owner must check this immediately before
/// starting each irreversible fan-out/rename operation.
pub fn can_commit(&self) -> bool {
self.state() == ScannerPublicationCommitState::InFlight && !self.is_cancelled() && !self.is_expired()
}
pub fn state(&self) -> ScannerPublicationCommitState {
ScannerPublicationCommitState::from_u8(self.inner.state.load(Ordering::Acquire))
}
/// Request cancellation without claiming that a mutation has stopped.
/// The owner must still report `AbortedBeforeCommit` or `Indeterminate`.
pub fn cancel(&self) {
self.inner.cancellation.cancel();
}
pub fn try_begin(&self) -> std::result::Result<(), ScannerPublicationCommitStartError> {
if self.inner.cancellation.is_cancelled() {
return Err(ScannerPublicationCommitStartError::Cancelled);
}
if self.is_expired() {
return Err(ScannerPublicationCommitStartError::DeadlineExceeded);
}
self.inner
.state
.compare_exchange(
SCANNER_PUBLICATION_SCOPE_ADMITTED,
SCANNER_PUBLICATION_SCOPE_IN_FLIGHT,
Ordering::AcqRel,
Ordering::Acquire,
)
.map(|_| ())
.map_err(|state| {
if ScannerPublicationCommitState::from_u8(state).permits_lease_release() {
ScannerPublicationCommitStartError::Terminal
} else {
ScannerPublicationCommitStartError::AlreadyStarted
}
})
}
pub fn mark_committed(&self) -> bool {
self.mark_terminal(ScannerPublicationCommitState::Committed)
}
pub fn mark_aborted_before_commit(&self) -> bool {
if self
.inner
.state
.compare_exchange(
SCANNER_PUBLICATION_SCOPE_ADMITTED,
SCANNER_PUBLICATION_SCOPE_ABORTED_BEFORE_COMMIT,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok()
{
self.inner.lease_release_safe.store(true, Ordering::Release);
self.inner.completed.notify_waiters();
return true;
}
false
}
pub fn mark_indeterminate(&self) -> bool {
self.mark_terminal(ScannerPublicationCommitState::Indeterminate)
}
fn mark_terminal(&self, terminal: ScannerPublicationCommitState) -> bool {
self.inner
.state
.compare_exchange(SCANNER_PUBLICATION_SCOPE_IN_FLIGHT, terminal.as_u8(), Ordering::AcqRel, Ordering::Acquire)
.is_ok()
.then(|| {
if terminal.permits_lease_release() {
self.inner.lease_release_safe.store(true, Ordering::Release);
}
self.inner.completed.notify_waiters()
})
.is_some()
}
/// Wait until the mutation owner has reported a definitive terminal
/// state. The permit remains owned by this scope until all scope clones are
/// dropped or [`Self::release_movement_permit`] is called safely.
pub async fn wait_for_completion(&self) -> ScannerPublicationCommitState {
loop {
let notified = self.inner.completed.notified();
tokio::pin!(notified);
notified.as_mut().enable();
let state = self.state();
if state != ScannerPublicationCommitState::Admitted && state != ScannerPublicationCommitState::InFlight {
return state;
}
notified.await;
}
}
/// Release the storage-owned movement permit only after a known-safe
/// terminal result. Returns `false` for in-flight or indeterminate work.
pub async fn release_movement_permit(&self) -> bool {
if !self.state().permits_lease_release() {
return false;
}
self.inner.movement_permit.lock().await.take().is_some()
}
}
impl Drop for ScannerPublicationCommitScopeInner {
fn drop(&mut self) {
if !ScannerPublicationCommitState::from_u8(self.state.load(Ordering::Acquire)).permits_lease_release() {
self.lease_release_safe.store(false, Ordering::Release);
}
}
}
#[derive(Default, Clone)]
pub struct ObjectOptions {
// Use the maximum parity (N/2), used when saving server configuration files
@@ -384,6 +701,11 @@ pub struct ObjectOptions {
#[doc(hidden)]
pub put_object_cancellation: Option<tokio_util::sync::CancellationToken>,
/// Storage-owned scanner publication capability. This field is an
/// in-memory hand-off only; it is never copied into object metadata.
#[doc(hidden)]
pub scanner_publication_commit_scope: Option<ScannerPublicationCommitScope>,
pub data_movement: bool,
pub raw_data_movement_read: bool,
/// Materialize the data-movement per-part checksum sidecar for APIs that
@@ -473,6 +795,7 @@ impl std::fmt::Debug for ObjectOptions {
.field("skip_rebalancing", &self.skip_rebalancing)
.field("skip_free_version", &self.skip_free_version)
.field("put_object_cancellation", &self.put_object_cancellation.is_some())
.field("scanner_publication_commit_scope", &self.scanner_publication_commit_scope)
.field("data_movement", &self.data_movement)
.field("raw_data_movement_read", &self.raw_data_movement_read)
.field("include_part_checksums", &self.include_part_checksums)
@@ -1483,7 +1483,7 @@ impl NotificationSys {
futures.push(async move {
let client = client.ok_or_else(|| Error::other(format!("scanner activity peer[{idx}] is unreachable")))?;
let host = client.grid_host.clone();
scanner_activity_with_timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, &host, client.scanner_activity())
scanner_activity_with_retry(&client, &host)
.await
.map(|activity| (host, activity))
});
@@ -1962,6 +1962,44 @@ where
.map_err(|_| Error::other(format!("scanner activity peer {host} timed out after {timeout_duration:?}")))?
}
fn scanner_activity_should_retry(first_error: Option<&Error>, timed_out: bool) -> bool {
timed_out || first_error.is_some_and(PeerRestClient::is_network_like_error)
}
/// Retry one activity probe after a bounded reconnect when the first attempt
/// failed at the transport boundary. A peer that answered with an invalid or
/// incompatible activity response is not retried here: it must remain a hard
/// fail-closed result for the all-peer publication proof.
async fn scanner_activity_with_retry(client: &PeerRestClient, host: &str) -> Result<ScannerPeerActivity> {
let first = timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, client.scanner_activity()).await;
let should_retry = match &first {
Ok(Ok(_)) => false,
Ok(Err(err)) => scanner_activity_should_retry(Some(err), false),
Err(_) => scanner_activity_should_retry(None, true),
};
match first {
Ok(Ok(activity)) => return Ok(activity),
Ok(Err(err)) if !should_retry => return Err(err),
Ok(Err(err)) => {
debug!(peer = host, error = %err, "scanner activity probe failed on first transport attempt; reconnecting");
client.prepare_retry().await;
}
Err(_) => {
debug!(peer = host, timeout = ?SCANNER_ACTIVITY_PROBE_TIMEOUT, "scanner activity probe timed out on first attempt; reconnecting");
client.prepare_retry().await;
}
}
match timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, client.scanner_activity()).await {
Ok(result) => result,
Err(_) => {
client.evict_connection().await;
Err(Error::Timeout)
}
}
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
async fn call_peer_with_timeout<F, Fut>(
timeout_dur: Duration,
@@ -2882,6 +2920,20 @@ mod tests {
assert!(err.to_string().contains("peer-1"));
}
#[test]
fn scanner_activity_retry_only_reconnects_transport_failures() {
assert!(scanner_activity_should_retry(None, true));
assert!(scanner_activity_should_retry(Some(&Error::other("connection refused")), false));
assert!(!scanner_activity_should_retry(
Some(&Error::other("peer returned an invalid scanner activity response proof")),
false
));
assert!(!scanner_activity_should_retry(
Some(&Error::from(tonic::Status::internal("peer rejected activity"))),
false
));
}
#[tokio::test]
async fn scanner_dirty_usage_acknowledgement_rejects_missing_and_duplicate_targets() {
let sys = NotificationSys {
@@ -3657,6 +3657,7 @@ pub(in crate::set_disk) struct RenameTailOutcome {
pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
write_quorum: usize,
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
scanner_publication_commit_scope: Option<crate::object_api::ScannerPublicationCommitScope>,
}
impl<'a> RenameDataFenceOptions<'a> {
@@ -3667,8 +3668,17 @@ impl<'a> RenameDataFenceOptions<'a> {
Self {
write_quorum,
scanner_publication_lease_tokens,
scanner_publication_commit_scope: None,
}
}
pub(in crate::set_disk) fn with_publication_scope(
mut self,
scanner_publication_commit_scope: Option<crate::object_api::ScannerPublicationCommitScope>,
) -> Self {
self.scanner_publication_commit_scope = scanner_publication_commit_scope;
self
}
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
@@ -3995,6 +4005,7 @@ impl SetDisks {
let RenameDataFenceOptions {
write_quorum,
scanner_publication_lease_tokens,
scanner_publication_commit_scope: _scanner_publication_commit_scope,
} = fence_options;
if let Some(file_info) = disks
.iter()
@@ -4352,6 +4363,7 @@ impl SetDisks {
let RenameDataFenceOptions {
write_quorum,
scanner_publication_lease_tokens,
scanner_publication_commit_scope,
} = fence_options;
if let Some(file_info) = disks
.iter()
@@ -4383,11 +4395,15 @@ impl SetDisks {
let fanout_src_object = src_object.clone();
let fanout_dst_bucket = dst_bucket.clone();
let fanout_dst_object = dst_object.clone();
let fanout_publication_scope = scanner_publication_commit_scope.clone();
// Keep one coordinator task so a cancelled caller cannot drop partially
// completed disk mutations. Per-disk futures stay ordered in `join_all`,
// preserving slot-indexed quorum and convergence accounting without a
// scheduler task for every disk.
let fanout = tokio::spawn(async move {
// Keep the storage-owned movement permit attached to the actual
// fan-out owner, even if the caller future is cancelled.
let _fanout_publication_scope = fanout_publication_scope;
let successful_rename_completion_rank =
rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
let futures = fanout_disks
@@ -4401,6 +4417,7 @@ impl SetDisks {
let dst_object = fanout_dst_object.clone();
let dst_bucket = fanout_dst_bucket.clone();
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
let publication_scope = scanner_publication_commit_scope.clone();
std::panic::AssertUnwindSafe(async move {
// Test-only introspection guard: counts this operation as
@@ -4433,6 +4450,13 @@ impl SetDisks {
return Err(err);
}
if let Some(scope) = publication_scope.as_ref()
&& !scope.can_commit()
{
let _ = scope.mark_indeterminate();
return Err(DiskError::other("scanner publication commit scope deadline or cancellation reached"));
}
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
let result = disk
.rename_data_borrowed_with_fence(
+44 -1
View File
@@ -105,7 +105,9 @@ use crate::{
SnapshotLeaseToken, UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3,
},
error::{StorageError, to_object_err},
object_api::{GetObjectReader, NamespaceLockFence, ObjectInfo, ObjectLockConfigSnapshot, PutObjReader},
object_api::{
GetObjectReader, NamespaceLockFence, ObjectInfo, ObjectLockConfigSnapshot, PutObjReader, ScannerPublicationCommitScope,
},
// event::name::EventName,
services::event_notification::{EventArgs, send_event},
store::init_format::{
@@ -3937,6 +3939,47 @@ impl SetDisks {
owner.scanner_data_usage_publication_admission_guard().await
}
/// Acquire a storage-owned scanner publication scope for this set's
/// instance movement fence. The scope keeps the read permit alive across
/// scanner future cancellation until the mutation owner drains.
pub async fn scanner_data_usage_publication_commit_scope(
&self,
expected_movement_epoch: u64,
safe_deadline: tokio::time::Instant,
remote_lease_tokens: Vec<Uuid>,
) -> Option<ScannerPublicationCommitScope> {
let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
if epoch != expected_movement_epoch {
return None;
}
Some(ScannerPublicationCommitScope::new_storage_owned(
epoch,
safe_deadline,
remote_lease_tokens,
movement_permit,
))
}
pub async fn scanner_data_usage_publication_commit_scope_with_release_flag(
&self,
expected_movement_epoch: u64,
safe_deadline: tokio::time::Instant,
remote_lease_tokens: Vec<Uuid>,
lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
) -> Option<ScannerPublicationCommitScope> {
let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
if epoch != expected_movement_epoch {
return None;
}
Some(ScannerPublicationCommitScope::new_storage_owned_with_release_flag(
epoch,
safe_deadline,
remote_lease_tokens,
movement_permit,
lease_release_safe,
))
}
/// Whether both sets' namespace-lock implementations cover the same object key.
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
+94 -4
View File
@@ -66,6 +66,7 @@ use crate::bucket::lifecycle::bucket_lifecycle_ops::LifecycleOps;
use crate::bucket::utils::is_meta_bucketname;
use crate::bucket::versioning::VersioningApi;
use crate::disk::DiskAPI;
use crate::object_api::ScannerPublicationCommitScopeGuard;
use crate::set_disk::coding;
use crate::set_disk::core::io_primitives::GetCodecStreamingReaderBuildOutcome;
use crate::set_disk::mem;
@@ -269,6 +270,22 @@ const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.js
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024;
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256;
fn begin_scanner_publication_delete_mutation(scope: Option<&crate::object_api::ScannerPublicationCommitScope>) -> Result<()> {
let Some(scope) = scope else {
return Ok(());
};
if scope.state() == crate::object_api::ScannerPublicationCommitState::Admitted {
scope
.try_begin()
.map_err(|err| Error::other(format!("scanner publication delete scope cannot start: {err:?}")))?;
}
if !scope.can_commit() {
let _ = scope.mark_indeterminate();
return Err(StorageError::OperationCanceled);
}
Ok(())
}
fn take_scanner_publication_lease_tokens(user_defined: &mut HashMap<String, String>) -> Result<Option<HashMap<String, Uuid>>> {
let Some(encoded) = user_defined.remove(SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY) else {
return Ok(None);
@@ -2624,6 +2641,10 @@ impl SetDisks {
opts: &ObjectOptions,
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
crate::hp_guard!("SetDisks::put_object");
let mut scope_outcome_guard = opts
.scanner_publication_commit_scope
.clone()
.map(ScannerPublicationCommitScopeGuard::new);
let storage_class_config = self.storage_class_config_snapshot();
self.invalidate_get_object_metadata_cache(bucket, object).await;
@@ -3394,8 +3415,16 @@ impl SetDisks {
let commit_tmp_dir = tmp_dir.clone();
let commit_object_lock_guard = object_lock_guard.take();
let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
let commit_allows_early_ack = commit_object_lock_guard.is_some();
let detach_commit_owner = commit_allows_early_ack || commit_bucket_lifecycle_guard.is_some() || quota_mutation_fence;
let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone();
// A scanner publication scope owns the movement permit until the
// complete rename fan-out drains. Keep this path synchronous so
// its terminal state is known before the coordinator releases
// remote leases.
let commit_allows_early_ack = commit_object_lock_guard.is_some() && commit_scanner_publication_scope.is_none();
let detach_commit_owner = commit_scanner_publication_scope.is_some()
|| commit_allows_early_ack
|| commit_bucket_lifecycle_guard.is_some()
|| quota_mutation_fence;
let commit_write_path_label = write_path.metric_label();
let commit_is_versioned = opts.versioned || opts.version_suspended;
let commit_versioned = opts.versioned;
@@ -3491,7 +3520,7 @@ impl SetDisks {
}
Ok(())
};
let pre_rename_result = if cancellation.is_some() || request_cancellation.is_some() {
let mut pre_rename_result = if cancellation.is_some() || request_cancellation.is_some() {
tokio::select! {
biased;
_ = wait_for_put_object_commit_cancellation(cancellation.as_ref(), request_cancellation.as_ref()) => {
@@ -3502,7 +3531,28 @@ impl SetDisks {
} else {
pre_rename.await
};
if pre_rename_result.is_ok()
&& let Some(scope) = commit_scanner_publication_scope.as_ref()
&& let Err(err) = scope.try_begin()
{
let _ = scope.mark_aborted_before_commit();
pre_rename_result = Err(Error::other(format!("scanner publication commit scope cannot start: {err:?}")));
}
if pre_rename_result.is_ok()
&& let Some(scope) = commit_scanner_publication_scope.as_ref()
&& !scope.can_commit()
{
let _ = scope.mark_indeterminate();
pre_rename_result = Err(StorageError::OperationCanceled);
}
if let Err(err) = pre_rename_result {
if let Some(scope) = commit_scanner_publication_scope.as_ref() {
if scope.state() == crate::object_api::ScannerPublicationCommitState::Admitted {
let _ = scope.mark_aborted_before_commit();
} else {
let _ = scope.mark_indeterminate();
}
}
SetDisks::abort_quota_reservation_after_fence(
quota_reservation,
&commit_disks,
@@ -3537,9 +3587,17 @@ impl SetDisks {
crate::set_disk::core::io_primitives::RenameDataFenceOptions::new(
write_quorum,
commit_scanner_publication_lease_tokens.as_ref(),
),
)
.with_publication_scope(commit_scanner_publication_scope.clone()),
)
.await;
if let Some(scope) = commit_scanner_publication_scope.as_ref() {
if rename_result.is_ok() {
let _ = scope.mark_committed();
} else {
let _ = scope.mark_indeterminate();
}
}
#[cfg(any(test, feature = "test-util"))]
if rename_result.is_ok() {
pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameQuorum).await;
@@ -3854,6 +3912,11 @@ impl SetDisks {
let _ = handoff.send(());
}
if detach_commit_owner {
if let Some(scope_outcome_guard) = scope_outcome_guard.as_mut() {
// The spawned commit closure owns the scope clone and is
// now responsible for its terminal outcome.
scope_outcome_guard.disarm();
}
let mut cancellation = PutObjectCommitCancellation::new();
let child_token = cancellation.child_token();
let result = tokio::spawn(async move { Box::pin(commit(Some(child_token))).await })
@@ -7051,6 +7114,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
#[tracing::instrument(skip(self, opts))]
async fn delete_object(&self, bucket: &str, object: &str, mut opts: ObjectOptions) -> Result<ObjectInfo> {
let _scope_outcome_guard = opts
.scanner_publication_commit_scope
.clone()
.map(ScannerPublicationCommitScopeGuard::new);
let scanner_publication_commit_scope = opts.scanner_publication_commit_scope.clone();
// Scanner cleanup carries the per-peer lease fence as transient
// request metadata. Consume it before any delete-prefix fanout so it
// cannot be persisted or treated as user metadata.
@@ -7145,6 +7213,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
delete_request.set_skip_tier_free_version();
}
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
self.delete_object_version(bucket, object, &delete_request, false).await?;
if let Some((_, deleted_object)) = replication_delete {
ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_object).await;
@@ -7159,6 +7228,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
..Default::default()
};
delete_request.set_tier_free_version_id(&Uuid::new_v4().to_string());
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
self.delete_object_version(bucket, object, &delete_request, false).await?;
}
for version in &versions.free_versions {
@@ -7170,10 +7240,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
..Default::default()
};
delete_request.set_tier_free_version();
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
self.delete_object_version(bucket, object, &delete_request, false).await?;
}
}
}
if let Some(scope) = scanner_publication_commit_scope.as_ref() {
let _ = scope.mark_committed();
}
self.invalidate_get_object_metadata_cache(bucket, object).await;
return Ok(ObjectInfo::default());
}
@@ -7181,10 +7255,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
}
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
self.delete_prefix_with_scanner_publication_lease(bucket, object, scanner_publication_lease_tokens.as_ref())
.await
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
if let Some(scope) = scanner_publication_commit_scope.as_ref() {
let _ = scope.mark_committed();
}
self.invalidate_all_get_object_metadata_cache();
return Ok(ObjectInfo::default());
}
@@ -7257,10 +7335,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
..Default::default()
};
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
self.delete_object_version(bucket, object, &dfi, false)
.await
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
self.invalidate_get_object_metadata_cache(bucket, object).await;
if let Some(scope) = scanner_publication_commit_scope.as_ref() {
let _ = scope.mark_committed();
}
return Ok(ObjectInfo::from_file_info(&dfi, bucket, object, opts.versioned || opts.version_suspended));
}
@@ -7334,6 +7416,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
};
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
self.delete_object_version(bucket, object, &fi, should_force_delete_marker_for_missing_version(&opts))
.await
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
@@ -7345,6 +7428,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
oi.user_tags = Arc::clone(&goi.user_tags);
oi.replication_decision = goi.replication_decision;
self.invalidate_get_object_metadata_cache(bucket, object).await;
if let Some(scope) = scanner_publication_commit_scope.as_ref() {
let _ = scope.mark_committed();
}
return Ok(oi);
}
@@ -7370,6 +7456,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
self.delete_object_version(bucket, object, &dfi, opts.delete_marker)
.await
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
@@ -7395,6 +7482,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
obj_info.delete_marker = true;
}
self.invalidate_get_object_metadata_cache(bucket, object).await;
if let Some(scope) = scanner_publication_commit_scope.as_ref() {
let _ = scope.mark_committed();
}
Ok(obj_info)
}
+167 -1
View File
@@ -58,7 +58,7 @@ use crate::{
core::sets::Sets,
disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET},
layout::endpoints::EndpointServerPools,
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader, ScannerPublicationCommitScope},
};
use futures::future::join_all;
use http::HeaderMap;
@@ -522,6 +522,52 @@ impl ECStore {
Some((operation_guard, self.ctx.data_movement_operation_epoch()))
}
/// Acquire a storage-owned scanner publication scope. Unlike the legacy
/// admission helper, the movement permit is owned by the returned scope
/// and therefore survives cancellation of the scanner coordinator while
/// the actual metadata mutation drains.
pub async fn scanner_data_usage_publication_commit_scope(
&self,
expected_movement_epoch: u64,
safe_deadline: tokio::time::Instant,
remote_lease_tokens: Vec<Uuid>,
) -> Option<ScannerPublicationCommitScope> {
let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
if epoch != expected_movement_epoch {
return None;
}
Some(ScannerPublicationCommitScope::new_storage_owned(
epoch,
safe_deadline,
remote_lease_tokens,
movement_permit,
))
}
/// Variant used by the scanner supervisor to observe whether a scope was
/// dropped before reaching a safe terminal state. The flag is in-memory
/// only and lets the supervisor avoid releasing remote leases on an
/// indeterminate cancellation path.
pub async fn scanner_data_usage_publication_commit_scope_with_release_flag(
&self,
expected_movement_epoch: u64,
safe_deadline: tokio::time::Instant,
remote_lease_tokens: Vec<Uuid>,
lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
) -> Option<ScannerPublicationCommitScope> {
let (movement_permit, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
if epoch != expected_movement_epoch {
return None;
}
Some(ScannerPublicationCommitScope::new_storage_owned_with_release_flag(
epoch,
safe_deadline,
remote_lease_tokens,
movement_permit,
lease_release_safe,
))
}
/// Capture the current publication epoch without holding the movement
/// gate across backend I/O. Callers must re-admit the same epoch before a
/// mutation commits.
@@ -1409,9 +1455,29 @@ mod tests {
.await
.expect("movement writer should proceed after lease expiry")
.expect("expiry writer task should not panic");
assert!(
store.validate_scanner_publication_lease(expiring_token, 0).await.is_err(),
"an expired lease must not validate after its read guard is released"
);
assert!(!store.release_scanner_publication_lease(expiring_token).await);
}
#[tokio::test]
async fn scanner_publication_lease_rejects_a_new_movement_generation() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let (token, generation) = store
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect("an idle store should grant a publication lease");
assert_eq!(store.ctx.advance_data_movement_generation(), Some(1));
assert!(
store.validate_scanner_publication_lease(token, generation).await.is_err(),
"a lease from the prior movement generation must fail closed"
);
assert!(store.release_scanner_publication_lease(token).await);
}
#[tokio::test]
async fn scanner_publication_lease_rejects_stale_generation_before_install() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
@@ -1422,6 +1488,106 @@ mod tests {
assert!(error.to_string().contains("generation is stale"));
}
#[tokio::test(start_paused = true)]
async fn scanner_publication_commit_scope_owns_permit_until_terminal_drain() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let scope = store
.scanner_data_usage_publication_commit_scope(
0,
tokio::time::Instant::now() + Duration::from_secs(30),
vec![Uuid::new_v4()],
)
.await
.expect("idle storage should grant a publication scope");
assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::Admitted);
assert_eq!(scope.remote_lease_tokens().len(), 1);
let gate = store.ctx.data_movement_operation_gate();
let writer = tokio::spawn(async move { gate.write_owned().await });
tokio::task::yield_now().await;
assert!(!writer.is_finished(), "the scope must own its movement permit after the caller returns");
scope.cancel();
assert!(scope.mark_aborted_before_commit());
assert_eq!(
scope.wait_for_completion().await,
crate::object_api::ScannerPublicationCommitState::AbortedBeforeCommit
);
assert!(scope.release_movement_permit().await);
tokio::time::timeout(Duration::from_secs(1), writer)
.await
.expect("movement writer should proceed after the scope drains")
.expect("movement writer task should not panic");
}
#[tokio::test(start_paused = true)]
async fn scanner_publication_commit_scope_rejects_late_start_and_keeps_indeterminate_permit() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let scope = store
.scanner_data_usage_publication_commit_scope(0, tokio::time::Instant::now() + Duration::from_secs(1), Vec::new())
.await
.expect("idle storage should grant a publication scope");
tokio::time::advance(Duration::from_secs(1)).await;
assert_eq!(
scope.try_begin(),
Err(crate::object_api::ScannerPublicationCommitStartError::DeadlineExceeded)
);
assert!(
!scope.release_movement_permit().await,
"an admitted scope is not safe to release before owner resolution"
);
assert!(scope.mark_aborted_before_commit());
assert!(scope.release_movement_permit().await);
let scope = store
.scanner_data_usage_publication_commit_scope(0, tokio::time::Instant::now() + Duration::from_secs(30), Vec::new())
.await
.expect("a second idle publication scope should be granted");
scope.try_begin().expect("scope should enter the mutation state");
scope.cancel();
assert!(
!scope.mark_aborted_before_commit(),
"an in-flight mutation cannot claim pre-commit abort without storage proof"
);
assert!(scope.mark_indeterminate());
assert_eq!(
scope.wait_for_completion().await,
crate::object_api::ScannerPublicationCommitState::Indeterminate
);
assert!(!scope.release_movement_permit().await, "indeterminate mutation must retain the permit");
}
#[tokio::test]
async fn scanner_publication_scope_guard_classifies_early_returns_conservatively() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let permit = store.ctx.data_movement_operation_gate().read_owned().await;
let scope = ScannerPublicationCommitScope::new_storage_owned(
0,
tokio::time::Instant::now() + Duration::from_secs(30),
Vec::new(),
permit,
);
{
let _guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
}
assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::AbortedBeforeCommit);
assert!(scope.release_movement_permit().await);
let permit = store.ctx.data_movement_operation_gate().read_owned().await;
let scope = ScannerPublicationCommitScope::new_storage_owned(
0,
tokio::time::Instant::now() + Duration::from_secs(30),
Vec::new(),
permit,
);
scope.try_begin().expect("scope should enter the mutation state");
{
let _guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
}
assert_eq!(scope.state(), crate::object_api::ScannerPublicationCommitState::Indeterminate);
assert!(!scope.release_movement_permit().await);
}
#[tokio::test]
async fn scanner_target_guard_keeps_movement_writer_fenced_after_lease_release() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+125 -10
View File
@@ -39,11 +39,11 @@ use storage_api::owner::{
EcstoreListPathRawOptions, EcstoreNsScannerOpenRequest, EcstoreObjectOpts, EcstoreReplicationConfigurationExt,
EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore,
EcstoreVersioningApi, HTTPPreconditions, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete,
ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_invalidate_admin_data_usage_snapshot_cache,
ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure, ecstore_is_erasure_sd,
ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
ScannerPublicationCommitScope, ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission,
ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr,
ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config,
ecstore_invalidate_admin_data_usage_snapshot_cache, ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure,
ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
scanner_replication_config_for_lifecycle_eval,
@@ -55,6 +55,7 @@ use storage_api::owner::{
ecstore_new_disk,
};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
pub mod data_usage_define;
pub mod error;
@@ -752,6 +753,48 @@ pub(crate) fn scanner_publication_epoch_changed(error: &EcstoreError) -> bool {
)
}
pub(crate) async fn delete_config_with_publication_scope_for_epoch<S>(
api: Arc<S>,
bucket: &str,
object: &str,
mut opts: ScannerObjectOptions,
expected_epoch: u64,
scanner_publication_commit_scope: Option<ScannerPublicationCommitScope>,
) -> EcstoreResult<ScannerObjectInfo>
where
S: ScannerObjectIO + ScannerConfigObjectDelete,
{
let legacy_admission = if scanner_publication_commit_scope.is_none() {
Some(
scanner_publication_admission_for_epoch(api.clone(), expected_epoch)
.await
.ok_or_else(|| EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED))?,
)
} else {
None
};
let result = if let Some(scope) = scanner_publication_commit_scope {
// Keep the storage mutation and its publication scope alive after the
// scanner persistence waiter is cancelled. The delete path can fan
// out to remote RPCs or blocking local namespace syscalls; dropping
// only the waiter must not release the movement permit early.
opts.scanner_publication_commit_scope = Some(scope.clone());
let bucket = bucket.to_owned();
let object = object.to_owned();
tokio::spawn(async move {
let _publication_scope_owner = scope;
api.delete_config_object(&bucket, &object, opts).await
})
.await
.map_err(|err| EcstoreError::other(format!("scanner publication delete owner failed: {err}")))?
} else {
opts.scanner_publication_commit_scope = None;
api.delete_config_object(bucket, object, opts).await
};
drop(legacy_admission);
result
}
pub(crate) async fn delete_config_with_publication_admission_for_epoch<S>(
api: Arc<S>,
bucket: &str,
@@ -762,10 +805,7 @@ pub(crate) async fn delete_config_with_publication_admission_for_epoch<S>(
where
S: ScannerObjectIO + ScannerConfigObjectDelete,
{
let Some(_admission) = scanner_publication_admission_for_epoch(api.clone(), expected_epoch).await else {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
api.delete_config_object(bucket, object, opts).await
delete_config_with_publication_scope_for_epoch(api, bucket, object, opts, expected_epoch, None).await
}
/// Capture the storage-owned publication epoch without retaining the read
@@ -796,13 +836,14 @@ where
Some(admission)
}
pub(crate) async fn save_config_shared_with_preconditions_and_lease_fence<S>(
pub(crate) async fn save_config_shared_with_preconditions_and_lease_fence_and_scope<S>(
api: Arc<S>,
file: &str,
data: Bytes,
sha256hex: Option<String>,
preconditions: HTTPPreconditions,
scanner_publication_lease_fence: Option<&str>,
scanner_publication_commit_scope: Option<ScannerPublicationCommitScope>,
) -> EcstoreResult<ScannerObjectInfo>
where
S: ScannerObjectIO,
@@ -822,6 +863,7 @@ where
&ScannerObjectOptions {
max_parity: true,
http_preconditions: Some(preconditions),
scanner_publication_commit_scope,
user_defined,
..Default::default()
},
@@ -886,6 +928,27 @@ pub trait ScannerConfigObjectDelete: Send + Sync + std::fmt::Debug + 'static {
async fn scanner_data_usage_publication_admission(&self) -> Option<ScannerDataUsagePublicationAdmission> {
None
}
/// Acquire a storage-owned scope for a fenced scanner metadata mutation.
/// Implementations without a storage movement owner fail closed.
async fn scanner_data_usage_publication_commit_scope(
&self,
_expected_movement_epoch: u64,
_safe_deadline: tokio::time::Instant,
_remote_lease_tokens: Vec<Uuid>,
) -> Option<ScannerPublicationCommitScope> {
None
}
async fn scanner_data_usage_publication_commit_scope_with_release_flag(
&self,
_expected_movement_epoch: u64,
_safe_deadline: tokio::time::Instant,
_remote_lease_tokens: Vec<Uuid>,
_lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
) -> Option<ScannerPublicationCommitScope> {
None
}
}
pub struct ScannerDataUsagePublicationAdmission {
@@ -929,6 +992,32 @@ impl ScannerConfigObjectDelete for ECStore {
let (read_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
Some(ScannerDataUsagePublicationAdmission::fenced(read_guard, epoch))
}
async fn scanner_data_usage_publication_commit_scope(
&self,
expected_movement_epoch: u64,
safe_deadline: tokio::time::Instant,
remote_lease_tokens: Vec<Uuid>,
) -> Option<ScannerPublicationCommitScope> {
self.scanner_data_usage_publication_commit_scope(expected_movement_epoch, safe_deadline, remote_lease_tokens)
.await
}
async fn scanner_data_usage_publication_commit_scope_with_release_flag(
&self,
expected_movement_epoch: u64,
safe_deadline: tokio::time::Instant,
remote_lease_tokens: Vec<Uuid>,
lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
) -> Option<ScannerPublicationCommitScope> {
self.scanner_data_usage_publication_commit_scope_with_release_flag(
expected_movement_epoch,
safe_deadline,
remote_lease_tokens,
lease_release_safe,
)
.await
}
}
#[async_trait::async_trait]
@@ -946,6 +1035,32 @@ impl ScannerConfigObjectDelete for SetDisks {
let (read_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
Some(ScannerDataUsagePublicationAdmission::fenced(read_guard, epoch))
}
async fn scanner_data_usage_publication_commit_scope(
&self,
expected_movement_epoch: u64,
safe_deadline: tokio::time::Instant,
remote_lease_tokens: Vec<Uuid>,
) -> Option<ScannerPublicationCommitScope> {
self.scanner_data_usage_publication_commit_scope(expected_movement_epoch, safe_deadline, remote_lease_tokens)
.await
}
async fn scanner_data_usage_publication_commit_scope_with_release_flag(
&self,
expected_movement_epoch: u64,
safe_deadline: tokio::time::Instant,
remote_lease_tokens: Vec<Uuid>,
lease_release_safe: Arc<std::sync::atomic::AtomicBool>,
) -> Option<ScannerPublicationCommitScope> {
self.scanner_data_usage_publication_commit_scope_with_release_flag(
expected_movement_epoch,
safe_deadline,
remote_lease_tokens,
lease_release_safe,
)
.await
}
}
#[cfg(test)]
+132 -33
View File
@@ -16,6 +16,7 @@ use std::collections::BTreeMap;
use std::future::Future;
#[cfg(test)]
use std::sync::Mutex as StdMutex;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, LazyLock, RwLock};
use self::heal_info::{BackgroundHealInfoReadStatus, read_background_heal_info_with_epoch, save_background_heal_info_for_epoch};
@@ -62,8 +63,8 @@ use tokio::time::{Duration, Instant};
use tokio_util::sync::CancellationToken;
use tokio_util::task::AbortOnDropHandle;
use tracing::{debug, error, info, instrument, warn};
use uuid::Uuid;
use crate::storage_api::owner::SCANNER_PUBLICATION_LEASE_TTL_MS;
use crate::storage_api::scan::{
BucketOperations, BucketOptions, NamespaceLocking as _, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
@@ -72,7 +73,7 @@ use crate::{
ECStore, EcstoreError, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerLifecycleConfigExt as _,
ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch, get_lifecycle_config,
get_replication_config, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, read_config,
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions_and_lease_fence,
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions_and_lease_fence_and_scope,
save_config_with_preconditions, save_config_with_publication_admission_for_epoch, scanner_is_erasure_sd,
scanner_publication_admission_for_epoch, scanner_publication_epoch, scanner_publication_epoch_changed,
};
@@ -454,6 +455,7 @@ fn data_usage_backup_due(data_usage_info: &DataUsageInfo) -> bool {
}
#[cfg(test)]
#[allow(dead_code)]
async fn sync_data_usage_backup_from_primary(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
@@ -461,12 +463,34 @@ async fn sync_data_usage_backup_from_primary(
sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(ctx, storeapi, None, None, None).await
}
#[allow(dead_code)]
async fn sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
expected_publication_epoch: Option<u64>,
remote_lease_deadline: Option<std::time::Instant>,
scanner_publication_lease_fence: Option<&str>,
) -> Result<(), EcstoreError> {
sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence_and_scope(
ctx,
storeapi,
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence,
Vec::new(),
Arc::new(AtomicBool::new(true)),
)
.await
}
async fn sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence_and_scope(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
expected_publication_epoch: Option<u64>,
remote_lease_deadline: Option<std::time::Instant>,
scanner_publication_lease_fence: Option<&str>,
remote_lease_tokens: Vec<Uuid>,
lease_release_safe: Arc<AtomicBool>,
) -> Result<(), EcstoreError> {
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
@@ -531,15 +555,48 @@ async fn sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
}
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
save_config_shared_with_preconditions_and_lease_fence(
let publication_scope = match expected_publication_epoch {
Some(expected_epoch) => {
storeapi
.scanner_data_usage_publication_commit_scope_with_release_flag(
expected_epoch,
usage_store::scanner_publication_scope_deadline(data_usage_persist_timeout(), remote_lease_deadline),
remote_lease_tokens.clone(),
Arc::clone(&lease_release_safe),
)
.await
}
None => None,
};
if expected_publication_epoch.is_some() && publication_scope.is_none() {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
}
let save_result = save_config_shared_with_preconditions_and_lease_fence_and_scope(
storeapi.clone(),
&backup_path,
primary.clone(),
sha256hex,
revision.preconditions(),
scanner_publication_lease_fence,
publication_scope.clone(),
)
.await
.await;
if let Some(scope) = publication_scope {
match scope.wait_for_completion().await {
crate::storage_api::owner::ScannerPublicationCommitState::Committed
| crate::storage_api::owner::ScannerPublicationCommitState::AbortedBeforeCommit => save_result,
crate::storage_api::owner::ScannerPublicationCommitState::Indeterminate
| crate::storage_api::owner::ScannerPublicationCommitState::Admitted
| crate::storage_api::owner::ScannerPublicationCommitState::InFlight => Err(EcstoreError::other(
"scanner backup publication commit scope did not reach a safe terminal state",
)),
}
} else {
save_result
}
};
match save_result {
@@ -1416,13 +1473,10 @@ async fn run_data_scanner_cycle_with_budget(
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
};
let usage_persist_baseline_result = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await;
let usage_persist_baseline_result = read_data_usage_persist_baseline(storeapi.clone()).await;
drop(baseline_publication_guard);
let usage_persist_baseline = match usage_persist_baseline_result {
Ok((data, revision)) => DataUsagePersistBaseline {
data: data.map(Bytes::from),
revision,
},
Ok(baseline) => baseline,
Err(err) => {
error!(
target: "rustfs::scanner",
@@ -1462,10 +1516,23 @@ async fn run_data_scanner_cycle_with_budget(
{
Some(ScannerCycleDeferReason::DataMovement)
}
// A complete walk can still be retained as an observational snapshot
// when only the final activity proof was unavailable. It must not
// block the observation receiver: the authoritative publication
// fence remains enforced by the usage store and the cycle is advanced
// as partial without acknowledging dirty usage.
Ok(result)
if result.has_observational_snapshot()
&& matches!(
result.status,
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
) =>
{
None
}
Ok(result) => final_data_usage_publication_defer_reason(storeapi.as_ref(), result.status).await,
Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
};
let publication_deferred = publication_defer_reason.is_some();
let publication_epoch = scan_result.as_ref().ok().and_then(ScannerCycleResult::publication_epoch);
let remote_publication_lease_targets = if publication_defer_reason.is_none() {
scan_result
@@ -1479,11 +1546,6 @@ async fn run_data_scanner_cycle_with_budget(
let mut remote_publication_leases = None;
let remote_lease_defer_reason = if remote_publication_lease_targets.is_empty() {
None
} else if usage_persist_timeout >= Duration::from_millis(SCANNER_PUBLICATION_LEASE_TTL_MS) {
// The lease is intentionally fixed-duration and has no renewal path.
// Refuse a persistence budget that could outlive it instead of
// allowing the peer to admit movement while a local PUT is in flight.
Some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
} else if let Some(notification_system) = storeapi.notification_system() {
match notification_system
.acquire_scanner_publication_leases(remote_publication_lease_targets.clone())
@@ -1534,21 +1596,16 @@ async fn run_data_scanner_cycle_with_budget(
remote_lease_fence.is_some(),
))
.then_some(ScannerCycleDeferReason::ActivityBaselineUnavailable);
let remote_lease_covers_persistence = remote_lease_deadline.is_none_or(|deadline| {
std::time::Instant::now()
.checked_add(usage_persist_timeout)
.is_some_and(|latest_finish| latest_finish < deadline)
});
let publication_defer_reason = publication_defer_reason
.or(remote_lease_defer_reason)
.or(remote_lease_fence_defer_reason);
let publication_defer_reason = (!remote_lease_covers_persistence)
.then_some(ScannerCycleDeferReason::ActivityBaselineUnavailable)
.or(publication_defer_reason);
// Include reasons discovered while acquiring or validating remote leases.
let publication_deferred = publication_defer_reason.is_some();
let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
let remote_lease_probe = remote_publication_leases
.as_ref()
.map(|(notification_system, grants)| (Arc::clone(notification_system), grants.clone()));
let remote_lease_release_safe = Arc::new(AtomicBool::new(true));
let mut usage_persist_outcome = match publication_defer_reason {
Some(reason) => {
drop(receiver);
@@ -1562,6 +1619,11 @@ async fn run_data_scanner_cycle_with_budget(
let ctx_clone = ctx.clone();
let route_probe_store = storeapi.clone();
let remote_lease_fence = remote_lease_fence.clone();
let remote_lease_release_safe_for_task = Arc::clone(&remote_lease_release_safe);
let remote_lease_tokens = remote_publication_leases
.as_ref()
.map(|(_, grants)| grants.iter().map(|grant| grant.lease.token).collect())
.unwrap_or_default();
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
ctx_clone,
@@ -1573,7 +1635,9 @@ async fn run_data_scanner_cycle_with_budget(
publication_epoch,
remote_lease_deadline,
remote_lease_fence,
),
)
.with_remote_lease_tokens(remote_lease_tokens)
.with_lease_release_flag(remote_lease_release_safe_for_task),
move || {
let storeapi = route_probe_store.clone();
let remote_lease_probe = remote_lease_probe.clone();
@@ -1638,16 +1702,28 @@ async fn run_data_scanner_cycle_with_budget(
let lease_expired = remote_publication_leases
.as_ref()
.is_some_and(|(_, grants)| grants.iter().any(|grant| !grant.lease.is_valid()));
if let Some((notification_system, grants)) = remote_publication_leases.take() {
if !remote_lease_release_safe.load(Ordering::Acquire) {
// A cancelled or detached storage mutation did not report a safe
// terminal state. Keep remote grants until their own expiry rather
// than releasing movement admission while a commit may be unknown.
usage_persist_outcome = if usage_persist_outcome == DataUsagePersistOutcome::Failed {
DataUsagePersistOutcome::Failed
} else {
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded)
};
} else if let Some((notification_system, grants)) = remote_publication_leases.take() {
let release_result = notification_system.release_scanner_publication_leases(grants).await;
if lease_expired || release_result.is_err() {
let lease_release_failed = release_result.is_err();
if lease_expired || lease_release_failed {
// A lease that expired or could not be released is never treated
// as a successful authoritative publication. The peer may have
// admitted movement immediately after the lease ended.
usage_persist_outcome = if usage_persist_outcome == DataUsagePersistOutcome::Failed {
DataUsagePersistOutcome::Failed
} else if lease_release_failed {
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseReleaseFailed)
} else {
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded)
};
}
}
@@ -2803,12 +2879,7 @@ fn finalize_scanner_cycle_result(
scan_cycle_result: crate::scanner_io::ScannerCycleResult,
usage_persist_outcome: DataUsagePersistOutcome,
) -> (ScannerCycleOutcome, bool, Vec<ScannerDirtyUsageAcknowledgement>) {
let completion_outcome = scanner_cycle_completion_outcome(
scan_cycle_result.status,
usage_persist_outcome,
scan_cycle_result.has_dirty_usage_to_acknowledge(),
scan_cycle_result.has_failed_dirty_usage(),
);
let completion_outcome = scanner_cycle_completion_outcome_for_result(&scan_cycle_result, usage_persist_outcome);
let pending_maintenance_work = scan_cycle_result.has_pending_maintenance_work();
let durable_complete_snapshot = scan_cycle_result.status == ScannerCycleStatus::Complete
&& matches!(
@@ -2823,6 +2894,34 @@ fn finalize_scanner_cycle_result(
(completion_outcome, pending_maintenance_work, remote_dirty_usage_acknowledgements)
}
fn scanner_cycle_completion_outcome_for_result(
scan_cycle_result: &crate::scanner_io::ScannerCycleResult,
usage_persist_outcome: DataUsagePersistOutcome,
) -> ScannerCycleOutcome {
let has_dirty_usage = scan_cycle_result.has_dirty_usage_to_acknowledge();
let has_failed_dirty_usage = scan_cycle_result.has_failed_dirty_usage();
if scan_cycle_result.has_observational_snapshot()
&& matches!(
scan_cycle_result.status,
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
)
{
return match usage_persist_outcome {
DataUsagePersistOutcome::Saved
| DataUsagePersistOutcome::AlreadyDurable
| DataUsagePersistOutcome::PriorCycleDurable
| DataUsagePersistOutcome::Current
if !has_failed_dirty_usage =>
{
ScannerCycleOutcome::Partial
}
DataUsagePersistOutcome::Deferred(reason) => ScannerCycleOutcome::Deferred(reason),
_ => ScannerCycleOutcome::Failed,
};
}
scanner_cycle_completion_outcome(scan_cycle_result.status, usage_persist_outcome, has_dirty_usage, has_failed_dirty_usage)
}
/// Decide whether an incoming usage snapshot must be skipped as stale, given the local
/// wall clock `now`. Mirrors `stale_data_usage_persist_reason` in
/// `crates/ecstore/src/data_usage/mod.rs` — keep the two consistent.
+41 -16
View File
@@ -1327,6 +1327,13 @@ pub(super) async fn persisted_usage_floor_for_startup(
let mut floor = PersistedUsageFloor::default();
let mut found_any = false;
let mut bootstrap_pending = false;
// A valid JSON object without a baseline identity is not a floor and must
// never be treated as an empty one. It can, however, be a partially
// written v2 primary left behind during an upgrade. Keep its epoch as a
// fence while looking for a durable companion snapshot; if no companion
// is new enough, the caller still fails closed below.
let mut invalid_baseline_path: Option<String> = None;
let mut invalid_baseline_epoch: Option<u64> = None;
let update_floor = |floor: &mut PersistedUsageFloor, usage: &DataUsageInfo, path: &str| -> Result<(), ScannerError> {
floor.leader_epoch = floor.leader_epoch.max(usage.scanner_epoch.unwrap_or_default());
if let Some(completed_cycle) = usage.scanner_cycle {
@@ -1340,6 +1347,7 @@ pub(super) async fn persisted_usage_floor_for_startup(
};
for primary_path in [DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()] {
let backup_path = format!("{primary_path}.bkp");
let is_v2_path = primary_path == DATA_USAGE_OBJ_NAME_PATH.as_str();
let primary_epoch = match read_config_with_revision(storeapi.clone(), primary_path).await {
Ok((Some(data), _)) => {
let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| {
@@ -1353,13 +1361,20 @@ pub(super) async fn persisted_usage_floor_for_startup(
update_floor(&mut floor, &usage, primary_path)?;
None
} else if !data_usage_info_has_persisted_baseline_identity(&usage) {
return Err(ScannerError::Other(format!(
"scanner usage floor from {primary_path} has no persisted baseline identity"
)));
invalid_baseline_path.get_or_insert_with(|| primary_path.to_string());
invalid_baseline_epoch = invalid_baseline_epoch.max(usage.scanner_epoch);
None
} else {
let epoch = usage.scanner_epoch.unwrap_or_default();
update_floor(&mut floor, &usage, primary_path)?;
Some(epoch)
// A legacy snapshot may be structurally valid but older
// than an incomplete v2 snapshot left by a newer leader.
// Do not let that candidate regress the startup floor.
if !is_v2_path && invalid_baseline_epoch.is_some_and(|fenced_epoch| epoch < fenced_epoch) {
None
} else {
update_floor(&mut floor, &usage, primary_path)?;
Some(epoch)
}
}
}
Ok((None, _)) => None,
@@ -1377,21 +1392,26 @@ pub(super) async fn persisted_usage_floor_for_startup(
"scanner usage bootstrap conflicts with a persisted backup".to_string(),
));
}
any_found = true;
let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| {
ScannerError::Other(format!("failed to decode scanner usage floor from {backup_path}: {err}"))
})?;
if !data_usage_info_has_persisted_baseline_identity(&usage) {
return Err(ScannerError::Other(format!(
"scanner usage floor from {backup_path} has no persisted baseline identity"
)));
}
let backup_epoch = usage.scanner_epoch.unwrap_or_default();
// A backup write from an older leader may complete after the
// primary epoch has been fenced. It must not advance the startup
// floor unless its epoch is at least as new as the primary.
if primary_epoch.is_none_or(|epoch| backup_epoch >= epoch) {
update_floor(&mut floor, &usage, &backup_path)?;
invalid_baseline_path.get_or_insert_with(|| backup_path.clone());
invalid_baseline_epoch = invalid_baseline_epoch.max(usage.scanner_epoch);
// This is still persisted state, so it must not enable a
// missing-state bootstrap. Continue to a legacy pair in
// case it contains a complete, fenced snapshot.
} else {
let backup_epoch = usage.scanner_epoch.unwrap_or_default();
// A backup write from an older leader may complete after the
// primary epoch has been fenced. It must not advance the startup
// floor unless its epoch is at least as new as the primary.
if primary_epoch.is_none_or(|epoch| backup_epoch >= epoch)
&& invalid_baseline_epoch.is_none_or(|epoch| backup_epoch >= epoch)
{
update_floor(&mut floor, &usage, &backup_path)?;
any_found = true;
}
}
}
Ok((None, _)) => {}
@@ -1413,6 +1433,11 @@ pub(super) async fn persisted_usage_floor_for_startup(
}
if !found_any && !bootstrap_pending {
if let Some(path) = invalid_baseline_path {
return Err(ScannerError::Other(format!(
"persisted scanner usage floor from {path} has no authoritative baseline or newer valid backup"
)));
}
if !allow_missing_for_bootstrap {
return Err(ScannerError::Other(
"persisted scanner usage floor has no authoritative baseline".to_string(),
+31 -4
View File
@@ -82,9 +82,30 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
primary: Option<&[u8]>,
allow_bootstrap_pending: bool,
) -> Result<Option<DataUsageInfo>, ScannerError> {
// A partially written v2 primary is not itself a baseline, but a durable
// companion may still provide one after an interrupted upgrade. Keep the
// primary epoch as a fence while checking those companions; malformed
// bytes and bootstrap markers retain their fail-closed behavior.
let mut invalid_primary_epoch = None;
if let Some(primary) = primary {
return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str(), allow_bootstrap_pending)
.map(Some);
let usage: DataUsageInfo = serde_json::from_slice(primary).map_err(|err| {
ScannerError::Other(format!(
"failed to decode scanner usage epoch fence from {}: {err}",
DATA_USAGE_OBJ_NAME_PATH.as_str()
))
})?;
if data_usage_info_has_persisted_baseline_identity(&usage)
|| (allow_bootstrap_pending && data_usage_info_is_bootstrap_pending(&usage))
{
return Ok(Some(usage));
}
if data_usage_info_is_bootstrap_pending(&usage) {
return Err(ScannerError::Other(format!(
"scanner usage epoch fence from {} has no persisted baseline identity",
DATA_USAGE_OBJ_NAME_PATH.as_str()
)));
}
invalid_primary_epoch = usage.scanner_epoch;
}
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
@@ -92,7 +113,10 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
.await
.map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence backup: {err}")))?;
if let Some(backup) = backup.as_deref() {
return decode_usage_snapshot_for_epoch_fence(backup, &backup_path, false).map(Some);
let usage = decode_usage_snapshot_for_epoch_fence(backup, &backup_path, false)?;
if invalid_primary_epoch.is_none_or(|epoch| usage.scanner_epoch.unwrap_or_default() >= epoch) {
return Ok(Some(usage));
}
}
for path in [
@@ -103,7 +127,10 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
.await
.map_err(|err| ScannerError::Other(format!("failed to read legacy scanner usage epoch fence: {err}")))?;
if let Some(legacy) = legacy.as_deref() {
return decode_usage_snapshot_for_epoch_fence(legacy, &path, false).map(Some);
let usage = decode_usage_snapshot_for_epoch_fence(legacy, &path, false)?;
if invalid_primary_epoch.is_none_or(|epoch| usage.scanner_epoch.unwrap_or_default() >= epoch) {
return Ok(Some(usage));
}
}
}
// A missing usage snapshot is an uninitialized state, not an empty
+383
View File
@@ -352,6 +352,7 @@ struct MemoryConfigStore {
objects: Mutex<HashMap<String, Vec<u8>>>,
revisions: Mutex<HashMap<String, u64>>,
insert_after_gets: Mutex<HashMap<String, Vec<u8>>>,
delayed_gets: Mutex<HashMap<String, Duration>>,
non_regular_objects: Mutex<HashSet<String>>,
fail_put_number: Mutex<HashMap<String, usize>>,
object_not_found_put_number: Mutex<HashMap<String, usize>>,
@@ -399,6 +400,9 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
_opts: &ObjectOptions,
) -> EcstoreResult<GetObjectReader> {
let key = memory_config_key(bucket, object);
if let Some(delay) = self.delayed_gets.lock().await.remove(&key) {
tokio::time::sleep(delay).await;
}
let inserted_data = self.insert_after_gets.lock().await.remove(&key);
let data = {
let mut objects = self.objects.lock().await;
@@ -1923,6 +1927,176 @@ async fn scanner_startup_uses_primary_and_backup_usage_floor() {
assert_eq!(epoch, 11);
}
#[tokio::test]
async fn scanner_usage_floor_keeps_valid_primary_when_backup_has_no_identity() {
let store = Arc::new(MemoryConfigStore::default());
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
let mut primary = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
primary.scanner_epoch = Some(8);
primary.scanner_cycle = Some(100);
let backup = DataUsageInfo {
scanner_epoch: Some(9),
scanner_cycle: Some(101),
usage_snapshot_complete: false,
..Default::default()
};
for (path, usage) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), primary), (backup_path.as_str(), backup)] {
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, path),
serde_json::to_vec(&usage).expect("usage snapshot should encode"),
);
}
assert_eq!(
persisted_usage_floor(store)
.await
.expect("valid primary should remain authoritative"),
PersistedUsageFloor {
next_cycle: 101,
leader_epoch: 8,
}
);
}
#[tokio::test]
async fn scanner_usage_floor_recovers_from_incomplete_v2_primary_using_fenced_backup() {
let store = Arc::new(MemoryConfigStore::default());
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
// This shape is valid JSON from an interrupted v2 publication, but it is
// not a durable baseline because the snapshot is incomplete. It must not
// be converted into an empty floor.
let primary = DataUsageInfo {
scanner_epoch: Some(7),
scanner_cycle: Some(100),
usage_snapshot_complete: false,
..Default::default()
};
let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
backup.scanner_epoch = Some(7);
backup.scanner_cycle = Some(103);
for (path, usage) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), primary), (backup_path.as_str(), backup)] {
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, path),
serde_json::to_vec(&usage).expect("usage snapshot should encode"),
);
}
assert_eq!(
persisted_usage_floor(store)
.await
.expect("valid backup should recover the usage floor"),
PersistedUsageFloor {
next_cycle: 104,
leader_epoch: 7,
}
);
}
#[tokio::test]
async fn scanner_usage_floor_does_not_bootstrap_over_incomplete_v2_primary() {
let store = Arc::new(MemoryConfigStore::default());
let primary = DataUsageInfo {
scanner_epoch: Some(7),
scanner_cycle: Some(100),
usage_snapshot_complete: false,
..Default::default()
};
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
serde_json::to_vec(&primary).expect("usage snapshot should encode"),
);
let err = persisted_usage_floor_for_startup(store, true)
.await
.expect_err("an existing incomplete primary must remain fail-closed");
assert!(err.to_string().contains("no authoritative baseline"));
}
#[tokio::test]
async fn scanner_usage_floor_rejects_backup_older_than_incomplete_v2_primary() {
let store = Arc::new(MemoryConfigStore::default());
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
let primary = DataUsageInfo {
scanner_epoch: Some(7),
scanner_cycle: Some(100),
usage_snapshot_complete: false,
..Default::default()
};
let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
backup.scanner_epoch = Some(6);
backup.scanner_cycle = Some(10_000);
for (path, usage) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), primary), (backup_path.as_str(), backup)] {
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, path),
serde_json::to_vec(&usage).expect("usage snapshot should encode"),
);
}
let err = persisted_usage_floor_for_startup(store, true)
.await
.expect_err("an older backup must not cross the incomplete primary epoch fence");
assert!(err.to_string().contains("no authoritative baseline"));
}
#[tokio::test]
async fn scanner_usage_floor_rejects_older_legacy_primary_after_incomplete_v2_primary() {
let store = Arc::new(MemoryConfigStore::default());
let primary = DataUsageInfo {
scanner_epoch: Some(7),
scanner_cycle: Some(100),
usage_snapshot_complete: false,
..Default::default()
};
let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
legacy.scanner_epoch = Some(6);
legacy.scanner_cycle = Some(103);
for (path, usage) in [
(DATA_USAGE_OBJ_NAME_PATH.as_str(), primary),
(LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), legacy),
] {
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, path),
serde_json::to_vec(&usage).expect("usage snapshot should encode"),
);
}
let err = persisted_usage_floor_for_startup(store, true)
.await
.expect_err("an older legacy baseline must not cross the incomplete v2 epoch fence");
assert!(err.to_string().contains("no authoritative baseline"));
}
#[tokio::test]
async fn scanner_leadership_fencing_recovers_incomplete_v2_primary_from_backup() {
let store = Arc::new(MemoryConfigStore::default());
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
let primary = serde_json::to_vec(&DataUsageInfo {
scanner_epoch: Some(7),
scanner_cycle: Some(100),
usage_snapshot_complete: false,
..Default::default()
})
.expect("incomplete usage snapshot should encode");
let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
backup.scanner_epoch = Some(7);
backup.scanner_cycle = Some(103);
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, &backup_path),
serde_json::to_vec(&backup).expect("backup usage snapshot should encode"),
);
let recovered = usage_snapshot_for_epoch_fence(store, Some(&primary), false)
.await
.expect("a valid backup should provide the fencing baseline")
.expect("the fencing baseline should be present");
assert_eq!(recovered.scanner_epoch, Some(7));
assert_eq!(recovered.scanner_cycle, Some(103));
}
#[tokio::test]
async fn scanner_usage_floor_ignores_older_backup_after_primary_epoch_fence() {
let store = Arc::new(MemoryConfigStore::default());
@@ -3182,6 +3356,79 @@ async fn test_observational_usage_defers_when_authoritative_baseline_is_missing(
);
}
#[tokio::test]
async fn test_observational_usage_uses_fenced_backup_when_v2_primary_has_no_identity() {
let store = Arc::new(MemoryConfigStore::default());
let primary = DataUsageInfo {
scanner_epoch: Some(7),
scanner_cycle: Some(100),
usage_snapshot_complete: false,
..Default::default()
};
let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
backup.scanner_epoch = Some(7);
backup.scanner_cycle = Some(103);
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
serde_json::to_vec(&primary).expect("incomplete primary should encode"),
);
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, &format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str())),
serde_json::to_vec(&backup).expect("backup baseline should encode"),
);
let (sender, receiver) = mpsc::channel(1);
let mut observation = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1);
observation.usage_snapshot_converged = Some(false);
sender.send(observation).await.expect("observation should enqueue");
drop(sender);
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
CancellationToken::new(),
store.clone(),
receiver,
None,
None,
|| async { false },
)
.await;
assert_eq!(outcome, DataUsagePersistOutcome::Saved);
let observed = read_config(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str())
.await
.expect("observational snapshot should be persisted");
let observed = serde_json::from_slice::<DataUsageInfo>(&observed).expect("observational snapshot should decode");
assert_eq!(observed.usage_snapshot_authoritative_baseline, Some(backup.snapshot_identity()));
}
#[tokio::test]
async fn usage_baseline_does_not_fall_back_to_older_legacy_snapshot() {
let store = Arc::new(MemoryConfigStore::default());
let primary = DataUsageInfo {
scanner_epoch: Some(7),
scanner_cycle: Some(100),
usage_snapshot_complete: false,
..Default::default()
};
let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
legacy.scanner_epoch = Some(6);
legacy.scanner_cycle = Some(103);
let primary_data = serde_json::to_vec(&primary).expect("incomplete primary should encode");
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
primary_data.clone(),
);
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
serde_json::to_vec(&legacy).expect("legacy baseline should encode"),
);
let baseline = read_data_usage_persist_baseline(store)
.await
.expect("baseline inspection should complete");
assert_eq!(baseline.data.as_deref(), Some(primary_data.as_slice()));
}
#[tokio::test]
#[serial]
async fn test_usage_route_barrier_precedes_durable_reconciliation() {
@@ -3251,6 +3498,85 @@ async fn coordinator_does_not_put_after_remote_generation_flip() {
assert_eq!(store.put_counts.lock().await.get(&key), None);
}
#[tokio::test]
async fn coordinator_classifies_an_expired_publication_lease() {
let store = Arc::new(MemoryConfigStore::default());
let (sender, receiver) = mpsc::channel(1);
sender
.send(complete_usage_with_bucket_count(
Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)),
1,
))
.await
.expect("usage snapshot should enqueue");
drop(sender);
let expired = std::time::Instant::now()
.checked_sub(std::time::Duration::from_secs(1))
.expect("test instant should support a one-second subtraction");
let outcome =
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence(
CancellationToken::new(),
store.clone(),
receiver,
None,
Some(DataUsagePersistBaseline {
data: None,
revision: DataUsageCacheRevision::Missing,
}),
ScannerPublicationFence::new(None, Some(expired), None),
|| async { false },
)
.await;
assert_eq!(
outcome,
DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded)
);
assert!(store.put_counts.lock().await.is_empty(), "expired lease must prevent a PUT");
}
#[tokio::test]
async fn backup_sync_checks_the_lease_deadline_after_a_slow_backup_read() {
let store = Arc::new(MemoryConfigStore::default());
let primary_path = DATA_USAGE_OBJ_NAME_PATH.as_str();
let backup_path = format!("{primary_path}.bkp");
let primary_key = memory_config_key(RUSTFS_META_BUCKET, primary_path);
let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path);
let primary = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
store
.objects
.lock()
.await
.insert(primary_key, serde_json::to_vec(&primary).expect("primary usage snapshot should encode"));
store
.delayed_gets
.lock()
.await
.insert(backup_key.clone(), Duration::from_millis(20));
// The primary read is allowed to start, but the backup read consumes the
// remaining lease window. The second deadline check must prevent a stale
// backup PUT after that window has elapsed.
let deadline = std::time::Instant::now()
.checked_add(std::time::Duration::from_millis(5))
.expect("test deadline should support a five-millisecond window");
let result = sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
&CancellationToken::new(),
store.clone(),
None,
Some(deadline),
None,
)
.await;
assert!(scanner_publication_epoch_changed(
&result.expect_err("an expired backup lease must defer publication")
));
assert!(!store.objects.lock().await.contains_key(&backup_key));
assert_eq!(store.put_counts.lock().await.get(&backup_key), None);
}
#[tokio::test]
#[serial]
async fn test_deferred_usage_save_keeps_last_real_save_metric() {
@@ -4302,6 +4628,8 @@ fn scanner_cycle_cache_floor_stays_pending_during_deferred_usage_publication() {
for reason in [
ScannerCycleDeferReason::DataMovement,
ScannerCycleDeferReason::ActivityBaselineUnavailable,
ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded,
ScannerCycleDeferReason::PublicationLeaseReleaseFailed,
] {
let deferred = DataUsagePersistOutcome::Deferred(reason);
assert_eq!(
@@ -4368,6 +4696,26 @@ fn finalizing_a_deferred_usage_save_keeps_dirty_work_pending() {
crate::scanner_io::clear_dirty_usage_bucket("photos");
}
#[test]
#[serial]
fn finalizing_post_scan_observation_advances_partially_without_dirty_ack() {
crate::scanner_io::clear_dirty_usage_bucket("photos");
crate::scanner_io::record_dirty_usage_bucket("photos");
let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests();
let observed = crate::scanner_io::ScannerCycleResult::new(
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
Some(dirty_snapshot),
)
.with_observational_snapshot_published(true);
let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(observed, DataUsagePersistOutcome::Saved);
assert_eq!(outcome, ScannerCycleOutcome::Partial);
assert!(acknowledgements.is_empty());
assert!(crate::scanner_io::dirty_usage_buckets_pending());
crate::scanner_io::clear_dirty_usage_bucket("photos");
}
#[tokio::test]
async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
let pending = remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::<bool, std::io::Error>(true))).await;
@@ -4451,6 +4799,18 @@ fn data_usage_persist_wait_covers_cache_retries_and_backup() {
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
}
#[test]
fn scanner_publication_lease_deadline_reason_remains_distinct() {
assert_eq!(
ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded.as_str(),
"publication_lease_deadline_exceeded"
);
assert_eq!(
ScannerCycleDeferReason::PublicationLeaseReleaseFailed.as_str(),
"publication_lease_release_failed"
);
}
#[tokio::test]
async fn data_usage_persist_wait_aborts_when_scanner_is_cancelled() {
let ctx = CancellationToken::new();
@@ -4480,6 +4840,29 @@ async fn data_usage_persist_wait_aborts_after_timeout() {
assert!(task.is_finished());
}
#[tokio::test(start_paused = true)]
async fn data_usage_persist_timeout_drops_owned_task_without_a_late_commit() {
let ctx = CancellationToken::new();
let commit_started = Arc::new(AtomicBool::new(false));
let commit_started_by_task = commit_started.clone();
let task_ready = Arc::new(tokio::sync::Notify::new());
let task_ready_by_task = task_ready.clone();
let mut task = AbortOnDropHandle::new(tokio::spawn(async move {
task_ready_by_task.notify_one();
std::future::pending::<()>().await;
commit_started_by_task.store(true, Ordering::Release);
DataUsagePersistOutcome::Saved
}));
task_ready.notified().await;
let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(1)).await;
assert!(matches!(result, DataUsagePersistTaskResult::TimedOut));
assert!(task.is_finished(), "the timed-out persistence task must be drained before return");
tokio::task::yield_now().await;
assert!(!commit_started.load(Ordering::Acquire), "an owned task must not commit after its timeout");
}
#[tokio::test(start_paused = true)]
async fn maintenance_feature_inspection_preserves_base_cycle_after_timeout() {
let ctx = CancellationToken::new();
+182 -14
View File
@@ -13,7 +13,10 @@
// limitations under the License.
/// Data-usage snapshot persistence: CAS store pipeline, epoch baselines, and observed-snapshot cleanup.
use super::*;
use crate::storage_api::owner::ScannerPublicationCommitState;
use std::collections::HashMap;
use std::sync::atomic::AtomicBool;
use uuid::Uuid;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(super) enum DataUsagePersistOutcome {
@@ -34,12 +37,100 @@ fn remote_lease_expired(deadline: Option<std::time::Instant>) -> bool {
deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline)
}
pub(super) fn scanner_publication_scope_deadline(
persist_timeout: Duration,
remote_lease_deadline: Option<std::time::Instant>,
) -> tokio::time::Instant {
let configured_deadline = tokio::time::Instant::now() + persist_timeout;
remote_lease_deadline
.map(tokio::time::Instant::from_std)
.map_or(configured_deadline, |lease_deadline| configured_deadline.min(lease_deadline))
}
#[derive(Clone, Debug)]
pub(super) struct DataUsagePersistBaseline {
pub(super) data: Option<Bytes>,
pub(super) revision: DataUsageCacheRevision,
}
/// Read the bytes used as the baseline for a usage publication while keeping
/// the v2 primary revision as the CAS fence. During an interrupted upgrade the
/// primary can be valid JSON without a baseline identity; in that case a
/// same-or-newer durable companion may still be used, but an older legacy
/// snapshot must not cross the primary's epoch fence.
pub(super) async fn read_data_usage_persist_baseline(
storeapi: Arc<impl ScannerObjectIO>,
) -> Result<DataUsagePersistBaseline, EcstoreError> {
let (primary, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?;
let Some(primary) = primary else {
for path in [
format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()),
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
] {
let (candidate, _) = read_config_with_revision(storeapi.clone(), &path).await?;
let Some(candidate) = candidate else {
continue;
};
let Ok(usage) = serde_json::from_slice::<DataUsageInfo>(&candidate) else {
continue;
};
if data_usage_info_has_persisted_baseline_identity(&usage) {
return Ok(DataUsagePersistBaseline {
data: Some(Bytes::from(candidate)),
revision,
});
}
}
return Ok(DataUsagePersistBaseline { data: None, revision });
};
let Ok(primary_info) = serde_json::from_slice::<DataUsageInfo>(&primary) else {
// Preserve the original bytes and revision. A completed scan may
// replace the invalid primary under this CAS fence; an observation
// will still reject it below because it has no verifiable identity.
return Ok(DataUsagePersistBaseline {
data: Some(Bytes::from(primary)),
revision,
});
};
if data_usage_info_has_persisted_baseline_identity(&primary_info) || data_usage_info_is_bootstrap_pending(&primary_info) {
return Ok(DataUsagePersistBaseline {
data: Some(Bytes::from(primary)),
revision,
});
}
let invalid_primary_epoch = primary_info.scanner_epoch;
for path in [
format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()),
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
] {
let (candidate, _) = read_config_with_revision(storeapi.clone(), &path).await?;
let Some(candidate) = candidate else {
continue;
};
let Ok(usage) = serde_json::from_slice::<DataUsageInfo>(&candidate) else {
continue;
};
let candidate_epoch = usage.scanner_epoch.unwrap_or_default();
if data_usage_info_has_persisted_baseline_identity(&usage)
&& invalid_primary_epoch.is_none_or(|epoch| candidate_epoch >= epoch)
{
return Ok(DataUsagePersistBaseline {
data: Some(Bytes::from(candidate)),
revision,
});
}
}
Ok(DataUsagePersistBaseline {
data: Some(Bytes::from(primary)),
revision,
})
}
/// Short-lived publication inputs captured for one usage persistence attempt.
/// Keeping the movement epoch, lease deadline, and target fence together makes
/// it explicit that they are one proof rather than independent options.
@@ -48,6 +139,8 @@ pub(super) struct ScannerPublicationFence {
pub(super) expected_publication_epoch: Option<u64>,
pub(super) remote_lease_deadline: Option<std::time::Instant>,
pub(super) scanner_publication_lease_fence: Option<String>,
pub(super) remote_lease_tokens: Vec<Uuid>,
pub(super) lease_release_safe: Arc<AtomicBool>,
}
impl ScannerPublicationFence {
@@ -60,8 +153,20 @@ impl ScannerPublicationFence {
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence,
remote_lease_tokens: Vec::new(),
lease_release_safe: Arc::new(AtomicBool::new(true)),
}
}
pub(super) fn with_remote_lease_tokens(mut self, remote_lease_tokens: Vec<Uuid>) -> Self {
self.remote_lease_tokens = remote_lease_tokens;
self
}
pub(super) fn with_lease_release_flag(mut self, lease_release_safe: Arc<AtomicBool>) -> Self {
self.lease_release_safe = lease_release_safe;
self
}
}
#[derive(Debug)]
@@ -212,6 +317,8 @@ where
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence,
remote_lease_tokens,
lease_release_safe,
} = publication_fence;
let mut outcome = DataUsagePersistOutcome::NoUpdate;
let mut next_baseline = initial_baseline;
@@ -225,7 +332,7 @@ where
data_usage_info.scanner_epoch = Some(leader_epoch);
}
if remote_lease_expired(remote_lease_deadline) {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded);
break 'updates;
}
if let Some(expected_epoch) = expected_publication_epoch
@@ -281,8 +388,8 @@ where
publication_epoch = Some(read_epoch);
let authoritative_data = match next_baseline.as_ref() {
Some(baseline) => baseline.data.clone(),
None => match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await {
Ok((data, _)) => data.map(Bytes::from),
None => match read_data_usage_persist_baseline(storeapi.clone()).await {
Ok(baseline) => baseline.data,
Err(err) => {
error!(
target: "rustfs::scanner",
@@ -497,30 +604,59 @@ where
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
if remote_lease_expired(remote_lease_deadline) {
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded);
}
let done_save = Metrics::time(Metric::SaveUsage);
let save_result = {
let Some(_publication_admission) =
scanner_publication_admission_for_epoch(storeapi.clone(), publication_epoch_for_save).await
else {
done_save();
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
let publication_scope = storeapi
.scanner_data_usage_publication_commit_scope_with_release_flag(
publication_epoch_for_save,
scanner_publication_scope_deadline(data_usage_persist_timeout(), remote_lease_deadline),
remote_lease_tokens.clone(),
Arc::clone(&lease_release_safe),
)
.await;
let legacy_publication_admission = if publication_scope.is_none() {
let Some(admission) =
scanner_publication_admission_for_epoch(storeapi.clone(), publication_epoch_for_save).await
else {
done_save();
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
};
Some(admission)
} else {
None
};
if remote_lease_expired(remote_lease_deadline) {
done_save();
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded);
}
save_config_shared_with_preconditions_and_lease_fence(
let save_result = crate::save_config_shared_with_preconditions_and_lease_fence_and_scope(
storeapi.clone(),
target_path,
data.clone(),
sha256hex.clone(),
revision.preconditions(),
scanner_publication_lease_fence.as_deref(),
publication_scope.clone(),
)
.await
.await;
drop(legacy_publication_admission);
if let Some(scope) = publication_scope {
match scope.wait_for_completion().await {
ScannerPublicationCommitState::Committed | ScannerPublicationCommitState::AbortedBeforeCommit => {
save_result
}
ScannerPublicationCommitState::Indeterminate
| ScannerPublicationCommitState::Admitted
| ScannerPublicationCommitState::InFlight => Err(EcstoreError::other(
"scanner publication commit scope did not reach a safe terminal state",
)),
}
} else {
save_result
}
};
done_save();
@@ -618,6 +754,8 @@ where
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
&remote_lease_tokens,
Arc::clone(&lease_release_safe),
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
@@ -641,6 +779,8 @@ where
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
&remote_lease_tokens,
Arc::clone(&lease_release_safe),
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
@@ -683,6 +823,8 @@ where
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
&remote_lease_tokens,
Arc::clone(&lease_release_safe),
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
@@ -700,12 +842,14 @@ where
if backup_due {
let done_save = Metrics::time(Metric::SaveUsage);
let backup_result = sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence(
let backup_result = sync_data_usage_backup_from_primary_for_epoch_and_lease_and_fence_and_scope(
&ctx,
storeapi.clone(),
expected_publication_epoch,
remote_lease_deadline,
scanner_publication_lease_fence.as_deref(),
remote_lease_tokens.clone(),
Arc::clone(&lease_release_safe),
)
.await;
done_save();
@@ -739,6 +883,8 @@ async fn cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
expected_publication_epoch: Option<u64>,
remote_lease_deadline: Option<std::time::Instant>,
scanner_publication_lease_fence: Option<&str>,
remote_lease_tokens: &[Uuid],
lease_release_safe: Arc<AtomicBool>,
) -> bool {
if remote_lease_expired(remote_lease_deadline) {
return false;
@@ -807,7 +953,15 @@ async fn cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
return false;
}
let result = delete_config_with_publication_admission_for_epoch(
let publication_scope = storeapi
.scanner_data_usage_publication_commit_scope_with_release_flag(
read_epoch,
scanner_publication_scope_deadline(data_usage_persist_timeout(), remote_lease_deadline),
remote_lease_tokens.to_vec(),
Arc::clone(&lease_release_safe),
)
.await;
let result = crate::delete_config_with_publication_scope_for_epoch(
storeapi,
RUSTFS_META_BUCKET,
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
@@ -826,9 +980,23 @@ async fn cleanup_observed_data_usage_snapshot_for_epoch_and_lease(
..Default::default()
},
read_epoch,
publication_scope.clone(),
)
.await;
let result = if let Some(scope) = publication_scope {
match scope.wait_for_completion().await {
ScannerPublicationCommitState::Committed | ScannerPublicationCommitState::AbortedBeforeCommit => result,
ScannerPublicationCommitState::Indeterminate
| ScannerPublicationCommitState::Admitted
| ScannerPublicationCommitState::InFlight => Err(EcstoreError::other(
"scanner publication cleanup scope did not reach a safe terminal state",
)),
}
} else {
result
};
match result {
Ok(_)
| Err(
+24
View File
@@ -269,6 +269,10 @@ fn should_publish_usage_snapshot(status: ScannerCycleStatus) -> bool {
matches!(status, ScannerCycleStatus::Complete | ScannerCycleStatus::Superseded)
}
fn should_publish_observational_snapshot(status: ScannerCycleStatus) -> bool {
matches!(status, ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable))
}
fn prepare_usage_snapshot_for_publication(
status: ScannerCycleStatus,
mut data_usage_info: DataUsageInfo,
@@ -570,6 +574,13 @@ pub(crate) async fn scanner_set_disk_inventory(set: &SetDisks) -> Vec<Arc<Disk>>
pub(crate) enum ScannerCycleDeferReason {
ActivityBaselineUnavailable,
DataMovement,
/// A granted lease's absolute deadline cannot cover the persistence
/// operation. This can occur even when the configured budget fits the
/// nominal TTL because lease acquisition consumed part of the window.
PublicationLeaseDeadlineExceeded,
/// A remote lease could not be released after the persistence attempt.
/// Keep the cycle deferred because the peer may still admit movement.
PublicationLeaseReleaseFailed,
}
impl ScannerCycleDeferReason {
@@ -577,6 +588,8 @@ impl ScannerCycleDeferReason {
match self {
Self::ActivityBaselineUnavailable => "activity_baseline_unavailable",
Self::DataMovement => "data_movement",
Self::PublicationLeaseDeadlineExceeded => "publication_lease_deadline_exceeded",
Self::PublicationLeaseReleaseFailed => "publication_lease_release_failed",
}
}
}
@@ -611,6 +624,7 @@ fn scanner_activity_preflight(
pub(crate) struct ScannerCycleResult {
pub(crate) status: ScannerCycleStatus,
publication_epoch: Option<u64>,
observational_snapshot_published: bool,
dirty_usage_clear: Option<DirtyUsageBuckets>,
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
remote_publication_lease_targets: Vec<(String, String, u64)>,
@@ -624,6 +638,7 @@ impl ScannerCycleResult {
Self {
status,
publication_epoch: None,
observational_snapshot_published: false,
dirty_usage_clear,
remote_dirty_usage_acknowledgements: Vec::new(),
remote_publication_lease_targets: Vec::new(),
@@ -642,6 +657,15 @@ impl ScannerCycleResult {
self.publication_epoch
}
pub(crate) fn with_observational_snapshot_published(mut self, published: bool) -> Self {
self.observational_snapshot_published = published;
self
}
pub(crate) fn has_observational_snapshot(&self) -> bool {
self.observational_snapshot_published
}
fn with_failed_dirty_usage(mut self, failed_dirty_usage: bool) -> Self {
self.failed_dirty_usage = failed_dirty_usage;
self
+91
View File
@@ -286,6 +286,16 @@ pub(super) fn scanner_task_join_error(stage: &str, err: tokio::task::JoinError)
mod tests {
use super::*;
use rustfs_scanner_contracts::metrics::{ScannerWorkSource, global_metrics};
use tokio::sync::oneshot;
fn active_bucket_drive_count(source: ScannerWorkSource, bucket: &str, drive: &str) -> u64 {
global_metrics()
.scanner_runtime_details_report()
.active_bucket_drive_scans
.into_iter()
.find(|active| active.source == source.as_str() && active.bucket == bucket && active.drive == drive)
.map_or(0, |active| active.count)
}
#[test]
fn bucket_drive_failure_guard_retires_active_scan_on_drop() {
@@ -305,4 +315,85 @@ mod tests {
.any(|active| active.source == source.as_str() && active.bucket == bucket && active.drive == drive)
);
}
#[tokio::test]
async fn bucket_drive_failure_guard_retires_active_scan_after_cancellation() {
let source = ScannerWorkSource::Usage;
let bucket = "__guard_cancel_lifecycle_test__";
let drive = "/__guard_cancel_lifecycle_test__";
global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
let cancellation = CancellationToken::new();
let worker_cancellation = cancellation.clone();
let worker = tokio::spawn(async move {
let mut guard = BucketDriveFailureGuard::new(source, bucket, drive);
worker_cancellation.cancelled().await;
guard.mark_not_failed();
});
cancellation.cancel();
worker.await.expect("cancelled scanner worker should finish");
assert_eq!(active_bucket_drive_count(source, bucket, drive), 0);
}
#[tokio::test]
async fn bucket_drive_failure_guard_retires_active_scan_when_worker_is_aborted() {
let source = ScannerWorkSource::Bitrot;
let bucket = "__guard_abort_lifecycle_test__";
let drive = "/__guard_abort_lifecycle_test__";
global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
let (started_sender, started_receiver) = oneshot::channel();
let worker = tokio::spawn(async move {
let _guard = BucketDriveFailureGuard::new(source, bucket, drive);
started_sender.send(()).expect("test should observe worker start");
std::future::pending::<()>().await;
});
started_receiver.await.expect("scanner worker should start");
assert_eq!(active_bucket_drive_count(source, bucket, drive), 1);
worker.abort();
worker.await.expect_err("aborted scanner worker should report cancellation");
assert_eq!(active_bucket_drive_count(source, bucket, drive), 0);
}
#[tokio::test]
async fn bucket_drive_failure_guards_track_overlapping_scans_independently() {
let source = ScannerWorkSource::Usage;
let bucket = "__guard_overlap_lifecycle_test__";
let drive = "/__guard_overlap_lifecycle_test__";
global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
global_metrics().record_scan_bucket_drive_start(source, bucket, drive);
let (first_release_sender, first_release_receiver) = oneshot::channel();
let (second_release_sender, second_release_receiver) = oneshot::channel();
let (first_started_sender, first_started_receiver) = oneshot::channel();
let (second_started_sender, second_started_receiver) = oneshot::channel();
let first = tokio::spawn(async move {
let _guard = BucketDriveFailureGuard::new(source, bucket, drive);
first_started_sender.send(()).expect("test should observe first worker start");
first_release_receiver.await.expect("first worker should be released");
});
let second = tokio::spawn(async move {
let _guard = BucketDriveFailureGuard::new(source, bucket, drive);
second_started_sender
.send(())
.expect("test should observe second worker start");
second_release_receiver.await.expect("second worker should be released");
});
first_started_receiver.await.expect("first scanner worker should start");
second_started_receiver.await.expect("second scanner worker should start");
assert_eq!(active_bucket_drive_count(source, bucket, drive), 2);
first_release_sender.send(()).expect("first worker should be released");
first.await.expect("first scanner worker should finish");
assert_eq!(active_bucket_drive_count(source, bucket, drive), 1);
second_release_sender.send(()).expect("second worker should be released");
second.await.expect("second scanner worker should finish");
assert_eq!(active_bucket_drive_count(source, bucket, drive), 0);
}
}
+24 -16
View File
@@ -162,18 +162,18 @@ impl ScannerIOCycle for ECStore {
dirty_usage_status,
activity_status,
);
if !publish_usage_snapshot(
&updates,
status,
DataUsageInfo {
last_update: Some(SystemTime::now()),
scanner_cycle: Some(want_cycle),
usage_snapshot_complete: true,
..Default::default()
},
)
.await?
{
let empty_usage = DataUsageInfo {
last_update: Some(SystemTime::now()),
scanner_cycle: Some(want_cycle),
usage_snapshot_complete: true,
..Default::default()
};
let observational_snapshot_published = if should_publish_observational_snapshot(status) {
publish_observational_snapshot(&updates, empty_usage).await?
} else {
publish_usage_snapshot(&updates, status, empty_usage).await?
};
if !observational_snapshot_published {
return Ok(ScannerCycleResult::new(status, None).with_publication_epoch(publication_epoch));
}
if status == ScannerCycleStatus::Complete {
@@ -188,6 +188,7 @@ impl ScannerIOCycle for ECStore {
};
return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_observational_snapshot_published(observational_snapshot_published)
.with_remote_publication_lease_targets(remote_publication_lease_targets)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements));
}
@@ -437,13 +438,19 @@ impl ScannerIOCycle for ECStore {
dirty_usage_status,
activity_status,
);
if let Some((data_usage_info, _)) = completed_usage {
publish_usage_snapshot(&updates, cycle_status, data_usage_info).await?;
let observational_snapshot_published = if let Some((data_usage_info, _)) = completed_usage {
if should_publish_observational_snapshot(cycle_status) {
publish_observational_snapshot(&updates, data_usage_info).await?
} else {
publish_usage_snapshot(&updates, cycle_status, data_usage_info).await?
}
} else if !ctx.is_cancelled()
&& let Some((data_usage_info, _)) = observational_usage
{
publish_observational_snapshot(&updates, data_usage_info).await?;
}
publish_observational_snapshot(&updates, data_usage_info).await?
} else {
false
};
let dirty_usage_clear = should_clear_dirty_usage_snapshot(
result.is_ok(),
structurally_complete_snapshot,
@@ -463,6 +470,7 @@ impl ScannerIOCycle for ECStore {
};
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_observational_snapshot_published(observational_snapshot_published)
.with_remote_publication_lease_targets(remote_publication_lease_targets)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)
.with_failed_dirty_usage(!failed_buckets.is_empty())
+41
View File
@@ -909,6 +909,47 @@ async fn structurally_complete_superseded_cycles_publish_without_claiming_conver
);
}
#[tokio::test]
async fn post_scan_activity_failure_retains_complete_usage_as_observation() {
let (updates, mut receiver) = mpsc::channel(1);
let status = ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
assert!(should_publish_observational_snapshot(status));
assert!(
publish_observational_snapshot(
&updates,
DataUsageInfo {
last_update: Some(SystemTime::now()),
scanner_cycle: Some(7),
objects_total_count: 3,
objects_total_size: 12,
usage_snapshot_complete: true,
..Default::default()
},
)
.await
.expect("post-scan activity failure should retain an observation")
);
let observed = receiver.recv().await.expect("observational update should be queued");
assert!(!observed.usage_snapshot_complete);
assert!(observed.usage_snapshot_partial);
assert_eq!(observed.usage_snapshot_converged, Some(false));
assert_eq!(observed.objects_total_count, 3);
assert_eq!(observed.objects_total_size, 12);
}
#[test]
fn only_unverified_activity_allows_post_scan_observation() {
assert!(should_publish_observational_snapshot(ScannerCycleStatus::Deferred(
ScannerCycleDeferReason::ActivityBaselineUnavailable
)));
assert!(!should_publish_observational_snapshot(ScannerCycleStatus::Deferred(
ScannerCycleDeferReason::DataMovement
)));
assert!(!should_publish_observational_snapshot(ScannerCycleStatus::Incomplete));
}
#[test]
fn scanner_cycle_fails_closed_for_namespace_disappearance() {
for activity_status in [ScannerCycleActivityStatus::Changed, ScannerCycleActivityStatus::Unchanged] {
+5 -3
View File
@@ -92,7 +92,9 @@ pub(crate) use rustfs_ecstore::api::event::{EventArgs as EcstoreEventArgs, send_
pub(crate) use rustfs_ecstore::api::layout::{
EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints,
};
pub(crate) use rustfs_ecstore::api::object::SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY;
pub(crate) use rustfs_ecstore::api::object::{
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitState,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rebalance::{
RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta,
@@ -106,9 +108,9 @@ pub(crate) use rustfs_ecstore::api::runtime::{
setup_is_erasure_sd as ecstore_is_erasure_sd,
};
pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::storage::init_local_disks_with_instance_ctx as ecstore_init_local_disks_with_instance_ctx;
pub(crate) use rustfs_ecstore::api::storage::{ECStore as EcstoreStore, SCANNER_PUBLICATION_LEASE_TTL_MS};
use rustfs_storage_api as storage_contracts;
pub(crate) mod owner {
@@ -125,7 +127,7 @@ pub(crate) mod owner {
EcstoreNsScannerOpenRequest, EcstoreObjectLockConfiguration, EcstoreObjectOpts, EcstoreReplicationConfigurationExt,
EcstoreReplicationScannerBridge, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore,
EcstoreVersioningApi, EcstoreVersioningConfiguration, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY,
SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerReplicationHealObject, ScannerReplicationHealResult,
ScannerPublicationCommitScope, ScannerPublicationCommitState, ScannerReplicationHealObject, ScannerReplicationHealResult,
ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_expiry_state_handle,
ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config, ecstore_get_object_lock_config,
ecstore_get_replication_config, ecstore_invalidate_admin_data_usage_snapshot_cache,
+48 -34
View File
@@ -33,6 +33,7 @@ use rustfs_io_metrics::internode_metrics::{
use rustfs_protos::proto_gen::node_service::*;
use serde::de::DeserializeOwned;
use std::io::Cursor;
use std::sync::Arc;
use std::time::Instant;
use tonic::{Request, Response, Status};
use tracing::debug;
@@ -1230,37 +1231,40 @@ impl NodeService {
// The target owns this read guard. It must span the complete
// disk rename, not merely the preflight, so a movement transition
// cannot restart after validation and before rename linearization.
let _scanner_publication_lease_guard = if let Some(token) = scanner_publication_lease_token {
let Some(store) = self.resolve_object_store() else {
return Ok(Response::new(RenameDataResponse {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Vec::new().into(),
error: Some(DiskError::other("scanner publication lease owner is unavailable").into()),
}));
};
match store.acquire_scanner_publication_lease_guard(token).await {
Ok(guard) => Some(guard),
Err(err) => {
let scanner_publication_lease_guard: Option<Arc<dyn Send + Sync>> =
if let Some(token) = scanner_publication_lease_token {
let Some(store) = self.resolve_object_store() else {
return Ok(Response::new(RenameDataResponse {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Vec::new().into(),
error: Some(DiskError::other(err.to_string()).into()),
error: Some(DiskError::other("scanner publication lease owner is unavailable").into()),
}));
};
match store.acquire_scanner_publication_lease_guard(token).await {
Ok(guard) => Some(Arc::new(guard)),
Err(err) => {
return Ok(Response::new(RenameDataResponse {
success: false,
rename_data_resp: String::new(),
rename_data_resp_bin: Vec::new().into(),
error: Some(DiskError::other(err.to_string()).into()),
}));
}
}
}
} else {
None
};
} else {
None
};
let request_decoded_from_msgpack = decoded_file_info.from_msgpack;
match disk
.rename_data(
.rename_data_borrowed_with_fence_and_guard(
&request.src_volume,
&request.src_path,
&decoded_file_info.value,
&request.dst_volume,
&request.dst_path,
scanner_publication_lease_token,
scanner_publication_lease_guard,
)
.await
{
@@ -1641,26 +1645,36 @@ impl NodeService {
// The target-side guard spans the complete delete operation. A
// lease expiry or movement transition cannot occur between this
// validation and the disk delete linearization point.
let _scanner_publication_lease_guard = if let Some(token) = scanner_publication_lease_token {
let Some(store) = self.resolve_object_store() else {
return Ok(Response::new(DeleteResponse {
success: false,
error: Some(DiskError::other("scanner publication lease owner is unavailable").into()),
}));
};
match store.acquire_scanner_publication_lease_guard(token).await {
Ok(guard) => Some(guard),
Err(err) => {
let scanner_publication_lease_guard: Option<Arc<dyn Send + Sync>> =
if let Some(token) = scanner_publication_lease_token {
let Some(store) = self.resolve_object_store() else {
return Ok(Response::new(DeleteResponse {
success: false,
error: Some(DiskError::other(err.to_string()).into()),
error: Some(DiskError::other("scanner publication lease owner is unavailable").into()),
}));
};
match store.acquire_scanner_publication_lease_guard(token).await {
Ok(guard) => Some(Arc::new(guard)),
Err(err) => {
return Ok(Response::new(DeleteResponse {
success: false,
error: Some(DiskError::other(err.to_string()).into()),
}));
}
}
}
} else {
None
};
match disk.delete(&request.volume, &request.path, options).await {
} else {
None
};
match disk
.delete_with_scanner_publication_lease_and_guard(
&request.volume,
&request.path,
options,
scanner_publication_lease_token,
scanner_publication_lease_guard,
)
.await
{
Ok(_) => Ok(Response::new(DeleteResponse {
success: true,
error: None,