// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use super::*; /// Grace window during which a recently modified object is never deleted as /// dangling. 0 disables the grace window. const ENV_HEAL_DANGLING_DELETE_GRACE_SECS: &str = "RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS"; const DEFAULT_HEAL_DANGLING_DELETE_GRACE_SECS: u64 = 3600; fn dangling_delete_grace() -> time::Duration { let secs = rustfs_utils::get_env_u64(ENV_HEAL_DANGLING_DELETE_GRACE_SECS, DEFAULT_HEAL_DANGLING_DELETE_GRACE_SECS); time::Duration::seconds(i64::try_from(secs).unwrap_or(i64::MAX)) } /// Result of scanning one disk's copy of a directory prefix while deciding /// whether an orphan (metadata-less) directory tree can be safely purged. enum OrphanDirScan { /// The subtree holds at least one regular file (object metadata or data), so /// it is a real object and must not be purged. HasData, /// The prefix exists on this disk and contains only nested empty directories. /// Carries every directory path in pre-order (parents before children). Empty(Vec), /// The prefix does not exist on this disk. Missing, } impl SetDisks { pub(super) fn default_read_quorum(&self) -> usize { self.set_drive_count - self.default_parity_count } pub(super) fn default_write_quorum(&self) -> usize { let mut data_count = self.set_drive_count - self.default_parity_count; if data_count == self.default_parity_count { data_count += 1 } data_count } #[tracing::instrument(level = "debug", skip(disks, file_infos))] #[allow(clippy::type_complexity)] pub(super) async fn rename_data( disks: &[Option], src_bucket: &str, src_object: &str, file_infos: &[FileInfo], dst_bucket: &str, dst_object: &str, write_quorum: usize, ) -> disk::error::Result<(Vec>, Option>, Option, Vec>)> { let mut futures = Vec::with_capacity(disks.len()); let mut errs = Vec::with_capacity(disks.len()); let src_bucket = Arc::new(src_bucket.to_string()); let src_object = Arc::new(src_object.to_string()); let dst_bucket = Arc::new(dst_bucket.to_string()); let dst_object = Arc::new(dst_object.to_string()); for (i, (disk, file_info)) in disks.iter().zip(file_infos.iter()).enumerate() { let mut file_info = file_info.clone(); let disk = disk.clone(); let src_bucket = src_bucket.clone(); let src_object = src_object.clone(); let dst_object = dst_object.clone(); let dst_bucket = dst_bucket.clone(); futures.push(tokio::spawn(async move { if file_info.erasure.index == 0 { file_info.erasure.index = i + 1; } if !file_info.is_valid() { return Err(DiskError::FileCorrupt); } if let Some(disk) = disk { disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object) .await } else { Err(DiskError::DiskNotFound) } })); } let mut disk_versions = vec![None; disks.len()]; let mut data_dirs = vec![None; disks.len()]; let results = join_all(futures).await; for (idx, result) in results.iter().enumerate() { match result.as_ref().map_err(|_| DiskError::Unexpected)? { Ok(res) => { data_dirs[idx] = res.old_data_dir; disk_versions[idx].clone_from(&res.sign); errs.push(None); } Err(e) => { errs.push(Some(e.clone())); } } } if issue3031_diag_enabled() { let success_count = errs.iter().filter(|err| err.is_none()).count(); let failure_count = errs.len().saturating_sub(success_count); let ignored_failure_count = errs .iter() .filter(|err| err.as_ref().is_some_and(|err| OBJECT_OP_IGNORED_ERRS.contains(err))) .count(); let data_dir_vote_count = data_dirs.iter().filter(|data_dir| data_dir.is_some()).count(); let reduced_data_dir = Self::reduce_common_data_dir(&data_dirs, write_quorum); warn!( target: "rustfs_ecstore::set_disk", src_bucket = %src_bucket, src_object = %src_object, dst_bucket = %dst_bucket, dst_object = %dst_object, write_quorum, disk_count = errs.len(), success_count, failure_count, ignored_failure_count, data_dir_vote_count, reduced_data_dir = ?reduced_data_dir, errs = ?errs, data_dirs = ?data_dirs, "issue3031_rename_data_quorum_context" ); } let mut futures = Vec::with_capacity(disks.len()); if let Some(ret_err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { // TODO: add concurrency for (i, err) in errs.iter().enumerate() { if err.is_some() { continue; } if let Some(disk) = disks[i].as_ref() { let fi = file_infos[i].clone(); let old_data_dir = data_dirs[i]; let disk = disk.clone(); let dst_bucket = dst_bucket.clone(); let dst_object = dst_object.clone(); futures.push(tokio::spawn(async move { disk.delete_version( &dst_bucket, &dst_object, fi, false, DeleteOptions { undo_write: true, old_data_dir, ..Default::default() }, ) .await })); } } if issue3031_diag_enabled() { warn!( target: "rustfs_ecstore::set_disk", src_bucket = %src_bucket, src_object = %src_object, dst_bucket = %dst_bucket, dst_object = %dst_object, write_quorum, ret_err = %ret_err, errs = ?errs, data_dirs = ?data_dirs, "issue3031_rename_data_quorum_failed" ); } let undo_results = join_all(futures).await; let undo_error_count = undo_results .iter() .filter(|result| match result { Err(_) | Ok(Err(_)) => true, Ok(Ok(_)) => false, }) .count(); if undo_error_count > 0 { warn!( target: "rustfs_ecstore::set_disk", dst_bucket = %dst_bucket, dst_object = %dst_object, undo_error_count, "rename_data quorum rollback reported errors" ); } return Err(ret_err); } let versions = None; // TODO: reduceCommonVersions let data_dir = Self::reduce_common_data_dir(&data_dirs, write_quorum); let online_disks = Self::eval_disks(disks, &errs); let cleanup_disks = if let Some(data_dir) = data_dir { disks .iter() .zip(errs.iter()) .zip(data_dirs.iter()) .map(|((disk, err), old_data_dir)| { if err.is_none() && *old_data_dir == Some(data_dir) { disk.clone() } else { None } }) .collect() } else { vec![None; disks.len()] }; // // TODO: reduce_common_data_dir // if let Some(old_dir) = rename_ress // .iter() // .filter_map(|v| if v.is_some() { v.as_ref().unwrap().old_data_dir } else { None }) // .map(|v| v.to_string()) // .next() // { // let cm_errs = self.commit_rename_data_dir(&shuffle_disks, &bucket, &object, &old_dir).await; // warn!("put_object commit_rename_data_dir:{:?}", &cm_errs); // } // self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?; Ok((online_disks, versions, data_dir, cleanup_disks)) } #[allow(dead_code)] #[tracing::instrument(level = "debug", skip(self, disks))] pub(super) async fn commit_rename_data_dir( &self, disks: &[Option], bucket: &str, object: &str, data_dir: &str, write_quorum: usize, ) -> disk::error::Result<()> { let file_path = Arc::new(format!("{object}/{data_dir}")); let bucket = Arc::new(bucket.to_string()); let futures = disks.iter().map(|disk| { let file_path = file_path.clone(); let bucket = bucket.clone(); let disk = disk.clone(); tokio::spawn(async move { if let Some(disk) = disk { (disk .delete( &bucket, &file_path, DeleteOptions { recursive: true, ..Default::default() }, ) .await) .err() } else { Some(DiskError::DiskNotFound) } }) }); let errs: Vec> = join_all(futures) .await .into_iter() .map(|e| e.unwrap_or(Some(DiskError::Unexpected))) .collect(); if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { return Err(err); } Ok(()) } #[tracing::instrument(skip(self))] pub(super) async fn cleanup_multipart_path(&self, paths: &[String]) { if paths.is_empty() { return; } let disks = self.get_disks_internal().await; let mut errs = Vec::with_capacity(disks.len()); // Use improved simple batch processor instead of join_all for better performance let processor = runtime_sources::batch_processors().write_processor(); let tasks: Vec<_> = disks .iter() .map(|disk| { let disk = disk.clone(); let paths = paths.to_vec(); async move { if let Some(disk) = disk { disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &paths).await } else { Err(DiskError::DiskNotFound) } } }) .collect(); let results = processor.execute_batch(tasks).await; for result in results { match result { Ok(_) => { errs.push(None); } Err(e) => { errs.push(Some(e)); } } } if errs.iter().any(|e| e.is_some()) { warn!("cleanup_multipart_path errs {:?}", &errs); } } #[tracing::instrument(skip(disks, meta))] #[allow(clippy::too_many_arguments)] pub(super) async fn rename_part( &self, disks: &[Option], src_bucket: &str, src_object: &str, dst_bucket: &str, dst_object: &str, meta: Bytes, write_quorum: usize, quorum_context: Option>, ) -> disk::error::Result>> { let src_bucket = Arc::new(src_bucket.to_string()); let src_object = Arc::new(src_object.to_string()); let dst_bucket = Arc::new(dst_bucket.to_string()); let dst_object = Arc::new(dst_object.to_string()); // Match MinIO's multipart overwrite semantics: clear any stale destination // part payload and metadata before the new per-disk rename fan-out begins. self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")]) .await; let mut errs = Vec::with_capacity(disks.len()); let futures = disks.iter().map(|disk| { let disk = disk.clone(); let meta = meta.clone(); let src_bucket = src_bucket.clone(); let src_object = src_object.clone(); let dst_bucket = dst_bucket.clone(); let dst_object = dst_object.clone(); tokio::spawn(async move { if let Some(disk) = disk { disk.rename_part(&src_bucket, &src_object, &dst_bucket, &dst_object, meta) .await } else { Err(DiskError::DiskNotFound) } }) }); let results = join_all(futures).await; for result in results { match result? { Ok(_) => { errs.push(None); } Err(e) => { errs.push(Some(e)); } } } if issue3031_diag_enabled() { let success_count = errs.iter().filter(|err| err.is_none()).count(); let error_count = errs.len().saturating_sub(success_count); let disk_not_found_count = errs.iter().filter(|err| matches!(err, Some(DiskError::DiskNotFound))).count(); let file_not_found_count = errs.iter().filter(|err| matches!(err, Some(DiskError::FileNotFound))).count(); warn!( target: "rustfs_ecstore::set_disk", src_bucket = %src_bucket, src_object = %src_object, dst_bucket = %dst_bucket, dst_object = %dst_object, write_quorum = write_quorum, disk_count = errs.len(), success_count = success_count, error_count = error_count, disk_not_found_count = disk_not_found_count, file_not_found_count = file_not_found_count, "issue3031_rename_part_context" ); } if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { if let Some(context) = quorum_context { log_multipart_write_quorum_failure(context, &errs, write_quorum, &err); } else { warn!("rename_part errs {:?}", &errs); } self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")]) .await; return Err(err); } let disks = Self::eval_disks(disks, &errs); Ok(disks) } pub(super) fn eval_disks(disks: &[Option], errs: &[Option]) -> Vec> { if disks.len() != errs.len() { return Vec::new(); } let mut online_disks = vec![None; disks.len()]; for (i, err_op) in errs.iter().enumerate() { if err_op.is_none() { online_disks[i].clone_from(&disks[i]); } } online_disks } #[tracing::instrument(skip(disks, files))] pub(super) async fn write_unique_file_info( disks: &[Option], org_bucket: &str, bucket: &str, prefix: &str, files: &[FileInfo], write_quorum: usize, ) -> disk::error::Result<()> { let mut futures = Vec::with_capacity(disks.len()); let mut errs = Vec::with_capacity(disks.len()); for (i, disk) in disks.iter().enumerate() { let mut file_info = files[i].clone(); file_info.erasure.index = i + 1; futures.push(async move { if let Some(disk) = disk { disk.write_metadata(org_bucket, bucket, prefix, file_info).await } else { Err(DiskError::DiskNotFound) } }); } let results = join_all(futures).await; for result in results { match result { Ok(_) => { errs.push(None); } Err(e) => { errs.push(Some(e)); } } } if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { let mut revert_futures = Vec::with_capacity(disks.len()); for (i, err) in errs.iter().enumerate() { if err.is_some() { continue; } if let Some(disk) = disks[i].as_ref() { let disk = disk.clone(); let bucket = bucket.to_string(); let path = path_join_buf(&[prefix, STORAGE_FORMAT_FILE]); revert_futures.push(async move { if let Err(err) = disk .delete( &bucket, &path, DeleteOptions { recursive: true, ..Default::default() }, ) .await { warn!("write meta revert err {:?}", err); } }); } } join_all(revert_futures).await; return Err(err); } Ok(()) } pub(super) async fn update_object_meta( &self, bucket: &str, object: &str, fi: FileInfo, disks: &[Option], ) -> disk::error::Result<()> { self.update_object_meta_with_opts(bucket, object, fi, disks, &UpdateMetadataOpts::default()) .await } pub(super) async fn update_object_meta_with_opts( &self, bucket: &str, object: &str, fi: FileInfo, disks: &[Option], opts: &UpdateMetadataOpts, ) -> disk::error::Result<()> { if fi.metadata.is_empty() && !opts.replace_user_metadata { return Ok(()); } self.invalidate_get_object_metadata_cache(bucket, object).await; let mut futures = Vec::with_capacity(disks.len()); let mut errs = Vec::with_capacity(disks.len()); for disk in disks.iter() { let fi = fi.clone(); futures.push(async move { if let Some(disk) = disk { disk.update_metadata(bucket, object, fi, opts).await } else { Err(DiskError::DiskNotFound) } }) } let results = join_all(futures).await; for result in results { match result { Ok(_) => { errs.push(None); } Err(e) => { errs.push(Some(e)); } } } if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, fi.write_quorum(self.default_write_quorum())) { return Err(err); } self.invalidate_get_object_metadata_cache(bucket, object).await; Ok(()) } pub(super) async fn delete_if_dangling( &self, bucket: &str, object: &str, meta_arr: &[FileInfo], errs: &[Option], data_errs_by_part: &HashMap>, opts: ObjectOptions, ) -> disk::error::Result { let (m, can_heal) = is_object_dangling(meta_arr, errs, data_errs_by_part); if !can_heal { return Err(DiskError::ErasureReadQuorum); } // Recently written objects get a grace window before dangling cleanup: after // an unclean shutdown some disks may still be catching up (or carry writes // that were never made durable), and deleting the surviving shards right away // turns a partial loss into a total one. Skip deletion and leave the object // for a later heal/scanner pass to re-evaluate. if m.is_valid() && let Some(mod_time) = m.mod_time { let grace = dangling_delete_grace(); if !grace.is_zero() && OffsetDateTime::now_utc() - mod_time < grace { info!( bucket = bucket, object = object, mod_time = %mod_time, grace_secs = grace.whole_seconds(), "skipping dangling-object deletion within grace window" ); return Err(DiskError::ErasureReadQuorum); } } let mut tags: HashMap = HashMap::new(); tags.insert("set".to_string(), self.set_index.to_string()); tags.insert("pool".to_string(), self.pool_index.to_string()); tags.insert("merrs".to_string(), join_errs(errs)); tags.insert("derrs".to_string(), format!("{data_errs_by_part:?}")); if m.is_valid() { tags.insert("sz".to_string(), m.size.to_string()); tags.insert( "mt".to_string(), m.mod_time .as_ref() .map_or(String::new(), |mod_time| mod_time.unix_timestamp().to_string()), ); tags.insert("d:p".to_string(), format!("{}:{}", m.erasure.data_blocks, m.erasure.parity_blocks)); } else { tags.insert("invalid".to_string(), "1".to_string()); tags.insert( "d:p".to_string(), format!("{}:{}", self.set_drive_count - self.default_parity_count, self.default_parity_count), ); } let mut offline = 0; for (i, err) in errs.iter().enumerate() { let mut found = false; if let Some(err) = err && err == &DiskError::DiskNotFound { found = true; } for p in data_errs_by_part { if let Some(v) = p.1.get(i) && *v == CHECK_PART_DISK_NOT_FOUND { found = true; break; } } if found { offline += 1; } } if offline > 0 { tags.insert("offline".to_string(), offline.to_string()); } let mut fi = FileInfo::default(); if let Some(ref version_id) = opts.version_id { fi.version_id = Uuid::parse_str(version_id).ok(); } fi.set_tier_free_version_id(&Uuid::new_v4().to_string()); let disks = self.get_disks_internal().await; let mut futures = Vec::with_capacity(disks.len()); for disk_op in disks.iter() { let bucket = bucket.to_string(); let object = object.to_string(); let fi = fi.clone(); futures.push(async move { if let Some(disk) = disk_op { disk.delete_version(&bucket, &object, fi, false, DeleteOptions::default()) .await } else { Err(DiskError::DiskNotFound) } }); } let results = join_all(futures).await; let mut delete_errs = Vec::with_capacity(results.len()); for (index, result) in results.into_iter().enumerate() { let key = format!("ddisk-{index}"); let already_absent = matches!( errs.get(index).and_then(Option::as_ref), Some(DiskError::FileNotFound | DiskError::FileVersionNotFound) ); match result { Ok(_) => { tags.insert(key, "".to_string()); delete_errs.push(None); } Err(e) => { tags.insert(key, e.to_string()); if already_absent || matches!(&e, DiskError::FileNotFound | DiskError::FileVersionNotFound) { delete_errs.push(None); } else { delete_errs.push(Some(e)); } } } } let write_quorum = if m.is_valid() { m.write_quorum(self.default_write_quorum()) } else { self.default_write_quorum() }; if let Some(err) = reduce_write_quorum_errs(&delete_errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { return Err(err); } Ok(m) } pub(super) async fn delete_prefix(&self, bucket: &str, prefix: &str) -> disk::error::Result<()> { let disks = self.get_disks_internal().await; let write_quorum = disks.len() / 2 + 1; let mut futures = Vec::with_capacity(disks.len()); for disk_op in disks.iter() { let bucket = bucket.to_string(); let prefix = prefix.to_string(); futures.push(async move { if let Some(disk) = disk_op { disk.delete( &bucket, &prefix, DeleteOptions { recursive: true, immediate: true, ..Default::default() }, ) .await } else { Ok(()) } }); } let errs = join_all(futures).await.into_iter().map(|v| v.err()).collect::>(); if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { return Err(err); } Ok(()) } /// Scan a single disk's copy of `prefix` and decide whether it is an orphan /// (metadata-less) directory subtree. Walks the tree iteratively and returns /// [`OrphanDirScan::HasData`] as soon as any regular file is found. async fn scan_orphan_dir(disk: &DiskStore, bucket: &str, prefix: &str) -> OrphanDirScan { let root = prefix.trim_end_matches(SLASH_SEPARATOR).to_string(); let mut stack = vec![root.clone()]; // Pre-order list of directories (a parent always precedes its descendants), // so reversing it yields a safe children-first removal order. let mut dirs: Vec = Vec::new(); let mut existed = false; while let Some(dir) = stack.pop() { let entries = match disk.list_dir("", bucket, &dir, 0).await { Ok(entries) => entries, Err(_) => { // The root missing (or never existing) means there is nothing to // purge on this disk. A nested directory vanishing mid-scan is a // benign race, so skip it and keep walking. if dir == root { return OrphanDirScan::Missing; } continue; } }; existed = true; dirs.push(dir.clone()); for entry in entries { match entry.strip_suffix(SLASH_SEPARATOR) { // `read_dir` marks directories with a trailing slash; anything else // is a regular file, which means real object data lives here. Some(child) => stack.push(format!("{dir}{SLASH_SEPARATOR}{child}")), None => return OrphanDirScan::HasData, } } } if existed { OrphanDirScan::Empty(dirs) } else { OrphanDirScan::Missing } } /// Purge an orphan directory prefix — a trailing-slash key that exists on disk /// as an empty directory tree with no object metadata on any disk of this set. /// Such prefixes are listable (see `scan_dir`) yet are not real objects, so the /// normal delete path returns NotFound and leaves them stranded (issue #4189). /// /// Callers pass the *decoded* directory name (`prefix/`), not the `__XLDIR__` /// encoded object key — the orphan tree on disk uses the plain path. /// /// Returns `Ok(true)` when the prefix was an orphan tree on this set and was /// removed, `Ok(false)` when it holds real data or does not exist on any disk /// of this set (the caller should surface the original NotFound), and `Err` on /// a hard disk failure. pub(crate) async fn purge_orphan_dir_object(&self, bucket: &str, object: &str) -> disk::error::Result { let disks = self.get_disks_internal().await; // Phase 1: classify every online disk. Refuse to purge if ANY disk holds // object data under the prefix, so a degraded/healable object is never // destroyed. let mut per_disk_dirs: Vec<(usize, Vec)> = Vec::new(); let mut existed = false; for (i, disk) in disks.iter().enumerate() { let Some(disk) = disk else { continue }; match Self::scan_orphan_dir(disk, bucket, object).await { OrphanDirScan::HasData => return Ok(false), OrphanDirScan::Empty(dirs) => { existed = true; per_disk_dirs.push((i, dirs)); } OrphanDirScan::Missing => {} } } if !existed { return Ok(false); } // Phase 2: remove the empty directories children-first on each disk. A // non-recursive delete performs an empty-only `rmdir`, so a directory that // concurrently gained an object fails with DirectoryNotEmpty and is skipped — // a racing PutObject is never clobbered. for (i, mut dirs) in per_disk_dirs { let Some(disk) = disks[i].as_ref() else { continue }; dirs.reverse(); for dir in dirs { if let Err(err) = disk .delete( bucket, &dir, DeleteOptions { recursive: false, immediate: true, ..Default::default() }, ) .await { // Best effort: a sibling removal may have already cleared a shared // parent, or a concurrent writer repopulated the directory. Neither // is fatal to purging the orphan tree. debug!(bucket, object, dir, error = ?err, "purge_orphan_dir_object: skipped non-empty/absent directory"); } } } Ok(true) } pub(super) async fn check_write_precondition( &self, bucket: &str, object: &str, opts: &ObjectOptions, ) -> Option { let mut opts = opts.clone(); let http_preconditions = opts.http_preconditions?; opts.http_preconditions = None; // Never claim a lock here, to avoid deadlock // - If no_lock is false, we must have obtained the lock out side of this function // - If no_lock is true, we should not obtain locks opts.no_lock = true; let oi = self.get_object_info(bucket, object, &opts).await; match oi { Ok(oi) => { // If top level is a delete marker proceed to upload. if oi.delete_marker { return None; } let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned); let if_match = http_preconditions.if_match_value().map(str::to_owned); if should_prevent_write(&oi, if_none_match, if_match) { return Some(StorageError::PreconditionFailed); } } Err(StorageError::VersionNotFound(_, _, _)) | Err(StorageError::ObjectNotFound(_, _)) | Err(StorageError::ErasureReadQuorum) => { // When the object is not found, // - if If-Match is set, we should return 404 NotFound // - if If-None-Match is set, we should be able to proceed with the request if http_preconditions.if_match_value().is_some() { return Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string())); } } Err(e) => { return Some(e); } } None } } #[cfg(test)] mod tests { use super::*; #[test] fn dangling_delete_grace_defaults_to_one_hour() { temp_env::with_var(ENV_HEAL_DANGLING_DELETE_GRACE_SECS, None::<&str>, || { assert_eq!(dangling_delete_grace(), time::Duration::seconds(3600)); }); } #[test] fn dangling_delete_grace_env_override_and_disable() { temp_env::with_var(ENV_HEAL_DANGLING_DELETE_GRACE_SECS, Some("120"), || { assert_eq!(dangling_delete_grace(), time::Duration::seconds(120)); }); temp_env::with_var(ENV_HEAL_DANGLING_DELETE_GRACE_SECS, Some("0"), || { assert!(dangling_delete_grace().is_zero()); }); } }