// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use crate::diagnostics::admin_server_info::get_local_server_property;
use crate::runtime::sources as runtime_sources;
use crate::storage_api_contracts::admin::StorageAdminApi;
#[cfg(test)]
use chrono::Utc;
use jiff::Timestamp;
use rustfs_heal_contracts::heal_channel::DriveState;
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_madmin::metrics::{
    DiskIOStats, DiskMetric, LastMinute as MadminLastMinute, NetDevLine, NetMetrics, RPCMetrics, RealtimeMetrics,
    ScannerCheckpointReport as MadminScannerCheckpointReport,
    ScannerLifecycleExpirySnapshot as MadminScannerLifecycleExpirySnapshot,
    ScannerLifecycleTransitionSnapshot as MadminScannerLifecycleTransitionSnapshot,
    ScannerMaintenanceControlSnapshot as MadminScannerMaintenanceControlSnapshot,
    ScannerMaintenanceSourceSnapshot as MadminScannerMaintenanceSourceSnapshot, ScannerMetrics as MadminScannerMetrics,
    ScannerPacingPressureSnapshot as MadminScannerPacingPressureSnapshot,
    ScannerReplicationRepairSnapshot as MadminScannerReplicationRepairSnapshot,
    ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, ScannerSourceWorkSnapshot as MadminScannerSourceWorkSnapshot,
    ScannerUsageFreshnessSnapshot as MadminScannerUsageFreshnessSnapshot, TimedAction as MadminTimedAction,
};
use rustfs_scanner_contracts::metrics::global_metrics;
use rustfs_utils::os::get_drive_stats;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use tracing::{debug, info};

#[derive(Debug, Default, Serialize, Deserialize)]
pub struct CollectMetricsOpts {
    pub hosts: HashSet<String>,
    pub disks: HashSet<String>,
    pub job_id: String,
    pub dep_id: String,
}

#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct MetricType(u32);

impl MetricType {
    // Define some constants
    pub const NONE: MetricType = MetricType(0);
    pub const SCANNER: MetricType = MetricType(1 << 0);
    pub const DISK: MetricType = MetricType(1 << 1);
    pub const OS: MetricType = MetricType(1 << 2);
    pub const BATCH_JOBS: MetricType = MetricType(1 << 3);
    pub const SITE_RESYNC: MetricType = MetricType(1 << 4);
    pub const NET: MetricType = MetricType(1 << 5);
    pub const MEM: MetricType = MetricType(1 << 6);
    pub const CPU: MetricType = MetricType(1 << 7);
    pub const RPC: MetricType = MetricType(1 << 8);

    // MetricsAll must be last.
    pub const ALL: MetricType = MetricType((1 << 9) - 1);

    pub fn new(t: u32) -> Self {
        Self(t)
    }
}

fn unix_millis_to_jiff_timestamp(millis: u64, fallback: Timestamp) -> Timestamp {
    let millis = match i64::try_from(millis) {
        Ok(millis) => millis,
        Err(_) => return fallback,
    };

    match Timestamp::from_millisecond(millis) {
        Ok(timestamp) => timestamp,
        Err(_) => fallback,
    }
}

fn to_madmin_scanner_metrics(metrics: rustfs_scanner_contracts::metrics::ScannerMetricsReport) -> MadminScannerMetrics {
    MadminScannerMetrics {
        collected_at: metrics.collected_at,
        current_cycle: metrics.current_cycle,
        current_cycle_active: Some(metrics.current_cycle_active),
        current_started: metrics.current_started,
        cycles_completed_at: metrics.cycles_completed_at,
        ongoing_buckets: metrics.ongoing_buckets,
        active_scan_paths: metrics.active_scan_paths,
        oldest_active_path_age_seconds: metrics.oldest_active_path_age_seconds,
        life_time_ops: metrics.life_time_ops,
        life_time_ilm: metrics.life_time_ilm,
        last_minute: MadminLastMinute {
            actions: metrics
                .last_minute
                .actions
                .into_iter()
                .map(|(key, value)| {
                    (
                        key,
                        MadminTimedAction {
                            count: value.count,
                            acc_time: value.acc_time,
                            bytes: value.bytes,
                        },
                    )
                })
                .collect(),
            ilm: metrics
                .last_minute
                .ilm
                .into_iter()
                .map(|(key, value)| {
                    (
                        key,
                        MadminTimedAction {
                            count: value.count,
                            acc_time: value.acc_time,
                            bytes: value.bytes,
                        },
                    )
                })
                .collect(),
        },
        active_paths: metrics.active_paths,
        current_scan_mode: metrics.current_scan_mode,
        current_set_scan_concurrency_limit: metrics.current_set_scan_concurrency_limit,
        current_set_scans_queued: metrics.current_set_scans_queued,
        current_set_scans_active: metrics.current_set_scans_active,
        current_disk_scan_concurrency_limit: metrics.current_disk_scan_concurrency_limit,
        current_disk_bucket_scans_queued: metrics.current_disk_bucket_scans_queued,
        current_disk_bucket_scans_active: metrics.current_disk_bucket_scans_active,
        current_cycle_objects_scanned: metrics.current_cycle_objects_scanned,
        current_cycle_directories_scanned: metrics.current_cycle_directories_scanned,
        current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans,
        current_cycle_bucket_drive_failures: metrics.current_cycle_bucket_drive_failures,
        current_cycle_yield_events: metrics.current_cycle_yield_events,
        current_cycle_yield_duration_seconds: metrics.current_cycle_yield_duration_seconds,
        current_cycle_throttle_sleep_events: metrics.current_cycle_throttle_sleep_events,
        current_cycle_throttle_sleep_duration_seconds: metrics.current_cycle_throttle_sleep_duration_seconds,
        current_cycle_ilm_actions: metrics.current_cycle_ilm_actions,
        current_cycle_lifecycle_expiry_actions: metrics.current_cycle_lifecycle_expiry_actions,
        current_cycle_lifecycle_transition_actions: metrics.current_cycle_lifecycle_transition_actions,
        current_cycle_heal_objects: metrics.current_cycle_heal_objects,
        current_cycle_replication_checks: metrics.current_cycle_replication_checks,
        current_cycle_usage_saves: metrics.current_cycle_usage_saves,
        last_cycle_result: metrics.last_cycle_result,
        last_cycle_result_code: metrics.last_cycle_result_code,
        last_cycle_partial_reason: metrics.last_cycle_partial_reason,
        last_cycle_partial_reason_code: metrics.last_cycle_partial_reason_code,
        last_cycle_lifecycle_expiry_actions: metrics.last_cycle_lifecycle_expiry_actions,
        last_cycle_lifecycle_transition_actions: metrics.last_cycle_lifecycle_transition_actions,
        last_cycle_partial_source: metrics.last_cycle_partial_source,
        last_cycle_partial_source_code: metrics.last_cycle_partial_source_code,
        last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
        last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
        last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
        last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans,
        last_cycle_bucket_drive_failures: metrics.last_cycle_bucket_drive_failures,
        last_cycle_yield_events: metrics.last_cycle_yield_events,
        last_cycle_yield_duration_seconds: metrics.last_cycle_yield_duration_seconds,
        last_cycle_throttle_sleep_events: metrics.last_cycle_throttle_sleep_events,
        last_cycle_throttle_sleep_duration_seconds: metrics.last_cycle_throttle_sleep_duration_seconds,
        last_cycle_ilm_actions: metrics.last_cycle_ilm_actions,
        last_cycle_heal_objects: metrics.last_cycle_heal_objects,
        last_cycle_replication_checks: metrics.last_cycle_replication_checks,
        last_cycle_usage_saves: metrics.last_cycle_usage_saves,
        failed_cycles: metrics.failed_cycles,
        superseded_cycles: metrics.superseded_cycles,
        partial_cycles_unknown: metrics.partial_cycles_unknown,
        partial_cycles_runtime: metrics.partial_cycles_runtime,
        partial_cycles_objects: metrics.partial_cycles_objects,
        partial_cycles_directories: metrics.partial_cycles_directories,
        pacing_pressure: MadminScannerPacingPressureSnapshot {
            primary_pressure: metrics.pacing_pressure.primary_pressure,
            current_queued_scans: metrics.pacing_pressure.current_queued_scans,
            current_active_scans: metrics.pacing_pressure.current_active_scans,
            last_cycle_budget_limited: metrics.pacing_pressure.last_cycle_budget_limited,
            last_cycle_pause_observed: metrics.pacing_pressure.last_cycle_pause_observed,
            last_cycle_throttle_sleep_ratio: metrics.pacing_pressure.last_cycle_throttle_sleep_ratio,
            last_cycle_yield_ratio: metrics.pacing_pressure.last_cycle_yield_ratio,
            last_cycle_total_pause_ratio: metrics.pacing_pressure.last_cycle_total_pause_ratio,
        },
        lifecycle_transition: MadminScannerLifecycleTransitionSnapshot {
            current_queue_capacity: metrics.lifecycle_transition.current_queue_capacity,
            current_queued: metrics.lifecycle_transition.current_queued,
            current_active: metrics.lifecycle_transition.current_active,
            current_workers: metrics.lifecycle_transition.current_workers,
            queue_full: metrics.lifecycle_transition.queue_full,
            queue_send_timeout: metrics.lifecycle_transition.queue_send_timeout,
            compensation_scheduled: metrics.lifecycle_transition.compensation_scheduled,
            compensation_pending: metrics.lifecycle_transition.compensation_pending,
            compensation_running: metrics.lifecycle_transition.compensation_running,
            scanner_queued: metrics.lifecycle_transition.scanner_queued,
            scanner_missed: metrics.lifecycle_transition.scanner_missed,
            completed: metrics.lifecycle_transition.completed,
            failed: metrics.lifecycle_transition.failed,
        },
        lifecycle_expiry: MadminScannerLifecycleExpirySnapshot {
            current_queue_capacity: metrics.lifecycle_expiry.current_queue_capacity,
            current_queued: metrics.lifecycle_expiry.current_queued,
            current_active: metrics.lifecycle_expiry.current_active,
            current_workers: metrics.lifecycle_expiry.current_workers,
            queue_missed: metrics.lifecycle_expiry.queue_missed,
            scanner_queued: metrics.lifecycle_expiry.scanner_queued,
            scanner_missed: metrics.lifecycle_expiry.scanner_missed,
            scanner_blocked: metrics.lifecycle_expiry.scanner_blocked,
            scanner_not_enqueued: metrics.lifecycle_expiry.scanner_not_enqueued,
            delete_failed: metrics.lifecycle_expiry.delete_failed,
        },
        usage_freshness: MadminScannerUsageFreshnessSnapshot {
            dirty_pending_buckets: metrics.usage_freshness.dirty_pending_buckets,
            last_dirty_mark_unix_secs: metrics.usage_freshness.last_dirty_mark_unix_secs,
            last_dirty_clear_unix_secs: metrics.usage_freshness.last_dirty_clear_unix_secs,
            last_cycle_dirty_buckets: metrics.usage_freshness.last_cycle_dirty_buckets,
            last_cycle_cleared_dirty_buckets: metrics.usage_freshness.last_cycle_cleared_dirty_buckets,
            last_usage_save_unix_secs: metrics.usage_freshness.last_usage_save_unix_secs,
            last_usage_save_result: metrics.usage_freshness.last_usage_save_result,
            last_usage_save_result_code: metrics.usage_freshness.last_usage_save_result_code,
            last_durable_success_unix_secs: metrics.usage_freshness.last_durable_success_unix_secs,
            last_publication_unix_secs: metrics.usage_freshness.last_publication_unix_secs,
            last_publication_state: metrics.usage_freshness.last_publication_state,
            last_publication_reason: metrics.usage_freshness.last_publication_reason,
            deferred_pending: metrics.usage_freshness.deferred_pending,
            deferred_total: metrics.usage_freshness.deferred_total,
            last_deferred_unix_secs: metrics.usage_freshness.last_deferred_unix_secs,
            last_deferred_reason: metrics.usage_freshness.last_deferred_reason,
        },
        maintenance_control: MadminScannerMaintenanceControlSnapshot {
            primary_control: metrics.maintenance_control.primary_control,
            sources: metrics
                .maintenance_control
                .sources
                .into_iter()
                .map(|source| MadminScannerMaintenanceSourceSnapshot {
                    source: source.source,
                    state: source.state,
                    reason: source.reason,
                    backlog: source.backlog,
                    current_checked: source.current_checked,
                    current_queued: source.current_queued,
                    current_missed: source.current_missed,
                    lifetime_missed: source.lifetime_missed,
                    partial_cycles: source.partial_cycles,
                })
                .collect(),
        },
        partial_cycles_by_source: metrics
            .partial_cycles_by_source
            .into_iter()
            .map(|source| MadminScannerSourceCycleSnapshot {
                source: source.source,
                cycles: source.cycles,
            })
            .collect(),
        throttle_idle_mode_enabled: metrics.throttle_idle_mode_enabled,
        throttle_sleep_factor: metrics.throttle_sleep_factor,
        throttle_max_sleep_seconds: metrics.throttle_max_sleep_seconds,
        yield_every_n_objects: metrics.yield_every_n_objects,
        cycle_interval_seconds: metrics.cycle_interval_seconds,
        cycle_max_duration_seconds: metrics.cycle_max_duration_seconds,
        cycle_max_objects: metrics.cycle_max_objects,
        cycle_max_directories: metrics.cycle_max_directories,
        cycle_timeout_total: metrics.cycle_timeout_total,
        cycle_recovery_required_total: metrics.cycle_recovery_required_total,
        cycle_last_progress_age: metrics.cycle_last_progress_age,
        leader_lease_without_progress: metrics.leader_lease_without_progress,
        bitrot_cycle_enabled: metrics.bitrot_cycle_enabled,
        bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
        scan_checkpoint: metrics.scan_checkpoint.map(|checkpoint| MadminScannerCheckpointReport {
            version: checkpoint.version,
            resume_after: checkpoint.resume_after,
            reason: checkpoint.reason,
            last_event: checkpoint.last_event,
        }),
        scan_checkpoint_used: metrics.scan_checkpoint_used,
        scan_checkpoint_cleared: metrics.scan_checkpoint_cleared,
        scan_checkpoint_ignored: metrics.scan_checkpoint_ignored,
        scan_checkpoint_stale: metrics.scan_checkpoint_stale,
        source_work: metrics
            .source_work
            .into_iter()
            .map(|work| MadminScannerSourceWorkSnapshot {
                source: work.source,
                checked: work.checked,
                queued: work.queued,
                executed: work.executed,
                failed: work.failed,
                skipped: work.skipped,
                missed: work.missed,
            })
            .collect(),
        current_cycle_source_work: metrics
            .current_cycle_source_work
            .into_iter()
            .map(|work| MadminScannerSourceWorkSnapshot {
                source: work.source,
                checked: work.checked,
                queued: work.queued,
                executed: work.executed,
                failed: work.failed,
                skipped: work.skipped,
                missed: work.missed,
            })
            .collect(),
        last_cycle_source_work: metrics
            .last_cycle_source_work
            .into_iter()
            .map(|work| MadminScannerSourceWorkSnapshot {
                source: work.source,
                checked: work.checked,
                queued: work.queued,
                executed: work.executed,
                failed: work.failed,
                skipped: work.skipped,
                missed: work.missed,
            })
            .collect(),
        replication_repair: metrics
            .replication_repair
            .into_iter()
            .map(|repair| MadminScannerReplicationRepairSnapshot {
                source: repair.source,
                kind: repair.kind,
                scanner_role: repair.scanner_role,
                execution_owner: repair.execution_owner,
                checked: repair.checked,
                queued: repair.queued,
                executed: repair.executed,
                failed: repair.failed,
                skipped: repair.skipped,
                missed: repair.missed,
            })
            .collect(),
        current_cycle_replication_repair: metrics
            .current_cycle_replication_repair
            .into_iter()
            .map(|repair| MadminScannerReplicationRepairSnapshot {
                source: repair.source,
                kind: repair.kind,
                scanner_role: repair.scanner_role,
                execution_owner: repair.execution_owner,
                checked: repair.checked,
                queued: repair.queued,
                executed: repair.executed,
                failed: repair.failed,
                skipped: repair.skipped,
                missed: repair.missed,
            })
            .collect(),
        last_cycle_replication_repair: metrics
            .last_cycle_replication_repair
            .into_iter()
            .map(|repair| MadminScannerReplicationRepairSnapshot {
                source: repair.source,
                kind: repair.kind,
                scanner_role: repair.scanner_role,
                execution_owner: repair.execution_owner,
                checked: repair.checked,
                queued: repair.queued,
                executed: repair.executed,
                failed: repair.failed,
                skipped: repair.skipped,
                missed: repair.missed,
            })
            .collect(),
        partial_cycles: metrics.partial_cycles,
    }
}

impl MetricType {
    fn contains(&self, x: &MetricType) -> bool {
        (self.0 & x.0) == x.0
    }
}

/// Collect local metrics based on the specified types and options.
///
/// # Arguments
///
/// * `types` - A `MetricType` specifying which types of metrics to collect.
/// * `opts` - A reference to `CollectMetricsOpts` containing additional options for metric collection.
///
/// # Returns
/// * A `RealtimeMetrics` struct containing the collected metrics.
///
pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts) -> RealtimeMetrics {
    debug!("collect_local_metrics");
    let mut real_time_metrics = RealtimeMetrics::default();
    if types.0 == MetricType::NONE.0 {
        info!("types is None, return");
        return real_time_metrics;
    }

    let mut by_host_name = runtime_sources::rustfs_addr().await;
    if !opts.hosts.is_empty() {
        let server = get_local_server_property().await;
        if opts.hosts.contains(&server.endpoint) {
            by_host_name = server.endpoint;
        } else {
            return real_time_metrics;
        }
    }
    let local_node_name = runtime_sources::local_node_name().await;
    if by_host_name.starts_with(":") && !local_node_name.starts_with(":") {
        by_host_name = local_node_name;
    }

    if types.contains(&MetricType::DISK) {
        debug!("start get disk metrics");
        let mut aggr = DiskMetric {
            collected_at: Timestamp::now(),
            ..Default::default()
        };
        for (name, disk) in collect_local_disks_metrics(&opts.disks).await.into_iter() {
            debug!("got disk metric, name: {name}, metric: {disk:?}");
            real_time_metrics.by_disk.insert(name, disk.clone());
            aggr.merge(&disk);
        }
        real_time_metrics.aggregated.disk = Some(aggr);
    }

    if types.contains(&MetricType::SCANNER) {
        debug!("start get scanner metrics");
        let metrics = global_metrics().report().await;
        real_time_metrics.aggregated.scanner = Some(to_madmin_scanner_metrics(metrics));
    }

    // if types.contains(&MetricType::OS) {}

    // if types.contains(&MetricType::BATCH_JOBS) {}

    // if types.contains(&MetricType::SITE_RESYNC) {}

    if types.contains(&MetricType::NET) {
        let snapshot = global_internode_metrics().snapshot();
        real_time_metrics.aggregated.net = Some(NetMetrics {
            collected_at: Timestamp::now(),
            interface_name: "internode".to_string(),
            net_stats: NetDevLine {
                name: "internode".to_string(),
                rx_bytes: snapshot.recv_bytes_total,
                tx_bytes: snapshot.sent_bytes_total,
                ..Default::default()
            },
        });
    }

    // if types.contains(&MetricType::MEM) {}

    // if types.contains(&MetricType::CPU) {}

    if types.contains(&MetricType::RPC) {
        let collected_at = Timestamp::now();
        let snapshot = global_internode_metrics().snapshot();
        let last_connect_time = unix_millis_to_jiff_timestamp(snapshot.last_dial_unix_millis, collected_at);

        real_time_metrics.aggregated.rpc = Some(RPCMetrics {
            collected_at,
            connected: i32::from(snapshot.last_dial_unix_millis > 0),
            reconnect_count: snapshot.dial_errors_total.min(i32::MAX as u64) as i32,
            disconnected: 0,
            outgoing_streams: 0,
            incoming_streams: 0,
            outgoing_bytes: snapshot.sent_bytes_total.min(i64::MAX as u64) as i64,
            incoming_bytes: snapshot.recv_bytes_total.min(i64::MAX as u64) as i64,
            outgoing_messages: snapshot.outgoing_requests_total.min(i64::MAX as u64) as i64,
            incoming_messages: snapshot.incoming_requests_total.min(i64::MAX as u64) as i64,
            out_queue: 0,
            last_pong_time: collected_at,
            last_ping_ms: snapshot.dial_avg_time_nanos as f64 / 1_000_000.0,
            max_ping_dur_ms: snapshot.dial_avg_time_nanos as f64 / 1_000_000.0,
            last_connect_time,
            by_destination: None,
            by_caller: None,
        });
    }

    real_time_metrics
        .by_host
        .insert(by_host_name.clone(), real_time_metrics.aggregated.clone());
    real_time_metrics.hosts.push(by_host_name);

    real_time_metrics
}

async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String, DiskMetric> {
    let store = match runtime_sources::object_store_handle() {
        Some(store) => store,
        None => return HashMap::new(),
    };

    let mut metrics = HashMap::new();
    let storage_info = StorageAdminApi::local_storage_info(store.as_ref()).await;
    for d in storage_info.disks.iter() {
        if !disks.is_empty() && !disks.contains(&d.endpoint) {
            continue;
        }

        if d.state != DriveState::Ok.to_string() && d.state != DriveState::Unformatted.to_string() {
            metrics.insert(
                d.endpoint.clone(),
                DiskMetric {
                    n_disks: 1,
                    offline: 1,
                    ..Default::default()
                },
            );
            continue;
        }

        let mut dm = DiskMetric {
            n_disks: 1,
            ..Default::default()
        };
        if d.healing {
            dm.healing += 1;
        }

        if let Some(m) = &d.metrics {
            for (k, v) in m.api_calls.iter() {
                if *v != 0 {
                    dm.life_time_ops.insert(k.clone(), *v);
                }
            }
            for (k, v) in m.last_minute.iter() {
                if v.count != 0 {
                    dm.last_minute.operations.insert(k.clone(), v.clone());
                }
            }
        }

        if let Ok(st) = get_drive_stats(d.major, d.minor) {
            dm.io_stats = DiskIOStats {
                read_ios: st.read_ios,
                read_merges: st.read_merges,
                read_sectors: st.read_sectors,
                read_ticks: st.read_ticks,
                write_ios: st.write_ios,
                write_merges: st.write_merges,
                write_sectors: st.write_sectors,
                write_ticks: st.write_ticks,
                current_ios: st.current_ios,
                total_ticks: st.total_ticks,
                req_ticks: st.req_ticks,
                discard_ios: st.discard_ios,
                discard_merges: st.discard_merges,
                discard_sectors: st.discard_sectors,
                discard_ticks: st.discard_ticks,
                flush_ios: st.flush_ios,
                flush_ticks: st.flush_ticks,
            };
        }
        metrics.insert(d.endpoint.clone(), dm);
    }

    metrics
}

#[cfg(test)]
mod test {
    use super::*;
    use rustfs_io_metrics::internode_metrics::global_internode_metrics;
    use rustfs_scanner_contracts::metrics::CurrentCycle;
    use serial_test::serial;
    use std::time::Duration;

    fn chrono_to_jiff_timestamp(timestamp: chrono::DateTime<Utc>) -> jiff::Timestamp {
        jiff::Timestamp::try_from(std::time::SystemTime::from(timestamp)).expect("test timestamp should fit in jiff")
    }

    #[test]
    fn tes_types() {
        let t = MetricType::ALL;
        assert!(t.contains(&MetricType::NONE));
        assert!(t.contains(&MetricType::DISK));
        assert!(t.contains(&MetricType::OS));
        assert!(t.contains(&MetricType::BATCH_JOBS));
        assert!(t.contains(&MetricType::SITE_RESYNC));
        assert!(t.contains(&MetricType::NET));
        assert!(t.contains(&MetricType::MEM));
        assert!(t.contains(&MetricType::CPU));
        assert!(t.contains(&MetricType::RPC));

        let disk = MetricType::new(1 << 1);
        assert!(disk.contains(&MetricType::DISK));
    }

    #[tokio::test]
    async fn collect_local_metrics_reports_internode_net_and_rpc() {
        let metrics = global_internode_metrics();
        metrics.reset_for_test();
        metrics.record_sent_bytes(128);
        metrics.record_recv_bytes(64);
        metrics.record_outgoing_request();
        metrics.record_incoming_request();
        metrics.record_dial_result(Duration::from_millis(4), true);

        let realtime = collect_local_metrics(MetricType::NET, &CollectMetricsOpts::default()).await;
        let net = realtime.aggregated.net.expect("net metrics");
        assert_eq!(net.net_stats.tx_bytes, 128);
        assert_eq!(net.net_stats.rx_bytes, 64);

        let realtime = collect_local_metrics(MetricType::RPC, &CollectMetricsOpts::default()).await;
        let rpc = realtime.aggregated.rpc.expect("rpc metrics");
        assert_eq!(rpc.outgoing_bytes, 128);
        assert_eq!(rpc.incoming_bytes, 64);
        assert_eq!(rpc.outgoing_messages, 1);
        assert_eq!(rpc.incoming_messages, 1);
        assert!(rpc.last_ping_ms > 0.0);

        metrics.reset_for_test();
    }

    #[test]
    fn scanner_metrics_mapping_preserves_partial_source_status() {
        let current_started = Utc::now() - chrono::Duration::seconds(5);
        let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
            current_cycle_active: true,
            current_started: chrono_to_jiff_timestamp(current_started),
            last_cycle_partial_source: "usage".to_string(),
            last_cycle_partial_source_code: 1,
            cycle_timeout_total: 3,
            cycle_recovery_required_total: 2,
            cycle_last_progress_age: 17,
            leader_lease_without_progress: true,
            partial_cycles_by_source: vec![rustfs_scanner_contracts::metrics::ScannerSourceCycleSnapshot {
                source: "usage".to_string(),
                cycles: 2,
            }],
            ..Default::default()
        });

        assert_eq!(scanner.current_cycle_active, Some(true));
        assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
        assert_eq!(scanner.last_cycle_partial_source, "usage");
        assert_eq!(scanner.last_cycle_partial_source_code, 1);
        assert_eq!(scanner.cycle_timeout_total, 3);
        assert_eq!(scanner.cycle_recovery_required_total, 2);
        assert_eq!(scanner.cycle_last_progress_age, 17);
        assert!(scanner.leader_lease_without_progress);
        let usage = scanner
            .partial_cycles_by_source
            .iter()
            .find(|source| source.source == "usage")
            .expect("usage partial source should be mapped");
        assert_eq!(usage.cycles, 2);
    }

    #[tokio::test]
    #[serial]
    async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
        let previous_init_time = *rustfs_scanner_contracts::GLOBAL_INIT_TIME.read().await;
        let previous_cycle = global_metrics().get_cycle().await;
        let init_time = Utc::now() - chrono::Duration::hours(1);
        let cycle_started = Utc::now() - chrono::Duration::seconds(5);
        *rustfs_scanner_contracts::GLOBAL_INIT_TIME.write().await = Some(init_time);
        let cycle = CurrentCycle {
            current: 0,
            next: 1,
            started: cycle_started,
            ..Default::default()
        };
        let cycle_start = global_metrics().start_scan_cycle_work_with_cycle(cycle).await;

        let realtime = collect_local_metrics(MetricType::SCANNER, &CollectMetricsOpts::default()).await;

        global_metrics()
            .finish_scan_cycle_work_with_cycle(cycle_start, previous_cycle.clone().unwrap_or_default())
            .await;
        global_metrics().set_cycle(previous_cycle).await;
        *rustfs_scanner_contracts::GLOBAL_INIT_TIME.write().await = previous_init_time;

        let encoded = rmp_serde::to_vec_named(&realtime).expect("realtime metrics should encode");
        let decoded: RealtimeMetrics = rmp_serde::from_slice(&encoded).expect("realtime metrics should decode");
        let mut aggregated = RealtimeMetrics::default();
        aggregated.merge(decoded);
        let scanner = aggregated.aggregated.scanner.expect("scanner metrics");
        assert_eq!(scanner.current_cycle_active, Some(true));
        assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(cycle_started));
    }

    #[test]
    fn scanner_metrics_mapping_preserves_pacing_pressure() {
        let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
            pacing_pressure: rustfs_scanner_contracts::metrics::ScannerPacingPressureSnapshot {
                primary_pressure: "cycle_budget".to_string(),
                current_queued_scans: 4,
                current_active_scans: 2,
                last_cycle_budget_limited: true,
                last_cycle_pause_observed: true,
                last_cycle_throttle_sleep_ratio: 0.25,
                last_cycle_yield_ratio: 0.05,
                last_cycle_total_pause_ratio: 0.3,
            },
            ..Default::default()
        });

        assert_eq!(scanner.pacing_pressure.primary_pressure, "cycle_budget");
        assert_eq!(scanner.pacing_pressure.current_queued_scans, 4);
        assert_eq!(scanner.pacing_pressure.current_active_scans, 2);
        assert!(scanner.pacing_pressure.last_cycle_budget_limited);
        assert!(scanner.pacing_pressure.last_cycle_pause_observed);
        assert_eq!(scanner.pacing_pressure.last_cycle_throttle_sleep_ratio, 0.25);
        assert_eq!(scanner.pacing_pressure.last_cycle_yield_ratio, 0.05);
        assert_eq!(scanner.pacing_pressure.last_cycle_total_pause_ratio, 0.3);
    }

    #[test]
    fn scanner_metrics_mapping_preserves_lifecycle_transition_status() {
        let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
            current_cycle_lifecycle_expiry_actions: 2,
            current_cycle_lifecycle_transition_actions: 3,
            last_cycle_lifecycle_expiry_actions: 5,
            last_cycle_lifecycle_transition_actions: 7,
            lifecycle_expiry: rustfs_scanner_contracts::metrics::ScannerLifecycleExpirySnapshot {
                current_queue_capacity: 16,
                current_queued: 5,
                current_active: 2,
                current_workers: 4,
                queue_missed: 3,
                scanner_queued: 6,
                scanner_missed: 2,
                scanner_blocked: 4,
                scanner_not_enqueued: 2,
                delete_failed: 1,
            },
            lifecycle_transition: rustfs_scanner_contracts::metrics::ScannerLifecycleTransitionSnapshot {
                current_queue_capacity: 16,
                current_queued: 5,
                current_active: 2,
                current_workers: 4,
                queue_full: 3,
                queue_send_timeout: 1,
                compensation_scheduled: 2,
                compensation_pending: 3,
                compensation_running: 1,
                scanner_queued: 6,
                scanner_missed: 2,
                completed: 4,
                failed: 1,
            },
            ..Default::default()
        });

        assert_eq!(scanner.lifecycle_expiry.current_queue_capacity, 16);
        assert_eq!(scanner.lifecycle_expiry.current_queued, 5);
        assert_eq!(scanner.lifecycle_expiry.current_active, 2);
        assert_eq!(scanner.lifecycle_expiry.current_workers, 4);
        assert_eq!(scanner.lifecycle_expiry.queue_missed, 3);
        assert_eq!(scanner.lifecycle_expiry.scanner_queued, 6);
        assert_eq!(scanner.lifecycle_expiry.scanner_missed, 2);
        assert_eq!(scanner.lifecycle_expiry.scanner_blocked, 4);
        assert_eq!(scanner.lifecycle_expiry.scanner_not_enqueued, 2);
        assert_eq!(scanner.lifecycle_expiry.delete_failed, 1);
        assert_eq!(scanner.lifecycle_transition.current_queue_capacity, 16);
        assert_eq!(scanner.lifecycle_transition.current_queued, 5);
        assert_eq!(scanner.lifecycle_transition.current_active, 2);
        assert_eq!(scanner.lifecycle_transition.current_workers, 4);
        assert_eq!(scanner.lifecycle_transition.queue_full, 3);
        assert_eq!(scanner.lifecycle_transition.queue_send_timeout, 1);
        assert_eq!(scanner.lifecycle_transition.compensation_scheduled, 2);
        assert_eq!(scanner.lifecycle_transition.compensation_pending, 3);
        assert_eq!(scanner.lifecycle_transition.compensation_running, 1);
        assert_eq!(scanner.lifecycle_transition.scanner_queued, 6);
        assert_eq!(scanner.lifecycle_transition.scanner_missed, 2);
        assert_eq!(scanner.lifecycle_transition.completed, 4);
        assert_eq!(scanner.lifecycle_transition.failed, 1);
        assert_eq!(scanner.current_cycle_lifecycle_expiry_actions, 2);
        assert_eq!(scanner.current_cycle_lifecycle_transition_actions, 3);
        assert_eq!(scanner.last_cycle_lifecycle_expiry_actions, 5);
        assert_eq!(scanner.last_cycle_lifecycle_transition_actions, 7);
    }

    #[test]
    fn scanner_metrics_mapping_preserves_maintenance_control_status() {
        let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
            maintenance_control: rustfs_scanner_contracts::metrics::ScannerMaintenanceControlSnapshot {
                primary_control: "blocked_source".to_string(),
                sources: vec![rustfs_scanner_contracts::metrics::ScannerMaintenanceSourceSnapshot {
                    source: "lifecycle".to_string(),
                    state: "blocked".to_string(),
                    reason: "missed_work".to_string(),
                    backlog: 4,
                    current_checked: 2,
                    current_queued: 1,
                    current_missed: 3,
                    lifetime_missed: 8,
                    partial_cycles: 5,
                }],
            },
            ..Default::default()
        });

        assert_eq!(scanner.maintenance_control.primary_control, "blocked_source");
        let lifecycle = scanner
            .maintenance_control
            .sources
            .iter()
            .find(|source| source.source == "lifecycle")
            .expect("lifecycle maintenance control should be mapped");
        assert_eq!(lifecycle.state, "blocked");
        assert_eq!(lifecycle.reason, "missed_work");
        assert_eq!(lifecycle.backlog, 4);
        assert_eq!(lifecycle.current_checked, 2);
        assert_eq!(lifecycle.current_queued, 1);
        assert_eq!(lifecycle.current_missed, 3);
        assert_eq!(lifecycle.lifetime_missed, 8);
        assert_eq!(lifecycle.partial_cycles, 5);
    }

    #[test]
    fn scanner_metrics_mapping_preserves_usage_freshness_status() {
        let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
            usage_freshness: rustfs_scanner_contracts::metrics::ScannerUsageFreshnessSnapshot {
                dirty_pending_buckets: 3,
                last_dirty_mark_unix_secs: 10,
                last_dirty_clear_unix_secs: 11,
                last_cycle_dirty_buckets: 2,
                last_cycle_cleared_dirty_buckets: 1,
                last_usage_save_unix_secs: 12,
                last_usage_save_result: "success".to_string(),
                last_usage_save_result_code: 1,
                last_durable_success_unix_secs: 13,
                last_publication_unix_secs: 14,
                last_publication_state: "published".to_string(),
                last_publication_reason: "complete".to_string(),
                deferred_pending: true,
                deferred_total: 15,
                last_deferred_unix_secs: 16,
                last_deferred_reason: "data_movement".to_string(),
            },
            ..Default::default()
        });

        assert_eq!(scanner.usage_freshness.dirty_pending_buckets, 3);
        assert_eq!(scanner.usage_freshness.last_dirty_mark_unix_secs, 10);
        assert_eq!(scanner.usage_freshness.last_dirty_clear_unix_secs, 11);
        assert_eq!(scanner.usage_freshness.last_cycle_dirty_buckets, 2);
        assert_eq!(scanner.usage_freshness.last_cycle_cleared_dirty_buckets, 1);
        assert_eq!(scanner.usage_freshness.last_usage_save_unix_secs, 12);
        assert_eq!(scanner.usage_freshness.last_usage_save_result, "success");
        assert_eq!(scanner.usage_freshness.last_usage_save_result_code, 1);
        assert_eq!(scanner.usage_freshness.last_durable_success_unix_secs, 13);
        assert_eq!(scanner.usage_freshness.last_publication_unix_secs, 14);
        assert_eq!(scanner.usage_freshness.last_publication_state, "published");
        assert_eq!(scanner.usage_freshness.last_publication_reason, "complete");
        assert!(scanner.usage_freshness.deferred_pending);
        assert_eq!(scanner.usage_freshness.deferred_total, 15);
        assert_eq!(scanner.usage_freshness.last_deferred_unix_secs, 16);
        assert_eq!(scanner.usage_freshness.last_deferred_reason, "data_movement");
    }

    #[test]
    fn scanner_metrics_mapping_preserves_distributed_status_fields() {
        let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
            active_scan_paths: 2,
            oldest_active_path_age_seconds: 45,
            active_paths: vec!["disk-a/bucket-a".to_string(), "disk-b/bucket-b".to_string()],
            current_scan_mode: "normal".to_string(),
            current_set_scan_concurrency_limit: 3,
            current_set_scans_queued: 4,
            current_set_scans_active: 5,
            current_disk_scan_concurrency_limit: 6,
            current_disk_bucket_scans_queued: 7,
            current_disk_bucket_scans_active: 8,
            current_cycle_objects_scanned: 9,
            current_cycle_directories_scanned: 10,
            current_cycle_bucket_drive_scans: 11,
            current_cycle_bucket_drive_failures: 12,
            current_cycle_yield_events: 13,
            current_cycle_yield_duration_seconds: 1.5,
            current_cycle_throttle_sleep_events: 14,
            current_cycle_throttle_sleep_duration_seconds: 2.5,
            current_cycle_ilm_actions: 15,
            current_cycle_heal_objects: 16,
            current_cycle_replication_checks: 17,
            current_cycle_usage_saves: 18,
            last_cycle_result: "partial".to_string(),
            last_cycle_result_code: 2,
            last_cycle_partial_reason: "objects".to_string(),
            last_cycle_partial_reason_code: 3,
            last_cycle_duration_seconds: 4.5,
            last_cycle_objects_scanned: 19,
            last_cycle_directories_scanned: 20,
            last_cycle_bucket_drive_scans: 21,
            last_cycle_bucket_drive_failures: 22,
            last_cycle_yield_events: 23,
            last_cycle_yield_duration_seconds: 5.5,
            last_cycle_throttle_sleep_events: 24,
            last_cycle_throttle_sleep_duration_seconds: 6.5,
            last_cycle_ilm_actions: 25,
            last_cycle_heal_objects: 26,
            last_cycle_replication_checks: 27,
            last_cycle_usage_saves: 28,
            failed_cycles: 29,
            superseded_cycles: 30,
            partial_cycles_unknown: 30,
            partial_cycles_runtime: 31,
            partial_cycles_objects: 32,
            partial_cycles_directories: 33,
            throttle_idle_mode_enabled: true,
            throttle_sleep_factor: 2.0,
            throttle_max_sleep_seconds: 3.0,
            yield_every_n_objects: 34,
            cycle_interval_seconds: 35.0,
            cycle_max_duration_seconds: 36.0,
            cycle_max_objects: 37,
            cycle_max_directories: 38,
            bitrot_cycle_enabled: true,
            bitrot_cycle_seconds: 39.0,
            scan_checkpoint: Some(rustfs_scanner_contracts::metrics::ScannerCheckpointReport {
                version: 1,
                resume_after: "bucket-a/prefix-a".to_string(),
                reason: "directories".to_string(),
                last_event: "used".to_string(),
            }),
            scan_checkpoint_used: 40,
            scan_checkpoint_cleared: 41,
            scan_checkpoint_ignored: 42,
            scan_checkpoint_stale: 43,
            source_work: vec![rustfs_scanner_contracts::metrics::ScannerSourceWorkSnapshot {
                source: "usage".to_string(),
                checked: 44,
                queued: 45,
                executed: 46,
                failed: 47,
                skipped: 48,
                missed: 49,
            }],
            current_cycle_source_work: vec![rustfs_scanner_contracts::metrics::ScannerSourceWorkSnapshot {
                source: "lifecycle".to_string(),
                checked: 50,
                queued: 51,
                executed: 52,
                failed: 53,
                skipped: 54,
                missed: 55,
            }],
            last_cycle_source_work: vec![rustfs_scanner_contracts::metrics::ScannerSourceWorkSnapshot {
                source: "heal".to_string(),
                checked: 56,
                queued: 57,
                executed: 58,
                failed: 59,
                skipped: 60,
                missed: 61,
            }],
            replication_repair: vec![rustfs_scanner_contracts::metrics::ScannerReplicationRepairSnapshot {
                source: "bucket_replication".to_string(),
                kind: "object".to_string(),
                scanner_role: "repair_admission".to_string(),
                execution_owner: "bucket_replication_queue".to_string(),
                checked: 62,
                queued: 63,
                executed: 64,
                failed: 65,
                skipped: 66,
                missed: 67,
            }],
            current_cycle_replication_repair: vec![rustfs_scanner_contracts::metrics::ScannerReplicationRepairSnapshot {
                source: "bucket_replication".to_string(),
                kind: "delete_marker".to_string(),
                scanner_role: "repair_admission".to_string(),
                execution_owner: "bucket_replication_queue".to_string(),
                checked: 68,
                queued: 69,
                executed: 70,
                failed: 71,
                skipped: 72,
                missed: 73,
            }],
            last_cycle_replication_repair: vec![rustfs_scanner_contracts::metrics::ScannerReplicationRepairSnapshot {
                source: "site_replication".to_string(),
                kind: "active_resync".to_string(),
                scanner_role: "boundary_signal".to_string(),
                execution_owner: "site_replication_runtime".to_string(),
                checked: 74,
                queued: 75,
                executed: 76,
                failed: 77,
                skipped: 78,
                missed: 79,
            }],
            partial_cycles: 80,
            ..Default::default()
        });

        assert_eq!(scanner.active_scan_paths, 2);
        assert_eq!(scanner.oldest_active_path_age_seconds, 45);
        assert_eq!(scanner.active_paths, vec!["disk-a/bucket-a".to_string(), "disk-b/bucket-b".to_string()]);
        assert_eq!(scanner.current_scan_mode, "normal");
        assert_eq!(scanner.current_set_scan_concurrency_limit, 3);
        assert_eq!(scanner.current_set_scans_queued, 4);
        assert_eq!(scanner.current_set_scans_active, 5);
        assert_eq!(scanner.current_disk_scan_concurrency_limit, 6);
        assert_eq!(scanner.current_disk_bucket_scans_queued, 7);
        assert_eq!(scanner.current_disk_bucket_scans_active, 8);
        assert_eq!(scanner.current_cycle_objects_scanned, 9);
        assert_eq!(scanner.current_cycle_directories_scanned, 10);
        assert_eq!(scanner.current_cycle_bucket_drive_scans, 11);
        assert_eq!(scanner.current_cycle_bucket_drive_failures, 12);
        assert_eq!(scanner.current_cycle_yield_events, 13);
        assert_eq!(scanner.current_cycle_yield_duration_seconds, 1.5);
        assert_eq!(scanner.current_cycle_throttle_sleep_events, 14);
        assert_eq!(scanner.current_cycle_throttle_sleep_duration_seconds, 2.5);
        assert_eq!(scanner.current_cycle_ilm_actions, 15);
        assert_eq!(scanner.current_cycle_heal_objects, 16);
        assert_eq!(scanner.current_cycle_replication_checks, 17);
        assert_eq!(scanner.current_cycle_usage_saves, 18);
        assert_eq!(scanner.last_cycle_result, "partial");
        assert_eq!(scanner.last_cycle_result_code, 2);
        assert_eq!(scanner.last_cycle_partial_reason, "objects");
        assert_eq!(scanner.last_cycle_partial_reason_code, 3);
        assert_eq!(scanner.last_cycle_duration_seconds, 4.5);
        assert_eq!(scanner.last_cycle_objects_scanned, 19);
        assert_eq!(scanner.last_cycle_directories_scanned, 20);
        assert_eq!(scanner.last_cycle_bucket_drive_scans, 21);
        assert_eq!(scanner.last_cycle_bucket_drive_failures, 22);
        assert_eq!(scanner.last_cycle_yield_events, 23);
        assert_eq!(scanner.last_cycle_yield_duration_seconds, 5.5);
        assert_eq!(scanner.last_cycle_throttle_sleep_events, 24);
        assert_eq!(scanner.last_cycle_throttle_sleep_duration_seconds, 6.5);
        assert_eq!(scanner.last_cycle_ilm_actions, 25);
        assert_eq!(scanner.last_cycle_heal_objects, 26);
        assert_eq!(scanner.last_cycle_replication_checks, 27);
        assert_eq!(scanner.last_cycle_usage_saves, 28);
        assert_eq!(scanner.failed_cycles, 29);
        assert_eq!(scanner.superseded_cycles, 30);
        assert_eq!(scanner.partial_cycles_unknown, 30);
        assert_eq!(scanner.partial_cycles_runtime, 31);
        assert_eq!(scanner.partial_cycles_objects, 32);
        assert_eq!(scanner.partial_cycles_directories, 33);
        assert!(scanner.throttle_idle_mode_enabled);
        assert_eq!(scanner.throttle_sleep_factor, 2.0);
        assert_eq!(scanner.throttle_max_sleep_seconds, 3.0);
        assert_eq!(scanner.yield_every_n_objects, 34);
        assert_eq!(scanner.cycle_interval_seconds, 35.0);
        assert_eq!(scanner.cycle_max_duration_seconds, 36.0);
        assert_eq!(scanner.cycle_max_objects, 37);
        assert_eq!(scanner.cycle_max_directories, 38);
        assert!(scanner.bitrot_cycle_enabled);
        assert_eq!(scanner.bitrot_cycle_seconds, 39.0);
        let checkpoint = scanner.scan_checkpoint.expect("checkpoint should be mapped");
        assert_eq!(checkpoint.resume_after, "bucket-a/prefix-a");
        assert_eq!(scanner.scan_checkpoint_used, 40);
        assert_eq!(scanner.scan_checkpoint_cleared, 41);
        assert_eq!(scanner.scan_checkpoint_ignored, 42);
        assert_eq!(scanner.scan_checkpoint_stale, 43);
        assert_eq!(scanner.source_work[0].source, "usage");
        assert_eq!(scanner.source_work[0].missed, 49);
        assert_eq!(scanner.current_cycle_source_work[0].source, "lifecycle");
        assert_eq!(scanner.current_cycle_source_work[0].queued, 51);
        assert_eq!(scanner.last_cycle_source_work[0].source, "heal");
        assert_eq!(scanner.last_cycle_source_work[0].skipped, 60);
        assert_eq!(scanner.replication_repair[0].source, "bucket_replication");
        assert_eq!(scanner.replication_repair[0].kind, "object");
        assert_eq!(scanner.replication_repair[0].scanner_role, "repair_admission");
        assert_eq!(scanner.replication_repair[0].execution_owner, "bucket_replication_queue");
        assert_eq!(scanner.replication_repair[0].missed, 67);
        assert_eq!(scanner.current_cycle_replication_repair[0].kind, "delete_marker");
        assert_eq!(scanner.current_cycle_replication_repair[0].scanner_role, "repair_admission");
        assert_eq!(scanner.current_cycle_replication_repair[0].execution_owner, "bucket_replication_queue");
        assert_eq!(scanner.current_cycle_replication_repair[0].queued, 69);
        assert_eq!(scanner.last_cycle_replication_repair[0].source, "site_replication");
        assert_eq!(scanner.last_cycle_replication_repair[0].kind, "active_resync");
        assert_eq!(scanner.last_cycle_replication_repair[0].scanner_role, "boundary_signal");
        assert_eq!(scanner.last_cycle_replication_repair[0].execution_owner, "site_replication_runtime");
        assert_eq!(scanner.last_cycle_replication_repair[0].skipped, 78);
        assert_eq!(scanner.partial_cycles, 80);
    }
}
