// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use crate::server::runtime_sources;
use crate::server::{ServiceState, ServiceStateManager};
use crate::server::{has_path_prefix, is_table_catalog_path};
use crate::storage_api::cluster::control_plane::ClusterControlPlane;
use crate::storage_api::server::readiness::contract::admin::StorageAdminApi;
use crate::storage_api::server::readiness::{Endpoint, EndpointServerPools, is_dist_erasure};
#[cfg(test)]
use crate::storage_api::server::readiness::{Endpoints, PoolEndpoints};
use crate::storage_api::startup::shutdown::mark_get_metadata_read_version_coalescing_service_ready;
use bytes::Bytes;
use http::HeaderValue;
use http::{Request as HttpRequest, Response, StatusCode};
use http_body::Body;
use http_body_util::{BodyExt, Full};
use hyper::body::Incoming;
use metrics::{counter, gauge};
use rustfs_common::GlobalReadiness;
use rustfs_madmin::{Disk, StorageInfo};
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::Duration;
use std::{
    collections::{HashMap, HashSet},
    sync::OnceLock,
    time::Instant,
};
use tokio::sync::Mutex;
use tower::{Layer, Service};
use tracing::{debug, info};

/// Default upper bound for the startup runtime-readiness wait.
///
/// This is the compile-time fallback used when
/// [`rustfs_config::ENV_STARTUP_READINESS_MAX_WAIT_SECS`] is unset or `0`. The
/// effective value is resolved at runtime by [`startup_runtime_readiness_max_wait`],
/// which lets slow multi-node cold starts (Docker/K8s/NAS) extend the budget past
/// the internal DNS-retry and format-load windows without a rebuild.
pub const STARTUP_RUNTIME_READINESS_MAX_WAIT: Duration =
    Duration::from_secs(rustfs_config::DEFAULT_STARTUP_READINESS_MAX_WAIT_SECS);
pub const STARTUP_RUNTIME_READINESS_POLL_INTERVAL: Duration = Duration::from_secs(1);

/// Resolve the effective startup runtime-readiness wait.
///
/// Reads `RUSTFS_STARTUP_READINESS_MAX_WAIT_SECS`, falling back to
/// [`STARTUP_RUNTIME_READINESS_MAX_WAIT`] when unset. A configured value of `0`
/// is treated as "use the default" rather than an instant timeout, so an empty or
/// misconfigured env var can never make the server give up on readiness immediately.
fn startup_runtime_readiness_max_wait() -> Duration {
    let secs = rustfs_utils::get_env_u64(
        rustfs_config::ENV_STARTUP_READINESS_MAX_WAIT_SECS,
        rustfs_config::DEFAULT_STARTUP_READINESS_MAX_WAIT_SECS,
    );
    if secs == 0 {
        STARTUP_RUNTIME_READINESS_MAX_WAIT
    } else {
        Duration::from_secs(secs)
    }
}
const METRIC_RUNTIME_READINESS_READY: &str = "rustfs_runtime_readiness_ready";
const METRIC_RUNTIME_READINESS_DEGRADED_TOTAL: &str = "rustfs_runtime_readiness_degraded_total";

pub use crate::shared_types::{DependencyReadiness, DependencyReadinessReport, ReadinessDegradedReason};

/// ReadinessGateLayer ensures that the system components (IAM, Storage)
/// are fully initialized before allowing any request to proceed.
#[derive(Clone)]
pub struct ReadinessGateLayer {
    readiness: Arc<GlobalReadiness>,
}

impl ReadinessGateLayer {
    /// Create a new ReadinessGateLayer
    /// # Arguments
    /// * `readiness` - An Arc to the GlobalReadiness instance
    ///
    /// # Returns
    /// A new instance of ReadinessGateLayer
    pub fn new(readiness: Arc<GlobalReadiness>) -> Self {
        Self { readiness }
    }
}

impl<S> Layer<S> for ReadinessGateLayer {
    type Service = ReadinessGateService<S>;

    /// Wrap the inner service with ReadinessGateService
    /// # Arguments
    /// * `inner` - The inner service to wrap
    /// # Returns
    /// An instance of ReadinessGateService
    fn layer(&self, inner: S) -> Self::Service {
        ReadinessGateService {
            inner,
            readiness: self.readiness.clone(),
        }
    }
}

#[derive(Clone)]
pub struct ReadinessGateService<S> {
    inner: S,
    readiness: Arc<GlobalReadiness>,
}

fn is_probe_path(path: &str) -> bool {
    let is_exact_probe = matches!(
        path,
        crate::server::PROFILE_MEMORY_PATH
            | crate::server::PROFILE_CPU_PATH
            | crate::server::HEALTH_PREFIX
            | crate::server::HEALTH_COMPAT_LIVE_PATH
            | crate::server::HEALTH_READY_PATH
            | crate::server::MINIO_HEALTH_LIVE_PATH
            | crate::server::MINIO_HEALTH_READY_PATH
            | crate::server::MINIO_HEALTH_CLUSTER_PATH
            | crate::server::MINIO_HEALTH_CLUSTER_READ_PATH
            | crate::server::FAVICON_PATH
    );

    let is_prefix_probe = has_path_prefix(path, crate::server::RUSTFS_ADMIN_PREFIX)
        || has_path_prefix(path, crate::server::MINIO_ADMIN_V3_PREFIX)
        || is_table_catalog_path(path)
        || has_path_prefix(path, crate::server::CONSOLE_PREFIX)
        || has_path_prefix(path, crate::server::RPC_PREFIX)
        || has_path_prefix(path, crate::server::ADMIN_PREFIX)
        || has_path_prefix(path, crate::server::MINIO_ADMIN_PREFIX)
        || has_path_prefix(path, crate::server::TONIC_PREFIX);

    is_exact_probe || is_prefix_probe
}

fn readiness_gate_blocks_path(path: &str, readiness: &GlobalReadiness) -> bool {
    !is_probe_path(path) && !readiness.is_ready()
}

type BoxError = Box<dyn std::error::Error + Send + Sync>;
type BoxBody = http_body_util::combinators::UnsyncBoxBody<Bytes, BoxError>;

/// Header exposing which startup dependency the readiness gate is waiting
/// on, so operators can diagnose a 503 without shell access (rustfs#4304).
const READINESS_PENDING_HEADER: &str = "x-rustfs-readiness-pending";

/// Maps the current startup stage to the dependency the gate is waiting on.
fn readiness_pending_dependency(stage: rustfs_common::SystemStage) -> &'static str {
    match stage {
        rustfs_common::SystemStage::Booting => "storage_quorum",
        rustfs_common::SystemStage::StorageReady => "iam",
        rustfs_common::SystemStage::IamReady | rustfs_common::SystemStage::FullReady => "startup_finalization",
    }
}

fn service_not_ready_response(stage: rustfs_common::SystemStage) -> Response<BoxBody> {
    let pending = readiness_pending_dependency(stage);
    let body: BoxBody = Full::new(Bytes::from(format!("Service not ready: waiting for {pending}")))
        .map_err(|e| -> BoxError { Box::new(e) })
        .boxed_unsync();

    let mut response = Response::new(body);
    *response.status_mut() = StatusCode::SERVICE_UNAVAILABLE;
    response
        .headers_mut()
        .insert(http::header::RETRY_AFTER, HeaderValue::from_static("5"));
    response
        .headers_mut()
        .insert(http::header::CONTENT_TYPE, HeaderValue::from_static("text/plain; charset=utf-8"));
    response
        .headers_mut()
        .insert(http::header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
    response
        .headers_mut()
        .insert(READINESS_PENDING_HEADER, HeaderValue::from_static(pending));
    response
}

impl<S, B> Service<HttpRequest<Incoming>> for ReadinessGateService<S>
where
    S: Service<HttpRequest<Incoming>, Response = Response<B>> + Clone + Send + 'static,
    S::Future: Send + 'static,
    S::Error: Send + 'static,
    B: Body<Data = Bytes> + Send + 'static,
    B::Error: Into<BoxError> + Send + 'static,
{
    type Response = Response<BoxBody>;
    type Error = S::Error;
    type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;

    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
        self.inner.poll_ready(cx)
    }

    fn call(&mut self, req: HttpRequest<Incoming>) -> Self::Future {
        let mut inner = self.inner.clone();
        let readiness = self.readiness.clone();
        Box::pin(async move {
            let path = req.uri().path();
            debug!("ReadinessGateService: Received request for path: {}", path);
            if readiness_gate_blocks_path(path, &readiness) {
                return Ok(service_not_ready_response(readiness.current_stage()));
            }
            if !is_probe_path(path) && readiness.is_ready() {
                mark_get_metadata_read_version_coalescing_service_ready();
            }
            let resp = inner.call(req).await?;
            // System is ready, forward to the actual S3/RPC handlers
            // Transparently converts any response body into a BoxBody, and then Trace/Cors/Compression continues to work
            let (parts, body) = resp.into_parts();
            let body: BoxBody = body.map_err(Into::into).boxed_unsync();
            Ok(Response::from_parts(parts, body))
        })
    }
}

pub async fn publish_ready_when_runtime_ready(
    readiness: &GlobalReadiness,
    state_manager: Option<&ServiceStateManager>,
) -> Result<(), std::io::Error> {
    wait_for_runtime_readiness_with(
        startup_runtime_readiness_max_wait(),
        STARTUP_RUNTIME_READINESS_POLL_INTERVAL,
        collect_node_readiness,
        |dependency_readiness| {
            readiness.mark_stage(rustfs_common::SystemStage::FullReady);
            mark_get_metadata_read_version_coalescing_service_ready();
            if let Some(state_manager) = state_manager {
                state_manager.update(ServiceState::Ready);
            }
            info!(
                target: "rustfs::server::readiness",
                storage_ready = dependency_readiness.storage_ready,
                iam_ready = dependency_readiness.iam_ready,
                lock_quorum_ready = dependency_readiness.lock_quorum_ready,
                peer_health_ready = dependency_readiness.peer_health_ready,
                "Runtime node readiness reached; publishing ready state"
            );
        },
    )
    .await
}

#[derive(Debug, Clone, Copy)]
struct StorageReadinessCacheEntry {
    captured_at: Instant,
    storage_ready: bool,
}

#[derive(Debug, Clone, Copy)]
struct LockQuorumCacheEntry {
    captured_at: Instant,
    status: LockQuorumStatus,
}

#[derive(Debug, Clone)]
struct ClusterHealthReportCacheEntry {
    captured_at: Instant,
    report: DependencyReadinessReport,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ClusterHealthProbeKind {
    Write,
    Read,
}

#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct LockQuorumStatus {
    pub ready: bool,
    pub connected_clients: usize,
    pub total_clients: usize,
    pub required_quorum: usize,
}

const DISK_STATE_OK: &str = "ok";
const DISK_STATE_UNFORMATTED: &str = "unformatted";
const RUNTIME_STATE_RETURNING: &str = "returning";

fn health_readiness_cache_ttl() -> Duration {
    Duration::from_millis(rustfs_utils::get_env_u64(
        rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS,
        rustfs_config::DEFAULT_HEALTH_READINESS_CACHE_TTL_MS,
    ))
}

fn health_cluster_timeout() -> Duration {
    Duration::from_millis(
        rustfs_utils::get_env_u64(
            rustfs_config::ENV_HEALTH_CLUSTER_TIMEOUT_MS,
            rustfs_config::DEFAULT_HEALTH_CLUSTER_TIMEOUT_MS,
        )
        .max(1),
    )
}

fn storage_readiness_cache() -> &'static Mutex<Option<StorageReadinessCacheEntry>> {
    static CACHE: OnceLock<Mutex<Option<StorageReadinessCacheEntry>>> = OnceLock::new();
    CACHE.get_or_init(|| Mutex::new(None))
}

fn lock_quorum_status_cache() -> &'static Mutex<Option<LockQuorumCacheEntry>> {
    static CACHE: OnceLock<Mutex<Option<LockQuorumCacheEntry>>> = OnceLock::new();
    CACHE.get_or_init(|| Mutex::new(None))
}

fn cluster_write_health_report_cache() -> &'static Mutex<Option<ClusterHealthReportCacheEntry>> {
    static CACHE: OnceLock<Mutex<Option<ClusterHealthReportCacheEntry>>> = OnceLock::new();
    CACHE.get_or_init(|| Mutex::new(None))
}

fn cluster_read_health_report_cache() -> &'static Mutex<Option<ClusterHealthReportCacheEntry>> {
    static CACHE: OnceLock<Mutex<Option<ClusterHealthReportCacheEntry>>> = OnceLock::new();
    CACHE.get_or_init(|| Mutex::new(None))
}

fn cluster_write_health_singleflight() -> &'static Mutex<()> {
    static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
    LOCK.get_or_init(|| Mutex::new(()))
}

fn cluster_read_health_singleflight() -> &'static Mutex<()> {
    static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
    LOCK.get_or_init(|| Mutex::new(()))
}

fn cluster_health_report_cache(kind: ClusterHealthProbeKind) -> &'static Mutex<Option<ClusterHealthReportCacheEntry>> {
    match kind {
        ClusterHealthProbeKind::Write => cluster_write_health_report_cache(),
        ClusterHealthProbeKind::Read => cluster_read_health_report_cache(),
    }
}

fn cluster_health_singleflight(kind: ClusterHealthProbeKind) -> &'static Mutex<()> {
    match kind {
        ClusterHealthProbeKind::Write => cluster_write_health_singleflight(),
        ClusterHealthProbeKind::Read => cluster_read_health_singleflight(),
    }
}

#[cfg(test)]
async fn reset_cluster_health_report_caches() {
    *cluster_write_health_report_cache().lock().await = None;
    *cluster_read_health_report_cache().lock().await = None;
}

async fn load_cached_storage_readiness() -> Option<bool> {
    let ttl = health_readiness_cache_ttl();
    if ttl.is_zero() {
        return None;
    }

    let cache = storage_readiness_cache().lock().await;
    let entry = cache.as_ref()?;
    if entry.captured_at.elapsed() <= ttl {
        return Some(entry.storage_ready);
    }

    None
}

async fn load_cached_cluster_health_report(kind: ClusterHealthProbeKind) -> Option<DependencyReadinessReport> {
    let ttl = health_readiness_cache_ttl();
    if ttl.is_zero() {
        return None;
    }

    let cache = cluster_health_report_cache(kind).lock().await;
    let entry = cache.as_ref()?;
    if entry.captured_at.elapsed() <= ttl {
        return Some(entry.report.clone());
    }

    None
}

async fn update_cluster_health_report_cache(kind: ClusterHealthProbeKind, report: DependencyReadinessReport) {
    if health_readiness_cache_ttl().is_zero() {
        return;
    }

    let mut cache = cluster_health_report_cache(kind).lock().await;
    *cache = Some(ClusterHealthReportCacheEntry {
        captured_at: Instant::now(),
        report,
    });
}

async fn update_storage_readiness_cache(storage_ready: bool) {
    if health_readiness_cache_ttl().is_zero() {
        return;
    }

    let mut cache = storage_readiness_cache().lock().await;
    *cache = Some(StorageReadinessCacheEntry {
        captured_at: Instant::now(),
        storage_ready,
    });
}

async fn load_cached_lock_quorum_status() -> Option<LockQuorumStatus> {
    let ttl = health_readiness_cache_ttl();
    if ttl.is_zero() {
        return None;
    }

    let cache = lock_quorum_status_cache().lock().await;
    let entry = cache.as_ref()?;
    if entry.captured_at.elapsed() <= ttl {
        return Some(entry.status);
    }

    None
}

async fn update_lock_quorum_status_cache(status: LockQuorumStatus) {
    if health_readiness_cache_ttl().is_zero() {
        return;
    }

    let mut cache = lock_quorum_status_cache().lock().await;
    *cache = Some(LockQuorumCacheEntry {
        captured_at: Instant::now(),
        status,
    });
}

fn disk_is_online_for_readiness(disk: &Disk) -> bool {
    let state_is_acceptable = disk.state.eq_ignore_ascii_case(DISK_STATE_OK)
        || disk.state.eq_ignore_ascii_case(rustfs_madmin::ITEM_ONLINE)
        || disk.state.eq_ignore_ascii_case(DISK_STATE_UNFORMATTED);

    if let Some(runtime_state) = disk.runtime_state.as_deref() {
        let runtime_state_is_acceptable = runtime_state.eq_ignore_ascii_case(rustfs_madmin::ITEM_ONLINE)
            || runtime_state.eq_ignore_ascii_case(RUNTIME_STATE_RETURNING);
        return runtime_state_is_acceptable && state_is_acceptable;
    }

    state_is_acceptable
}

fn pool_erasure_layout(info: &StorageInfo, pool_idx: usize, set_drive_count: usize) -> Option<(usize, usize)> {
    if set_drive_count == 0 {
        return None;
    }

    if !info.backend.drives_per_set.is_empty() {
        if info.backend.drives_per_set.get(pool_idx).copied() != Some(set_drive_count) {
            return None;
        }
        if (!info.backend.standard_sc_data.is_empty() && info.backend.standard_sc_data.len() != info.backend.drives_per_set.len())
            || (!info.backend.standard_sc_parities.is_empty()
                && info.backend.standard_sc_parities.len() != info.backend.drives_per_set.len())
        {
            return None;
        }
    }

    let has_data = !info.backend.standard_sc_data.is_empty();
    let has_parities = !info.backend.standard_sc_parities.is_empty();
    let (data_drives, parity_drives) = match (has_data, has_parities) {
        (true, true) => (
            info.backend.standard_sc_data.get(pool_idx).copied()?,
            info.backend.standard_sc_parities.get(pool_idx).copied()?,
        ),
        (true, false) => {
            let data = info.backend.standard_sc_data.get(pool_idx).copied()?;
            // The per-pool data vector is exact. A legacy scalar may be stale
            // for a heterogeneous topology, so derive the matching parity from
            // the pool drive count instead of combining two representations.
            let parity = set_drive_count.checked_sub(data)?;
            (data, parity)
        }
        (false, true) => {
            let parity = info.backend.standard_sc_parities.get(pool_idx).copied()?;
            (set_drive_count.checked_sub(parity)?, parity)
        }
        (false, false) => {
            let parity = info.backend.standard_sc_parity?;
            (set_drive_count.checked_sub(parity)?, parity)
        }
    };

    if data_drives == 0 || parity_drives > data_drives || data_drives.checked_add(parity_drives) != Some(set_drive_count) {
        return None;
    }

    Some((data_drives, parity_drives))
}

fn pool_write_quorum(info: &StorageInfo, pool_idx: usize, set_drive_count: usize) -> Option<usize> {
    let (data_drives, parity_drives) = pool_erasure_layout(info, pool_idx, set_drive_count)?;
    if data_drives == parity_drives {
        data_drives.checked_add(1)
    } else {
        Some(data_drives)
    }
}

fn pool_read_quorum(info: &StorageInfo, pool_idx: usize, set_drive_count: usize) -> Option<usize> {
    pool_erasure_layout(info, pool_idx, set_drive_count).map(|(data_drives, _)| data_drives)
}

fn configured_readiness_topology(info: &StorageInfo) -> Option<(&[usize], &[usize])> {
    if info.backend.total_sets.is_empty()
        || info.backend.total_sets.len() != info.backend.drives_per_set.len()
        || info.backend.total_sets.contains(&0)
        || info.backend.drives_per_set.contains(&0)
    {
        return None;
    }

    Some((&info.backend.total_sets, &info.backend.drives_per_set))
}

fn storage_ready_from_runtime_state_with_quorum<F>(info: &StorageInfo, quorum_for_set: F) -> bool
where
    F: Fn(&StorageInfo, usize, usize) -> Option<usize>,
{
    if info.disks.is_empty() {
        return false;
    }

    let configured_topology_present = !info.backend.total_sets.is_empty() || !info.backend.drives_per_set.is_empty();
    let mut total_online = 0usize;
    let mut set_online_counts: HashMap<(usize, usize), usize> = HashMap::new();
    let mut observed_set_drive_counts: HashMap<(usize, usize), usize> = HashMap::new();
    let mut seen_disks: HashSet<(String, String, i32, i32, i32)> = HashSet::new();

    for disk in &info.disks {
        if disk.pool_index < 0 || disk.set_index < 0 {
            continue;
        }

        let dedup_key = (
            disk.endpoint.clone(),
            disk.drive_path.clone(),
            disk.pool_index,
            disk.set_index,
            disk.disk_index,
        );
        if !seen_disks.insert(dedup_key) {
            continue;
        }

        let pool_idx = disk.pool_index as usize;
        let set_idx = disk.set_index as usize;
        let key = (pool_idx, set_idx);
        if !configured_topology_present {
            *observed_set_drive_counts.entry(key).or_default() += 1;
        }

        if disk_is_online_for_readiness(disk) {
            total_online += 1;
            *set_online_counts.entry(key).or_default() += 1;
        }
    }

    if total_online == 0 {
        return false;
    }

    if configured_topology_present {
        let Some((total_sets, drives_per_set)) = configured_readiness_topology(info) else {
            return false;
        };

        return total_sets
            .iter()
            .zip(drives_per_set)
            .enumerate()
            .all(|(pool_idx, (&set_count, &configured_drive_count))| {
                (0..set_count).all(|set_idx| {
                    let online = set_online_counts.get(&(pool_idx, set_idx)).copied().unwrap_or_default();
                    quorum_for_set(info, pool_idx, configured_drive_count).is_some_and(|quorum| online >= quorum)
                })
            });
    }

    if observed_set_drive_counts.is_empty() {
        return false;
    }

    observed_set_drive_counts
        .into_iter()
        .all(|((pool_idx, set_idx), observed_drive_count)| {
            let online = set_online_counts.get(&(pool_idx, set_idx)).copied().unwrap_or_default();
            let layout_drive_count = info
                .backend
                .drives_per_set
                .get(pool_idx)
                .copied()
                .unwrap_or(observed_drive_count);
            quorum_for_set(info, pool_idx, layout_drive_count).is_some_and(|quorum| online >= quorum)
        })
}

fn storage_ready_from_runtime_state(info: &StorageInfo) -> bool {
    storage_ready_from_runtime_state_with_quorum(info, pool_write_quorum)
}

fn storage_read_ready_from_runtime_state(info: &StorageInfo) -> bool {
    storage_ready_from_runtime_state_with_quorum(info, pool_read_quorum)
}

fn peer_health_ready_check_enabled() -> bool {
    rustfs_utils::get_env_bool(
        rustfs_config::ENV_HEALTH_PEER_READY_CHECK_ENABLE,
        rustfs_config::DEFAULT_HEALTH_PEER_READY_CHECK_ENABLE,
    )
}

fn peer_health_ready_from_endpoint_pools(endpoint_pools: &EndpointServerPools) -> bool {
    if !peer_health_ready_check_enabled() {
        return true;
    }

    ClusterControlPlane::new(endpoint_pools.clone())
        .peer_health_snapshot()
        .peers
        .iter()
        .all(|peer| peer.status.state.is_supported())
}

fn collect_peer_health_readiness() -> bool {
    if !peer_health_ready_check_enabled() {
        return true;
    }

    runtime_sources::current_endpoints_handle()
        .as_ref()
        .is_some_and(peer_health_ready_from_endpoint_pools)
}

fn base_degraded_reasons(storage_ready: bool, iam_ready_raw: bool, lock_quorum_ready: bool) -> Vec<ReadinessDegradedReason> {
    match (storage_ready, iam_ready_raw, lock_quorum_ready) {
        (true, true, true) => Vec::new(),
        (false, false, false) => vec![ReadinessDegradedReason::StorageIamAndLockUnavailable],
        (false, false, true) => vec![ReadinessDegradedReason::StorageAndIamUnavailable],
        (false, true, false) => vec![ReadinessDegradedReason::StorageAndLockUnavailable],
        (true, false, false) => vec![ReadinessDegradedReason::IamAndLockUnavailable],
        (false, true, true) => vec![ReadinessDegradedReason::StorageQuorumUnavailable],
        (true, false, true) => vec![ReadinessDegradedReason::IamNotReady],
        (true, true, false) => vec![ReadinessDegradedReason::LockQuorumUnavailable],
    }
}

fn degraded_reasons(readiness: DependencyReadiness) -> Vec<ReadinessDegradedReason> {
    let mut reasons = base_degraded_reasons(readiness.storage_ready, readiness.iam_ready, readiness.lock_quorum_ready);
    if !readiness.peer_health_ready {
        reasons.push(ReadinessDegradedReason::PeerHealthUnavailable);
    }
    reasons
}

fn record_readiness_report(report: &DependencyReadinessReport) {
    let ready = report.readiness.storage_ready
        && report.readiness.iam_ready
        && report.readiness.lock_quorum_ready
        && report.readiness.peer_health_ready;
    gauge!(METRIC_RUNTIME_READINESS_READY).set(if ready { 1.0 } else { 0.0 });
    for reason in &report.degraded_reasons {
        counter!(METRIC_RUNTIME_READINESS_DEGRADED_TOTAL, "reason" => reason.as_str()).increment(1);
    }
}

pub(crate) fn record_readiness_overlay_reason(reason: ReadinessDegradedReason) {
    gauge!(METRIC_RUNTIME_READINESS_READY).set(0.0);
    counter!(METRIC_RUNTIME_READINESS_DEGRADED_TOTAL, "reason" => reason.as_str()).increment(1);
}

fn dependency_readiness_report_from_readiness(readiness: DependencyReadiness) -> DependencyReadinessReport {
    DependencyReadinessReport {
        degraded_reasons: degraded_reasons(readiness),
        readiness,
    }
}

pub async fn collect_dependency_readiness_report() -> DependencyReadinessReport {
    let iam_ready_raw = runtime_sources::current_iam_ready();
    let storage_ready = if let Some(cached) = load_cached_storage_readiness().await {
        cached
    } else {
        let computed = collect_storage_readiness_uncached().await;
        update_storage_readiness_cache(computed).await;
        computed
    };
    let lock_quorum_status = collect_lock_quorum_status().await;

    let readiness = DependencyReadiness {
        storage_ready,
        iam_ready: iam_ready_raw,
        lock_quorum_ready: lock_quorum_status.ready,
        peer_health_ready: collect_peer_health_readiness(),
    };
    let report = dependency_readiness_report_from_readiness(readiness);
    record_readiness_report(&report);
    report
}

pub async fn collect_cluster_write_health_report() -> DependencyReadinessReport {
    collect_cluster_health_report_with(ClusterHealthProbeKind::Write, collect_dependency_readiness_report).await
}

pub async fn collect_cluster_read_health_report() -> DependencyReadinessReport {
    collect_cluster_health_report_with(ClusterHealthProbeKind::Read, collect_cluster_read_dependency_readiness_report).await
}

pub async fn collect_node_readiness_report() -> DependencyReadinessReport {
    let readiness = DependencyReadiness {
        storage_ready: runtime_sources::current_object_store_handle().is_some(),
        iam_ready: runtime_sources::current_iam_ready(),
        lock_quorum_ready: collect_lock_quorum_status().await.ready,
        peer_health_ready: collect_peer_health_readiness(),
    };
    let report = dependency_readiness_report_from_readiness(readiness);
    record_readiness_report(&report);
    report
}

async fn collect_cluster_health_report_with<LoadFn, Fut>(
    kind: ClusterHealthProbeKind,
    mut load_report: LoadFn,
) -> DependencyReadinessReport
where
    LoadFn: FnMut() -> Fut,
    Fut: Future<Output = DependencyReadinessReport>,
{
    if let Some(cached) = load_cached_cluster_health_report(kind).await {
        return cached;
    }

    let _singleflight = cluster_health_singleflight(kind).lock().await;
    if let Some(cached) = load_cached_cluster_health_report(kind).await {
        return cached;
    }

    let report = match tokio::time::timeout(health_cluster_timeout(), load_report()).await {
        Ok(report) => report,
        Err(_) => cluster_health_timeout_report(),
    };
    update_cluster_health_report_cache(kind, report.clone()).await;
    report
}

fn cluster_health_timeout_report() -> DependencyReadinessReport {
    DependencyReadinessReport {
        readiness: DependencyReadiness {
            storage_ready: false,
            iam_ready: false,
            lock_quorum_ready: false,
            peer_health_ready: false,
        },
        degraded_reasons: vec![ReadinessDegradedReason::ClusterHealthTimeout],
    }
}

async fn collect_node_readiness() -> DependencyReadiness {
    collect_node_readiness_report().await.readiness
}

pub async fn collect_cluster_read_dependency_readiness_report() -> DependencyReadinessReport {
    let iam_ready_raw = runtime_sources::current_iam_ready();
    let storage_ready = collect_storage_read_readiness_uncached().await;
    let lock_quorum_status = collect_lock_quorum_status().await;

    let readiness = DependencyReadiness {
        storage_ready,
        iam_ready: iam_ready_raw,
        lock_quorum_ready: lock_quorum_status.ready,
        peer_health_ready: collect_peer_health_readiness(),
    };
    let report = dependency_readiness_report_from_readiness(readiness);
    record_readiness_report(&report);
    report
}

pub(crate) async fn snapshot_dependency_readiness_report() -> DependencyReadinessReport {
    let readiness = DependencyReadiness {
        storage_ready: collect_storage_readiness_uncached().await,
        iam_ready: runtime_sources::current_iam_ready(),
        lock_quorum_ready: collect_lock_quorum_status_uncached().await.ready,
        peer_health_ready: collect_peer_health_readiness(),
    };

    dependency_readiness_report_from_readiness(readiness)
}

async fn collect_lock_quorum_status() -> LockQuorumStatus {
    if let Some(cached) = load_cached_lock_quorum_status().await {
        cached
    } else {
        let computed = collect_lock_quorum_status_uncached().await;
        update_lock_quorum_status_cache(computed).await;
        computed
    }
}

async fn collect_storage_readiness_uncached() -> bool {
    if let Some(store) = runtime_sources::current_object_store_handle() {
        let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
        storage_ready_from_runtime_state(&storage_info)
    } else {
        false
    }
}

async fn collect_storage_read_readiness_uncached() -> bool {
    if let Some(store) = runtime_sources::current_object_store_handle() {
        let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
        storage_read_ready_from_runtime_state(&storage_info)
    } else {
        false
    }
}

fn set_lock_quorum_status(online_hosts: &HashSet<String>, set_endpoints: &[Endpoint]) -> LockQuorumStatus {
    let total_clients = set_endpoints
        .iter()
        .map(Endpoint::host_port)
        .filter(|host| !host.is_empty())
        .collect::<HashSet<_>>();
    let total_clients_len = total_clients.len();
    if total_clients_len == 0 {
        return LockQuorumStatus::default();
    }

    let connected_clients = total_clients.iter().filter(|host| online_hosts.contains(*host)).count();
    let required_quorum = if total_clients_len > 1 {
        (total_clients_len / 2) + 1
    } else {
        1
    };

    LockQuorumStatus {
        ready: connected_clients >= required_quorum,
        connected_clients,
        total_clients: total_clients_len,
        required_quorum,
    }
}

fn aggregate_lock_quorum_status(pool_endpoints: &EndpointServerPools, online_hosts: &HashSet<String>) -> LockQuorumStatus {
    let mut connected_clients = 0usize;
    let mut total_clients = 0usize;
    let mut required_quorum = 0usize;

    for pool in pool_endpoints.as_ref() {
        for set_idx in 0..pool.set_count {
            let set_endpoints = pool
                .endpoints
                .as_ref()
                .iter()
                .filter(|endpoint| endpoint.set_idx == set_idx as i32)
                .cloned()
                .collect::<Vec<_>>();

            let status = set_lock_quorum_status(online_hosts, &set_endpoints);
            if status.total_clients == 0 {
                return LockQuorumStatus::default();
            }

            connected_clients += status.connected_clients;
            total_clients += status.total_clients;
            required_quorum += status.required_quorum;

            if !status.ready {
                return LockQuorumStatus {
                    ready: false,
                    connected_clients,
                    total_clients,
                    required_quorum,
                };
            }
        }
    }

    if total_clients == 0 {
        LockQuorumStatus::default()
    } else {
        LockQuorumStatus {
            ready: true,
            connected_clients,
            total_clients,
            required_quorum,
        }
    }
}

async fn collect_lock_quorum_status_uncached() -> LockQuorumStatus {
    if !is_dist_erasure().await {
        return LockQuorumStatus {
            ready: true,
            connected_clients: 1,
            total_clients: 1,
            required_quorum: 1,
        };
    }

    let Some(pool_endpoints) = runtime_sources::current_endpoints_handle() else {
        return LockQuorumStatus::default();
    };
    let Some(lock_clients) = runtime_sources::current_lock_clients_handle() else {
        return LockQuorumStatus::default();
    };

    let online_hosts = futures::future::join_all(lock_clients.iter().map(|(host, client)| {
        let host = host.clone();
        let client = client.clone();
        async move { (host, client.is_online().await) }
    }))
    .await
    .into_iter()
    .filter_map(|(host, online)| online.then_some(host))
    .collect::<HashSet<_>>();

    aggregate_lock_quorum_status(&pool_endpoints, &online_hosts)
}

pub async fn wait_for_runtime_readiness_with<F, Fut, ReadyFn>(
    max_wait: Duration,
    poll_interval: Duration,
    mut load_readiness: F,
    mut on_ready: ReadyFn,
) -> Result<(), std::io::Error>
where
    F: FnMut() -> Fut,
    Fut: Future<Output = DependencyReadiness>,
    ReadyFn: FnMut(DependencyReadiness),
{
    let startup_deadline = tokio::time::Instant::now() + max_wait;

    loop {
        let readiness = load_readiness().await;
        if readiness.storage_ready && readiness.iam_ready && readiness.lock_quorum_ready && readiness.peer_health_ready {
            on_ready(readiness);
            return Ok(());
        }

        if tokio::time::Instant::now() >= startup_deadline {
            let reason = format!(
                "startup readiness timed out after {}s: storage_ready={}, iam_ready={}, lock_quorum_ready={}, peer_health_ready={}",
                max_wait.as_secs(),
                readiness.storage_ready,
                readiness.iam_ready,
                readiness.lock_quorum_ready,
                readiness.peer_health_ready
            );
            return Err(std::io::Error::other(reason));
        }

        info!(
            target: "rustfs::server::readiness",
            storage_ready = readiness.storage_ready,
            iam_ready = readiness.iam_ready,
            lock_quorum_ready = readiness.lock_quorum_ready,
            peer_health_ready = readiness.peer_health_ready,
            "Runtime node readiness has not been reached yet; delaying ready state publication"
        );
        tokio::time::sleep(poll_interval).await;
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use rustfs_madmin::{BackendInfo, Disk};
    use serial_test::serial;
    use std::future;
    use std::sync::atomic::{AtomicUsize, Ordering};
    use temp_env::{async_with_vars, with_var};

    fn online_readiness_disks(set_idx: i32, count: i32) -> Vec<Disk> {
        (0..count)
            .map(|disk_index| Disk {
                endpoint: format!("node-{set_idx}-{disk_index}:9000"),
                drive_path: format!("/set{set_idx}/data{disk_index}"),
                pool_index: 0,
                set_index: set_idx,
                disk_index,
                state: "ok".to_string(),
                runtime_state: Some("online".to_string()),
                ..Default::default()
            })
            .collect()
    }

    #[test]
    fn startup_runtime_readiness_wait_constants_are_ordered() {
        assert!(STARTUP_RUNTIME_READINESS_MAX_WAIT > STARTUP_RUNTIME_READINESS_POLL_INTERVAL);
        assert_eq!(
            STARTUP_RUNTIME_READINESS_MAX_WAIT.as_secs(),
            rustfs_config::DEFAULT_STARTUP_READINESS_MAX_WAIT_SECS
        );
        assert_eq!(STARTUP_RUNTIME_READINESS_POLL_INTERVAL.as_secs(), 1);
    }

    #[test]
    #[serial]
    fn startup_runtime_readiness_max_wait_reads_env_override() {
        // An explicit override extends (or shortens) the budget.
        with_var(rustfs_config::ENV_STARTUP_READINESS_MAX_WAIT_SECS, Some("240"), || {
            assert_eq!(startup_runtime_readiness_max_wait(), Duration::from_secs(240));
        });

        // Unset falls back to the compile-time default.
        with_var(rustfs_config::ENV_STARTUP_READINESS_MAX_WAIT_SECS, None::<&str>, || {
            assert_eq!(startup_runtime_readiness_max_wait(), STARTUP_RUNTIME_READINESS_MAX_WAIT);
        });

        // Zero is treated as "use default", never an instant timeout.
        with_var(rustfs_config::ENV_STARTUP_READINESS_MAX_WAIT_SECS, Some("0"), || {
            assert_eq!(startup_runtime_readiness_max_wait(), STARTUP_RUNTIME_READINESS_MAX_WAIT);
        });
    }

    fn peer_health_test_pools() -> EndpointServerPools {
        EndpointServerPools::from(vec![PoolEndpoints {
            legacy: false,
            set_count: 1,
            drives_per_set: 2,
            endpoints: Endpoints::from(vec![
                Endpoint {
                    url: url::Url::parse("http://node1:9000/data1").expect("test endpoint url"),
                    is_local: true,
                    pool_idx: 0,
                    set_idx: 0,
                    disk_idx: 0,
                },
                Endpoint {
                    url: url::Url::parse("http://node2:9000/data2").expect("test endpoint url"),
                    is_local: false,
                    pool_idx: 0,
                    set_idx: 0,
                    disk_idx: 1,
                },
            ]),
            cmd_line: String::new(),
            platform: String::new(),
        }])
    }

    #[test]
    fn peer_health_gate_defaults_to_ready() {
        with_var(rustfs_config::ENV_HEALTH_PEER_READY_CHECK_ENABLE, Some("false"), || {
            assert!(peer_health_ready_from_endpoint_pools(&peer_health_test_pools()));
        });
    }

    #[test]
    fn peer_health_gate_degrades_unknown_peer_health_when_enabled() {
        with_var(rustfs_config::ENV_HEALTH_PEER_READY_CHECK_ENABLE, Some("true"), || {
            assert!(!peer_health_ready_from_endpoint_pools(&peer_health_test_pools()));
        });
    }

    #[tokio::test]
    #[serial]
    async fn cluster_health_report_singleflight_reuses_inflight_result() {
        reset_cluster_health_report_caches().await;
        async_with_vars(
            [
                (rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS, Some("60000")),
                (rustfs_config::ENV_HEALTH_CLUSTER_TIMEOUT_MS, Some("1000")),
            ],
            async {
                let calls = Arc::new(AtomicUsize::new(0));
                let ready_report = DependencyReadinessReport {
                    readiness: DependencyReadiness {
                        storage_ready: true,
                        iam_ready: true,
                        lock_quorum_ready: true,
                        peer_health_ready: true,
                    },
                    degraded_reasons: Vec::new(),
                };

                let first_calls = calls.clone();
                let first_report = ready_report.clone();
                let first = collect_cluster_health_report_with(ClusterHealthProbeKind::Write, move || {
                    let first_calls = first_calls.clone();
                    let first_report = first_report.clone();
                    async move {
                        first_calls.fetch_add(1, Ordering::SeqCst);
                        tokio::time::sleep(Duration::from_millis(20)).await;
                        first_report
                    }
                });

                let second_calls = calls.clone();
                let second_report = ready_report.clone();
                let second = collect_cluster_health_report_with(ClusterHealthProbeKind::Write, move || {
                    let second_calls = second_calls.clone();
                    let second_report = second_report.clone();
                    async move {
                        second_calls.fetch_add(1, Ordering::SeqCst);
                        second_report
                    }
                });

                let (first, second) = tokio::join!(first, second);

                assert_eq!(calls.load(Ordering::SeqCst), 1);
                assert_eq!(first, ready_report);
                assert_eq!(second, ready_report);
            },
        )
        .await;
        reset_cluster_health_report_caches().await;
    }

    #[tokio::test]
    #[serial]
    async fn cluster_health_report_timeout_is_sanitized() {
        reset_cluster_health_report_caches().await;
        async_with_vars(
            [
                (rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS, Some("0")),
                (rustfs_config::ENV_HEALTH_CLUSTER_TIMEOUT_MS, Some("1")),
            ],
            async {
                let report = collect_cluster_health_report_with(ClusterHealthProbeKind::Read, || async {
                    tokio::time::sleep(Duration::from_millis(50)).await;
                    DependencyReadinessReport {
                        readiness: DependencyReadiness {
                            storage_ready: true,
                            iam_ready: true,
                            lock_quorum_ready: true,
                            peer_health_ready: true,
                        },
                        degraded_reasons: Vec::new(),
                    }
                })
                .await;

                assert_eq!(report, cluster_health_timeout_report());
                assert_eq!(report.degraded_reasons, vec![ReadinessDegradedReason::ClusterHealthTimeout]);
            },
        )
        .await;
        reset_cluster_health_report_caches().await;
    }

    #[tokio::test]
    async fn wait_for_runtime_readiness_with_does_not_publish_ready_when_runtime_readiness_is_not_reached() {
        let readiness = GlobalReadiness::new();
        let state_manager = ServiceStateManager::new();

        let err = wait_for_runtime_readiness_with(
            Duration::ZERO,
            Duration::from_millis(1),
            || {
                future::ready(DependencyReadiness {
                    storage_ready: false,
                    iam_ready: false,
                    lock_quorum_ready: false,
                    peer_health_ready: true,
                })
            },
            |_| {
                readiness.mark_stage(rustfs_common::SystemStage::FullReady);
                state_manager.update(ServiceState::Ready);
            },
        )
        .await
        .expect_err("unready startup should time out");

        assert!(err.to_string().contains("startup readiness timed out"));
        assert!(!readiness.is_ready());
        assert_eq!(state_manager.current_state(), ServiceState::Starting);
    }

    #[tokio::test]
    async fn wait_for_runtime_readiness_with_does_not_publish_ready_without_lock_quorum() {
        let readiness = GlobalReadiness::new();
        let state_manager = ServiceStateManager::new();

        let err = wait_for_runtime_readiness_with(
            Duration::ZERO,
            Duration::from_millis(1),
            || {
                future::ready(DependencyReadiness {
                    storage_ready: true,
                    iam_ready: true,
                    lock_quorum_ready: false,
                    peer_health_ready: true,
                })
            },
            |_| {
                readiness.mark_stage(rustfs_common::SystemStage::FullReady);
                state_manager.update(ServiceState::Ready);
            },
        )
        .await
        .expect_err("startup readiness should require lock quorum");

        assert!(err.to_string().contains("lock_quorum_ready=false"));
        assert!(!readiness.is_ready());
        assert_eq!(state_manager.current_state(), ServiceState::Starting);
    }

    #[tokio::test]
    async fn wait_for_runtime_readiness_with_publishes_ready_when_dependencies_are_ready() {
        let readiness = GlobalReadiness::new();
        let state_manager = ServiceStateManager::new();

        let result = wait_for_runtime_readiness_with(
            Duration::ZERO,
            Duration::from_millis(1),
            || {
                future::ready(DependencyReadiness {
                    storage_ready: true,
                    iam_ready: true,
                    lock_quorum_ready: true,
                    peer_health_ready: true,
                })
            },
            |_| {
                readiness.mark_stage(rustfs_common::SystemStage::FullReady);
                state_manager.update(ServiceState::Ready);
            },
        )
        .await;

        assert!(result.is_ok(), "all runtime dependencies should publish readiness");
        assert!(readiness.is_ready());
        assert_eq!(state_manager.current_state(), ServiceState::Ready);
    }

    #[test]
    fn probe_path_checks_admin_boundaries() {
        assert!(is_probe_path("/minio/admin/v3/info"));
        assert!(is_probe_path("/rustfs/admin/v3/info"));
        assert!(is_probe_path(&format!("{}/config", crate::server::TABLE_CATALOG_PREFIX)));
        assert!(is_probe_path("/_iceberg/v1/config"));
        assert!(is_probe_path("/rustfs/console/"));
        assert!(!is_probe_path("/minio/adminx/object"));
        assert!(!is_probe_path("/rustfs/adminx/object"));
        assert!(!is_probe_path("/bucket/object"));
    }

    #[test]
    fn readiness_gate_blocks_normal_paths_until_runtime_ready() {
        let readiness = GlobalReadiness::new();

        assert!(readiness_gate_blocks_path("/bucket/object", &readiness));
        assert!(!readiness_gate_blocks_path(crate::server::HEALTH_READY_PATH, &readiness));
        assert!(!readiness_gate_blocks_path("/minio/admin/v3/info", &readiness));

        readiness.mark_stage(rustfs_common::SystemStage::FullReady);
        assert!(!readiness_gate_blocks_path("/bucket/object", &readiness));
    }

    #[tokio::test]
    async fn service_not_ready_response_preserves_observable_contract() {
        let response = service_not_ready_response(rustfs_common::SystemStage::Booting);

        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
        assert_eq!(
            response
                .headers()
                .get(http::header::RETRY_AFTER)
                .and_then(|value| value.to_str().ok()),
            Some("5")
        );
        assert_eq!(
            response
                .headers()
                .get(http::header::CACHE_CONTROL)
                .and_then(|value| value.to_str().ok()),
            Some("no-store")
        );
        assert_eq!(
            response
                .headers()
                .get(http::header::CONTENT_TYPE)
                .and_then(|value| value.to_str().ok()),
            Some("text/plain; charset=utf-8")
        );
        assert_eq!(
            response
                .headers()
                .get(READINESS_PENDING_HEADER)
                .and_then(|value| value.to_str().ok()),
            Some("storage_quorum")
        );
        let body = match response.into_body().collect().await {
            Ok(body) => body.to_bytes(),
            Err(err) => panic!("not-ready body should collect: {err}"),
        };
        assert_eq!(body, Bytes::from_static(b"Service not ready: waiting for storage_quorum"));
    }

    /// rustfs#4304: operators must be able to tell from the 503 alone which
    /// startup dependency the node is blocked on.
    #[test]
    fn readiness_pending_dependency_maps_every_stage() {
        assert_eq!(readiness_pending_dependency(rustfs_common::SystemStage::Booting), "storage_quorum");
        assert_eq!(readiness_pending_dependency(rustfs_common::SystemStage::StorageReady), "iam");
        assert_eq!(readiness_pending_dependency(rustfs_common::SystemStage::IamReady), "startup_finalization");
    }

    #[test]
    fn pool_quorum_uses_exact_heterogeneous_backend_layout() {
        let info = StorageInfo {
            backend: BackendInfo {
                standard_sc_data: vec![2, 1],
                standard_sc_parities: vec![2, 1],
                // Deliberately stale: exact per-pool vectors must take precedence.
                standard_sc_parity: Some(2),
                drives_per_set: vec![4, 2],
                ..Default::default()
            },
            disks: Vec::new(),
        };

        assert_eq!(pool_erasure_layout(&info, 0, 4), Some((2, 2)));
        assert_eq!(pool_erasure_layout(&info, 1, 2), Some((1, 1)));
        assert_eq!(pool_write_quorum(&info, 0, 4), Some(3));
        assert_eq!(pool_write_quorum(&info, 1, 2), Some(2));
        assert_eq!(pool_read_quorum(&info, 0, 4), Some(2));
        assert_eq!(pool_read_quorum(&info, 1, 2), Some(1));
    }

    #[test]
    fn pool_write_quorum_does_not_fall_back_to_half_when_exact_data_exists() {
        let info = StorageInfo {
            backend: BackendInfo {
                standard_sc_data: vec![6],
                standard_sc_parities: vec![2],
                drives_per_set: vec![8],
                ..Default::default()
            },
            disks: Vec::new(),
        };

        assert_eq!(pool_write_quorum(&info, 0, 8), Some(6));
    }

    #[test]
    fn exact_data_vector_ignores_stale_scalar_for_runtime_readiness() {
        let info = StorageInfo {
            backend: BackendInfo {
                standard_sc_data: vec![6],
                // Deliberately stale: the exact data vector and topology imply
                // parity 2, so this legacy scalar must not make readiness fail.
                standard_sc_parity: Some(4),
                total_sets: vec![1],
                drives_per_set: vec![8],
                ..Default::default()
            },
            disks: online_readiness_disks(0, 8),
        };

        assert_eq!(pool_erasure_layout(&info, 0, 8), Some((6, 2)));
        assert!(storage_read_ready_from_runtime_state(&info));
        assert!(storage_ready_from_runtime_state(&info));
    }

    #[test]
    fn exact_data_vector_with_invalid_geometry_fails_closed() {
        let info = StorageInfo {
            backend: BackendInfo {
                standard_sc_data: vec![9],
                standard_sc_parity: Some(1),
                total_sets: vec![1],
                drives_per_set: vec![8],
                ..Default::default()
            },
            disks: online_readiness_disks(0, 8),
        };

        assert_eq!(pool_erasure_layout(&info, 0, 8), None);
        assert!(!storage_read_ready_from_runtime_state(&info));
        assert!(!storage_ready_from_runtime_state(&info));
    }

    #[test]
    fn pool_quorum_accepts_valid_legacy_scalar_layout() {
        let info = StorageInfo {
            backend: BackendInfo {
                standard_sc_parity: Some(1),
                drives_per_set: vec![4],
                ..Default::default()
            },
            disks: Vec::new(),
        };

        assert_eq!(pool_erasure_layout(&info, 0, 4), Some((3, 1)));
        assert_eq!(pool_write_quorum(&info, 0, 4), Some(3));
        assert_eq!(pool_read_quorum(&info, 0, 4), Some(3));
    }

    #[test]
    fn legacy_payload_without_topology_uses_observed_sets_and_scalar_layout() {
        let info = StorageInfo {
            backend: BackendInfo {
                standard_sc_parity: Some(1),
                ..Default::default()
            },
            disks: online_readiness_disks(0, 3),
        };

        assert_eq!(pool_erasure_layout(&info, 0, 3), Some((2, 1)));
        assert!(storage_read_ready_from_runtime_state(&info));
        assert!(storage_ready_from_runtime_state(&info));
    }

    #[test]
    fn configured_topology_uses_configured_drive_count_when_rows_are_missing() {
        let backend = BackendInfo {
            standard_sc_data: vec![2],
            standard_sc_parities: vec![2],
            total_sets: vec![1],
            drives_per_set: vec![4],
            ..Default::default()
        };
        let three_online = StorageInfo {
            backend: backend.clone(),
            disks: online_readiness_disks(0, 3),
        };
        let two_online = StorageInfo {
            backend,
            disks: online_readiness_disks(0, 2),
        };

        assert!(storage_read_ready_from_runtime_state(&three_online));
        assert!(storage_ready_from_runtime_state(&three_online));
        assert!(storage_read_ready_from_runtime_state(&two_online));
        assert!(!storage_ready_from_runtime_state(&two_online));
    }

    #[test]
    fn configured_topology_requires_sets_with_no_disk_rows() {
        let info = StorageInfo {
            backend: BackendInfo {
                standard_sc_data: vec![2],
                standard_sc_parities: vec![2],
                total_sets: vec![2],
                drives_per_set: vec![4],
                ..Default::default()
            },
            disks: online_readiness_disks(0, 3),
        };

        assert!(!storage_read_ready_from_runtime_state(&info));
        assert!(!storage_ready_from_runtime_state(&info));
    }

    #[test]
    fn configured_topology_fails_closed_with_only_total_sets() {
        let info = StorageInfo {
            backend: BackendInfo {
                standard_sc_parity: Some(1),
                total_sets: vec![1],
                ..Default::default()
            },
            disks: online_readiness_disks(0, 3),
        };

        assert!(!storage_read_ready_from_runtime_state(&info));
        assert!(!storage_ready_from_runtime_state(&info));
    }

    #[test]
    fn configured_topology_fails_closed_with_only_drives_per_set() {
        let info = StorageInfo {
            backend: BackendInfo {
                standard_sc_parity: Some(1),
                drives_per_set: vec![3],
                ..Default::default()
            },
            disks: online_readiness_disks(0, 3),
        };

        assert!(!storage_read_ready_from_runtime_state(&info));
        assert!(!storage_ready_from_runtime_state(&info));
    }

    #[test]
    fn storage_readiness_fails_closed_when_backend_layout_is_empty() {
        let disks = (0..4)
            .map(|disk_index| Disk {
                endpoint: format!("127.0.0.1:900{disk_index}"),
                drive_path: format!("/data{disk_index}"),
                pool_index: 0,
                set_index: 0,
                disk_index,
                state: "ok".to_string(),
                runtime_state: Some("online".to_string()),
                ..Default::default()
            })
            .collect();
        let info = StorageInfo {
            backend: BackendInfo {
                drives_per_set: vec![4],
                ..Default::default()
            },
            disks,
        };

        assert!(!storage_ready_from_runtime_state(&info));
        assert!(!storage_read_ready_from_runtime_state(&info));
    }

    #[test]
    fn storage_ready_from_runtime_state_returns_false_when_all_disks_faulty() {
        let info = StorageInfo {
            backend: BackendInfo {
                standard_sc_data: vec![1],
                total_sets: vec![1],
                drives_per_set: vec![1],
                ..Default::default()
            },
            disks: vec![Disk {
                pool_index: 0,
                set_index: 0,
                state: "offline".to_string(),
                runtime_state: Some("offline".to_string()),
                ..Default::default()
            }],
        };

        assert!(!storage_ready_from_runtime_state(&info));
    }

    #[test]
    fn storage_ready_from_runtime_state_returns_true_when_set_meets_write_quorum() {
        let info = StorageInfo {
            backend: BackendInfo {
                standard_sc_data: vec![1],
                total_sets: vec![1],
                drives_per_set: vec![1],
                ..Default::default()
            },
            disks: vec![Disk {
                pool_index: 0,
                set_index: 0,
                state: "ok".to_string(),
                runtime_state: Some("online".to_string()),
                ..Default::default()
            }],
        };

        assert!(storage_ready_from_runtime_state(&info));
    }

    #[test]
    fn storage_read_ready_from_runtime_state_allows_read_quorum_below_write_quorum() {
        let disks = (0..4)
            .map(|disk_index| Disk {
                endpoint: format!("127.0.0.1:900{disk_index}"),
                drive_path: format!("/data{disk_index}"),
                pool_index: 0,
                set_index: 0,
                disk_index,
                state: if disk_index < 2 { "ok" } else { "offline" }.to_string(),
                runtime_state: Some(if disk_index < 2 { "online" } else { "offline" }.to_string()),
                ..Default::default()
            })
            .collect::<Vec<_>>();
        let info = StorageInfo {
            backend: BackendInfo {
                standard_sc_data: vec![2],
                total_sets: vec![1],
                drives_per_set: vec![4],
                ..Default::default()
            },
            disks,
        };

        assert!(storage_read_ready_from_runtime_state(&info));
        assert!(!storage_ready_from_runtime_state(&info));
    }

    #[test]
    fn storage_ready_from_runtime_state_deduplicates_duplicate_disk_rows() {
        let duplicate_disk = Disk {
            endpoint: "127.0.0.1:9000".to_string(),
            drive_path: "/data0".to_string(),
            pool_index: 0,
            set_index: 0,
            disk_index: 0,
            state: "ok".to_string(),
            runtime_state: Some("online".to_string()),
            ..Default::default()
        };
        let info = StorageInfo {
            backend: BackendInfo {
                standard_sc_data: vec![1],
                total_sets: vec![1],
                drives_per_set: vec![2],
                ..Default::default()
            },
            disks: vec![duplicate_disk.clone(), duplicate_disk],
        };

        assert!(!storage_ready_from_runtime_state(&info), "duplicate rows must not satisfy write quorum");
    }

    #[test]
    fn disk_online_for_readiness_requires_runtime_and_state_both_acceptable() {
        let disk = Disk {
            state: "disk io error".to_string(),
            runtime_state: Some("online".to_string()),
            ..Default::default()
        };
        assert!(!disk_is_online_for_readiness(&disk));
    }

    #[test]
    fn storage_ready_from_runtime_state_requires_all_sets_meet_quorum() {
        let info = StorageInfo {
            backend: BackendInfo {
                standard_sc_data: vec![1],
                total_sets: vec![2],
                drives_per_set: vec![2],
                ..Default::default()
            },
            disks: vec![
                Disk {
                    endpoint: "127.0.0.1:9000".to_string(),
                    drive_path: "/set0d0".to_string(),
                    pool_index: 0,
                    set_index: 0,
                    disk_index: 0,
                    state: "ok".to_string(),
                    runtime_state: Some("online".to_string()),
                    ..Default::default()
                },
                Disk {
                    endpoint: "127.0.0.1:9001".to_string(),
                    drive_path: "/set0d1".to_string(),
                    pool_index: 0,
                    set_index: 0,
                    disk_index: 1,
                    state: "ok".to_string(),
                    runtime_state: Some("online".to_string()),
                    ..Default::default()
                },
                Disk {
                    endpoint: "127.0.0.1:9002".to_string(),
                    drive_path: "/set1d0".to_string(),
                    pool_index: 0,
                    set_index: 1,
                    disk_index: 0,
                    state: "offline".to_string(),
                    runtime_state: Some("offline".to_string()),
                    ..Default::default()
                },
            ],
        };

        assert!(
            !storage_ready_from_runtime_state(&info),
            "if any set fails write quorum, readiness must be false"
        );
    }

    #[test]
    fn aggregate_lock_quorum_status_requires_each_set_to_meet_quorum() {
        let endpoints = vec![
            Endpoint {
                url: url::Url::parse("http://node1:9000/data1").unwrap(),
                is_local: false,
                pool_idx: 0,
                set_idx: 0,
                disk_idx: 0,
            },
            Endpoint {
                url: url::Url::parse("http://node2:9000/data2").unwrap(),
                is_local: false,
                pool_idx: 0,
                set_idx: 0,
                disk_idx: 1,
            },
            Endpoint {
                url: url::Url::parse("http://node3:9000/data3").unwrap(),
                is_local: false,
                pool_idx: 0,
                set_idx: 1,
                disk_idx: 0,
            },
            Endpoint {
                url: url::Url::parse("http://node4:9000/data4").unwrap(),
                is_local: false,
                pool_idx: 0,
                set_idx: 1,
                disk_idx: 1,
            },
        ];
        let pools = EndpointServerPools::from(vec![PoolEndpoints {
            legacy: false,
            set_count: 2,
            drives_per_set: 2,
            endpoints: Endpoints::from(endpoints),
            cmd_line: String::new(),
            platform: String::new(),
        }]);

        let status = aggregate_lock_quorum_status(
            &pools,
            &["node1:9000", "node2:9000", "node3:9000", "node4:9000"]
                .into_iter()
                .map(str::to_string)
                .collect::<HashSet<_>>(),
        );

        assert!(status.ready);
        assert_eq!(status.connected_clients, 4);
        assert_eq!(status.total_clients, 4);
        assert_eq!(status.required_quorum, 4);
    }

    #[test]
    fn aggregate_lock_quorum_status_fails_when_any_set_loses_quorum() {
        let endpoints = vec![
            Endpoint {
                url: url::Url::parse("http://node1:9000/data1").unwrap(),
                is_local: false,
                pool_idx: 0,
                set_idx: 0,
                disk_idx: 0,
            },
            Endpoint {
                url: url::Url::parse("http://node2:9000/data2").unwrap(),
                is_local: false,
                pool_idx: 0,
                set_idx: 0,
                disk_idx: 1,
            },
            Endpoint {
                url: url::Url::parse("http://node3:9000/data3").unwrap(),
                is_local: false,
                pool_idx: 0,
                set_idx: 1,
                disk_idx: 0,
            },
            Endpoint {
                url: url::Url::parse("http://node4:9000/data4").unwrap(),
                is_local: false,
                pool_idx: 0,
                set_idx: 1,
                disk_idx: 1,
            },
        ];
        let pools = EndpointServerPools::from(vec![PoolEndpoints {
            legacy: false,
            set_count: 2,
            drives_per_set: 2,
            endpoints: Endpoints::from(endpoints),
            cmd_line: String::new(),
            platform: String::new(),
        }]);

        let status =
            aggregate_lock_quorum_status(&pools, &["node1:9000"].into_iter().map(str::to_string).collect::<HashSet<_>>());

        assert!(!status.ready);
    }

    #[test]
    fn degraded_reasons_include_lock_quorum_failures() {
        assert_eq!(
            base_degraded_reasons(true, true, false),
            vec![ReadinessDegradedReason::LockQuorumUnavailable]
        );
        assert_eq!(
            base_degraded_reasons(false, true, false),
            vec![ReadinessDegradedReason::StorageAndLockUnavailable]
        );
        assert_eq!(
            base_degraded_reasons(true, false, false),
            vec![ReadinessDegradedReason::IamAndLockUnavailable]
        );
        assert_eq!(
            base_degraded_reasons(false, false, false),
            vec![ReadinessDegradedReason::StorageIamAndLockUnavailable]
        );
    }

    #[test]
    fn degraded_reasons_append_peer_health_gate_failures() {
        let readiness = DependencyReadiness {
            storage_ready: true,
            iam_ready: true,
            lock_quorum_ready: true,
            peer_health_ready: false,
        };

        assert_eq!(degraded_reasons(readiness), vec![ReadinessDegradedReason::PeerHealthUnavailable]);
    }

    #[tokio::test]
    #[serial]
    async fn lock_quorum_status_cache_roundtrip() {
        async_with_vars([(rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS, Some("60000"))], async {
            let cache = lock_quorum_status_cache();
            {
                let mut guard = cache.lock().await;
                *guard = None;
            }

            update_lock_quorum_status_cache(LockQuorumStatus {
                ready: true,
                connected_clients: 2,
                total_clients: 3,
                required_quorum: 2,
            })
            .await;

            let cached = load_cached_lock_quorum_status().await;
            assert_eq!(
                cached,
                Some(LockQuorumStatus {
                    ready: true,
                    connected_clients: 2,
                    total_clients: 3,
                    required_quorum: 2,
                })
            );
        })
        .await;
    }
}
