feat(observability): Add grafana dashboard, observability changes (#1770 )

Co-authored-by: loverustfs <[email protected]>
build(deps): bump the dependencies group with 2+ updates (#1769 )
2026-02-11 15:55:08 +08:00 · 2026-02-11 09:39:47 +08:00 · 2026-02-11 08:24:55 +08:00 · 2026-02-10 21:37:24 +08:00 · 2026-02-10 12:07:08 +08:00 · 2026-02-10 11:23:21 +08:00
265 changed files with 19263 additions and 7475 deletions
@@ -69,6 +69,9 @@ services:
      - TZ=Asia/Shanghai
    networks:
      - rustfs-network
+    volumes:
+      - ../../.docker/observability/grafana/provisioning:/etc/grafana/provisioning:ro
+      - ../../.docker/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro

  node1:
    build:
@@ -79,7 +82,7 @@ services:
      - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
      - RUSTFS_ADDRESS=:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4317
+      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
      - RUSTFS_OBS_LOGGER_LEVEL=debug
    platform: linux/amd64
    ports:
@@ -96,7 +99,7 @@ services:
      - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
      - RUSTFS_ADDRESS=:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4317
+      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
      - RUSTFS_OBS_LOGGER_LEVEL=debug
    platform: linux/amd64
    ports:
@@ -113,7 +116,7 @@ services:
      - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
      - RUSTFS_ADDRESS=:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4317
+      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
      - RUSTFS_OBS_LOGGER_LEVEL=debug
    platform: linux/amd64
    ports:
@@ -130,7 +133,7 @@ services:
      - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
      - RUSTFS_ADDRESS=:9000
      - RUSTFS_CONSOLE_ENABLE=true
-      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4317
+      - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
      - RUSTFS_OBS_LOGGER_LEVEL=debug
    platform: linux/amd64
    ports:
@@ -56,7 +56,7 @@ docker compose -f docker-compose.yml -f docker-compose.override.yml up -d
 - docker compose v2.0.0 or before

 ```bash
-docke-compose -f docker-compose.yml -f docker-compose.override.yml up -d
+docker-compose -f docker-compose.yml -f docker-compose.override.yml up -d
 ```

 To access the Grafana dashboard, navigate to `http://localhost:3000` in your browser. The default username and password
@@ -69,7 +69,6 @@ exporters:
      queue_size: 5000  # Queue size
  prometheus: # Prometheus exporter for metrics data
    endpoint: "0.0.0.0:8889"  # Prometheus scraping endpoint
-    namespace: "metrics"  # indicator prefix
    send_timestamps: true  # Send timestamp
    metric_expiration: 5m  # Metric expiration time
    resource_to_telemetry_conversion:
@@ -67,7 +67,7 @@ To send telemetry data from your application, configure your OpenTelemetry SDK t
 For example, in a Rust application using the `rustfs-obs` library:

 ```bash
-export RUSTFS_OBS_ENDPOINT=http://localhost:4317
+export RUSTFS_OBS_ENDPOINT=http://localhost:4318
 export RUSTFS_OBS_SERVICE_NAME=yourservice
 export RUSTFS_OBS_SERVICE_VERSION=1.0.0
 export RUSTFS_OBS_ENVIRONMENT=development
@@ -0,0 +1 @@
+../AGENTS.md
@@ -266,6 +266,17 @@ jobs:
    if: needs.build-check.outputs.should_build == 'true'
    runs-on: ubicloud-standard-2
    timeout-minutes: 60
+    strategy:
+          fail-fast: false
+          matrix:
+            include:
+              - variant: musl
+                file: Dockerfile
+                suffix: ""
+              - variant: glibc
+                file: Dockerfile.glibc
+                suffix: "-glibc"
+
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
@@ -296,6 +307,7 @@ jobs:
          VERSION="${{ needs.build-check.outputs.version }}"
          SHORT_SHA="${{ needs.build-check.outputs.short_sha }}"
          CREATE_LATEST="${{ needs.build-check.outputs.create_latest }}"
+          VARIANT_SUFFIX="${{ matrix.suffix }}"

          # Convert version format for Dockerfile compatibility
          case "$VERSION" in
@@ -326,14 +338,14 @@ jobs:

          # Generate tags based on build type
          # Only support release and prerelease builds (no development builds)
-          TAGS="${{ env.REGISTRY_DOCKERHUB }}:${VERSION}"
+          TAGS="${{ env.REGISTRY_DOCKERHUB }}:${VERSION}${VARIANT_SUFFIX}"

          # Add channel tags for prereleases and latest for stable
          if [[ "$CREATE_LATEST" == "true" ]]; then
            # TODO: Temporary change - the current alpha version will also create the latest tag
            # After the version is stabilized, the logic here remains unchanged, but the upstream CREATE_LATEST setting needs to be restored.
            # Stable release (and temporary alpha versions)
-            TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest"
+            TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest${VARIANT_SUFFIX}"
          elif [[ "$BUILD_TYPE" == "prerelease" ]]; then
            # Prerelease channel tags (alpha, beta, rc)
            if [[ "$VERSION" == *"alpha"* ]]; then
@@ -345,7 +357,7 @@ jobs:
            fi

            if [[ -n "$CHANNEL" ]]; then
-              TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:${CHANNEL}"
+              TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:${CHANNEL}${VARIANT_SUFFIX}"
            fi
          fi

@@ -372,15 +384,15 @@ jobs:
        uses: docker/build-push-action@v6
        with:
          context: .
-          file: Dockerfile
+          file: ${{ matrix.file }}
          platforms: ${{ env.DOCKER_PLATFORMS }}
          push: ${{ needs.build-check.outputs.should_push == 'true' }}
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: |
-            type=gha,scope=docker-binary
+            type=gha,scope=docker-${{ matrix.variant }}
          cache-to: |
-            type=gha,mode=max,scope=docker-binary
+            type=gha,mode=max,scope=docker-${{ matrix.variant }}
          build-args: |
            BUILDTIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
            VERSION=${{ needs.build-check.outputs.version }}
@@ -39,3 +39,4 @@ artifacts/
 PR_DESCRIPTION.md
 IMPLEMENTATION_PLAN.md
 scripts/s3-tests/selected_tests.txt
+docs
@@ -99,7 +99,17 @@
            "name": "Debug executable target/debug/rustfs",
            "type":  "lldb",
            "request": "launch",
-            "program": "${workspaceFolder}/target/debug/rustfs",
+            "cargo": {
+                "args": [
+                    "run",
+                    "--bin",
+                    "rustfs",
+                    "-j",
+                    "1",
+                    "--profile",
+                    "dev"
+                ]
+            },
            "args": [],
            "cwd": "${workspaceFolder}",
            //"stopAtEntry": false,
@@ -107,7 +117,7 @@
            "env": {
                "RUSTFS_ACCESS_KEY": "rustfsadmin",
                "RUSTFS_SECRET_KEY": "rustfsadmin",
-                "RUSTFS_VOLUMES": "./target/volume/test{1...4}",
+                //"RUSTFS_VOLUMES": "./target/volume/test{1...4}",
                "RUSTFS_ADDRESS": ":9000",
                "RUSTFS_CONSOLE_ENABLE": "true",
                // "RUSTFS_OBS_TRACE_ENDPOINT": "http://127.0.0.1:4318/v1/traces", // jeager otlp http endpoint
@@ -116,11 +126,31 @@
                // "RUSTFS_COMPRESS_ENABLE": "true",
                "RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
                "RUSTFS_OBS_LOG_DIRECTORY": "./target/logs",
+                "RUST_LOG":"rustfs=debug,ecstore=debug,s3s=debug,iam=debug",
            },
            "sourceLanguages": [
                "rust"
            ],
        },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug test_lifecycle_transition_basic",
+            "cargo": {
+                "args": [
+                    "test",
+                    "-p",
+                    "rustfs-scanner",
+                    "--test",
+                    "lifecycle_integration_test",
+                    "serial_tests::test_lifecycle_transition_basic",
+                    "-j",
+                    "1"
+                ]
+            },
+            "args": [],
+            "cwd": "${workspaceFolder}"
+        },
        {
            "name": "Debug executable target/debug/test",
            "type":  "lldb",
@@ -140,6 +170,81 @@
            "sourceLanguages": [
                "rust"
            ],
-        }
+        },
+        {
+            "name": "Debug executable target/debug/rustfs with sse",
+            "type":  "lldb",
+            "request": "launch",
+            "program": "${workspaceFolder}/target/debug/rustfs",
+            "args": [],
+            "cwd": "${workspaceFolder}",
+            //"stopAtEntry": false,
+            //"preLaunchTask": "cargo build",
+            "env": {
+                "RUSTFS_ACCESS_KEY": "rustfsadmin",
+                "RUSTFS_SECRET_KEY": "rustfsadmin",
+                "RUSTFS_VOLUMES": "./target/volumes/test{1...4}",
+                "RUSTFS_ADDRESS": ":9000",
+                "RUSTFS_CONSOLE_ENABLE": "true",
+                "RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
+                "RUSTFS_OBS_LOG_DIRECTORY": "./target/logs",
+                // "RUSTFS_OBS_TRACE_ENDPOINT": "http://127.0.0.1:4318/v1/traces", // jeager otlp http endpoint
+                // "RUSTFS_OBS_METRIC_ENDPOINT": "http://127.0.0.1:4318/v1/metrics", // default otlp http endpoint
+                // "RUSTFS_OBS_LOG_ENDPOINT": "http://127.0.0.1:4318/v1/logs", // default otlp http endpoint
+                // "RUSTFS_COMPRESS_ENABLE": "true",
+
+                // 1. simple sse test key (no kms system)
+                // "__RUSTFS_SSE_SIMPLE_CMK": "2dfNXGHlsEflGVCxb+5DIdGEl1sIvtwX+QfmYasi5QM=",
+
+                // 2. kms local backend test key
+                "RUSTFS_KMS_ENABLE": "true",
+                "RUSTFS_KMS_BACKEND": "local",
+                "RUSTFS_KMS_KEY_DIR": "./target/kms-key-dir",
+                "RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password
+                "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
+
+                // 3. kms vault backend test key
+                // "RUSTFS_KMS_ENABLE": "true",
+                // "RUSTFS_KMS_BACKEND": "vault",
+                // "RUSTFS_KMS_VAULT_ADDRESS": "http://127.0.0.1:8200",
+                // "RUSTFS_KMS_VAULT_TOKEN": "Dev Token",
+                // "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
+
+            },
+            "sourceLanguages": [
+                "rust"
+            ],
+        },
+        {
+            "name": "E2E test executable target/debug/rustfs",
+            "type":  "lldb",
+            "request": "launch",
+            "program": "${workspaceFolder}/target/debug/rustfs",
+            "args": [],
+            "cwd": "${workspaceFolder}",
+            //"stopAtEntry": false,
+            //"preLaunchTask": "cargo build",
+            "env": {
+                "RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug,iam=debug",
+                "RUST_BACKTRACE": "full",
+                "RUSTFS_ACCESS_KEY": "rustfsadmin",
+                "RUSTFS_SECRET_KEY": "rustfsadmin",
+                "RUSTFS_VOLUMES": "./target/e2e-test/test{1...4}",
+                "RUSTFS_REGION": "us-east-1",
+                "RUSTFS_ADDRESS": ":9000",
+                "RUSTFS_CONSOLE_ENABLE": "true",
+                "RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
+                "RUSTFS_OBS_LOG_DIRECTORY": "./target/logs",
+
+                "RUSTFS_KMS_ENABLE": "true",
+                "RUSTFS_KMS_BACKEND": "local",
+                "RUSTFS_KMS_KEY_DIR": "./target/e2e-key-dir",
+                "RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password
+                "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
+            },
+            "sourceLanguages": [
+                "rust"
+            ],
+        },
    ]
 }
@@ -17,6 +17,7 @@ members = [
    "rustfs", # Core file system implementation
    "crates/appauth", # Application authentication and authorization
    "crates/audit", # Audit target management system with multi-target fan-out
+    "crates/checksums", # client checksums
    "crates/common", # Shared utilities and data structures
    "crates/config", # Configuration management
    "crates/credentials", # Credential management system
@@ -24,26 +25,28 @@ members = [
    "crates/ecstore", # Erasure coding storage implementation
    "crates/e2e_test", # End-to-end test suite
    "crates/filemeta", # File metadata management
+    "crates/heal", # Erasure set and object healing
    "crates/iam", # Identity and Access Management
+    "crates/kms", # Key Management Service
    "crates/lock", # Distributed locking implementation
    "crates/madmin", # Management dashboard and admin API interface
+    "crates/mcp", # MCP server for S3 operations
+    "crates/metrics", # Metrics collection and reporting
    "crates/notify", # Notification system for events
    "crates/obs", # Observability utilities
    "crates/policy", # Policy management
+    "crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
    "crates/protos", # Protocol buffer definitions
    "crates/rio", # Rust I/O utilities and abstractions
-    "crates/targets", # Target-specific configurations and utilities
    "crates/s3select-api", # S3 Select API interface
    "crates/s3select-query", # S3 Select query engine
    "crates/scanner", # Scanner for data integrity checks and health monitoring
    "crates/signer", # client signer
-    "crates/checksums", # client checksums
+    "crates/targets", # Target-specific configurations and utilities
+    "crates/trusted-proxies", # Trusted proxies management
    "crates/utils", # Utility functions and helpers
    "crates/workers", # Worker thread pools and task scheduling
    "crates/zip", # ZIP file handling and compression
-    "crates/heal", # Erasure set and object healing
-    "crates/mcp", # MCP server for S3 operations
-    "crates/kms", # Key Management Service
 ]
 resolver = "2"

@@ -51,12 +54,13 @@ resolver = "2"
 edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/rustfs/rustfs"
-rust-version = "1.90"
+rust-version = "1.93.0"
 version = "0.0.5"
 homepage = "https://rustfs.com"
 description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
 keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
 categories = ["web-programming", "development-tools", "filesystem", "network-programming"]
+authors = ["RustFS Team"]

 [workspace.lints.rust]
 unsafe_code = "deny"
@@ -82,6 +86,7 @@ rustfs-kms = { path = "crates/kms", version = "0.0.5" }
 rustfs-lock = { path = "crates/lock", version = "0.0.5" }
 rustfs-madmin = { path = "crates/madmin", version = "0.0.5" }
 rustfs-mcp = { path = "crates/mcp", version = "0.0.5" }
+rustfs-metrics = { path = "crates/metrics", version = "0.0.5" }
 rustfs-notify = { path = "crates/notify", version = "0.0.5" }
 rustfs-obs = { path = "crates/obs", version = "0.0.5" }
 rustfs-policy = { path = "crates/policy", version = "0.0.5" }
@@ -91,14 +96,16 @@ rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.5" }
 rustfs-s3select-query = { path = "crates/s3select-query", version = "0.0.5" }
 rustfs-scanner = { path = "crates/scanner", version = "0.0.5" }
 rustfs-signer = { path = "crates/signer", version = "0.0.5" }
+rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "0.0.5" }
 rustfs-targets = { path = "crates/targets", version = "0.0.5" }
 rustfs-utils = { path = "crates/utils", version = "0.0.5" }
 rustfs-workers = { path = "crates/workers", version = "0.0.5" }
 rustfs-zip = { path = "./crates/zip", version = "0.0.5" }
+rustfs-protocols = { path = "crates/protocols", version = "0.0.5" }

 # Async Runtime and Networking
 async-channel = "2.5.0"
-async-compression = { version = "0.4.37" }
+async-compression = { version = "0.4.39" }
 async-recursion = "1.1.1"
 async-trait = "0.1.89"
 axum = "0.8.8"
@@ -109,11 +116,11 @@ futures-util = "0.3.31"
 pollster = "0.4.0"
 hyper = { version = "1.8.1", features = ["http2", "http1", "server"] }
 hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
-hyper-util = { version = "0.1.19", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
+hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
 http = "1.4.0"
 http-body = "1.0.1"
 http-body-util = "0.1.3"
-reqwest = { version = "0.13.1", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
+reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
 socket2 = { version = "0.6.2", features = ["all"] }
 tokio = { version = "1.49.0", features = ["fs", "rt-multi-thread"] }
 tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
@@ -127,69 +134,70 @@ tower = { version = "0.5.3", features = ["timeout"] }
 tower-http = { version = "0.6.8", features = ["cors"] }

 # Serialization and Data Formats
-bytes = { version = "1.11.0", features = ["serde"] }
+bytes = { version = "1.11.1", features = ["serde"] }
 bytesize = "2.3.1"
 byteorder = "1.5.0"
 flatbuffers = "25.12.19"
 form_urlencoded = "1.2.2"
 prost = "0.14.3"
 quick-xml = "0.39.0"
-rmcp = { version = "0.14.0" }
+rmcp = { version = "0.15.0" }
 rmp = { version = "0.8.15" }
 rmp-serde = { version = "1.3.1" }
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = { version = "1.0.149", features = ["raw_value"] }
 serde_urlencoded = "0.7.1"
-schemars = "1.2.0"
+schemars = "1.2.1"

 # Cryptography and Security
-aes-gcm = { version = "0.11.0-rc.2", features = ["rand_core"] }
-argon2 = { version = "0.6.0-rc.6" }
+aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] }
+argon2 = { version = "0.6.0-rc.7" }
 blake3 = { version = "1.8.3", features = ["rayon", "mmap"] }
-chacha20poly1305 = { version = "0.11.0-rc.2" }
+chacha20poly1305 = { version = "0.11.0-rc.3" }
 crc-fast = "1.9.0"
-hmac = { version = "0.13.0-rc.3" }
+hmac = { version = "0.13.0-rc.5" }
 jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
-pbkdf2 = "0.13.0-rc.8"
-rsa = { version = "0.10.0-rc.12" }
+pbkdf2 = "0.13.0-rc.9"
+rsa = { version = "0.10.0-rc.15" }
 rustls = { version = "0.23.36", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
 rustls-pemfile = "2.2.0"
 rustls-pki-types = "1.14.0"
-sha1 = "0.11.0-rc.3"
-sha2 = "0.11.0-rc.3"
+sha1 = "0.11.0-rc.5"
+sha2 = "0.11.0-rc.5"
 subtle = "2.6"
 zeroize = { version = "1.8.2", features = ["derive"] }

 # Time and Date
 chrono = { version = "0.4.43", features = ["serde"] }
 humantime = "2.3.0"
-jiff = { version = "0.2.18", features = ["serde"] }
-time = { version = "0.3.46", features = ["std", "parsing", "formatting", "macros", "serde"] }
+jiff = { version = "0.2.19", features = ["serde"] }
+time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }

 # Utilities and Tools
-anyhow = "1.0.100"
-arc-swap = "1.8.0"
+anyhow = "1.0.101"
+arc-swap = "1.8.1"
 astral-tokio-tar = "0.5.6"
 atoi = "2.0.0"
 atomic_enum = "0.3.0"
-aws-config = { version = "1.8.12" }
+aws-config = { version = "1.8.13" }
 aws-credential-types = { version = "1.2.11" }
-aws-sdk-s3 = { version = "1.121.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
-aws-smithy-types = { version = "1.4.1" }
+aws-sdk-s3 = { version = "1.122.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
+aws-smithy-types = { version = "1.4.4" }
+backtrace = "0.3.76"
 base64 = "0.22.1"
 base64-simd = "0.8.0"
 brotli = "8.0.2"
 cfg-if = "1.0.4"
-clap = { version = "4.5.56", features = ["derive", "env"] }
-const-str = { version = "1.0.0", features = ["std", "proc"] }
-convert_case = "0.10.0"
+clap = { version = "4.5.57", features = ["derive", "env"] }
+const-str = { version = "1.1.0", features = ["std", "proc"] }
+convert_case = "0.11.0"
 criterion = { version = "0.8", features = ["html_reports"] }
 crossbeam-queue = "0.3.12"
 datafusion = "52.1.0"
 derive_builder = "0.20.2"
 enumset = "1.1.10"
 faster-hex = "0.10.0"
-flate2 = "1.1.8"
+flate2 = "1.1.9"
 flexi_logger = { version = "0.31.8", features = ["trc", "dont_minimize_extra_stacks", "compress", "kv", "json"] }
 glob = "0.3.3"
 google-cloud-storage = "1.7.0"
@@ -199,12 +207,12 @@ hex-simd = "0.8.0"
 highway = { version = "1.3.0" }
 ipnetwork = { version = "0.21.1", features = ["serde"] }
 lazy_static = "1.5.0"
-libc = "0.2.180"
+libc = "0.2.181"
 libsystemd = "0.7.2"
-local-ip-address = "0.6.9"
+local-ip-address = "0.6.10"
 lz4 = "1.28.1"
 matchit = "0.9.1"
-md-5 = "0.11.0-rc.3"
+md-5 = "0.11.0-rc.5"
 md5 = "0.8.0"
 mime_guess = "2.0.5"
 moka = { version = "0.12.13", features = ["future"] }
@@ -218,15 +226,15 @@ path-absolutize = "3.1.1"
 path-clean = "1.0.1"
 pin-project-lite = "0.2.16"
 pretty_assertions = "1.4.1"
-rand = { version = "0.10.0-rc.6", features = ["serde"] }
+rand = { version = "0.10.0", features = ["serde"] }
 rayon = "1.11.0"
 reed-solomon-simd = { version = "3.1.0" }
-regex = { version = "1.12.2" }
+regex = { version = "1.12.3" }
 rumqttc = { version = "0.25.1" }
 rustix = { version = "1.1.3", features = ["fs"] }
 rust-embed = { version = "8.11.0" }
 rustc-hash = { version = "2.1.1" }
-s3s = { version = "0.13.0-alpha.2", features = ["minio"], git = "https://github.com/s3s-project/s3s.git", rev = "3cdb3fe22fe8a1b7fc3f71ead4beacac2683ba7f" }
+s3s = { version = "0.13.0-alpha.3", features = ["minio"], git = "https://github.com/s3s-project/s3s.git", rev = "7b8e22b49bc76839e6085a4ddfa553323f1b8049" }
 serial_test = "3.3.1"
 shadow-rs = { version = "1.7.0", default-features = false }
 siphasher = "1.0.2"
@@ -234,11 +242,11 @@ smallvec = { version = "1.15.1", features = ["serde"] }
 smartstring = "1.0.1"
 snafu = "0.8.9"
 snap = "1.1.1"
-starshard = { version = "0.6.0", features = ["rayon", "async", "serde"] }
+starshard = { version = "1.1.0", features = ["rayon", "async", "serde"] }
 strum = { version = "0.27.2", features = ["derive"] }
-sysinfo = "0.38.0"
+sysinfo = "0.38.1"
 temp-env = "0.3.6"
-tempfile = "3.24.0"
+tempfile = "3.25.0"
 test-case = "3.3.1"
 thiserror = "2.0.18"
 tracing = { version = "0.1.44" }
@@ -255,7 +263,7 @@ walkdir = "2.5.0"
 wildmatch = { version = "2.6.1", features = ["serde"] }
 windows = { version = "0.62.2" }
 xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
-zip = "7.2.0"
+zip = "7.4.0"
 zstd = "0.13.3"

 # Observability and Metrics
@@ -268,11 +276,8 @@ opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_
 opentelemetry-stdout = { version = "0.31.0" }

 # FTP and SFTP
-libunftp = { version = "0.21.0" }
-russh = { version = "0.57.0", features = ["aws-lc-rs", "rsa"], default-features = false }
-russh-sftp = { version = "2.1.1" }
-ssh-key = { version = "0.7.0-rc.6", features = ["std", "rsa", "ed25519"] }
-suppaftp = { version = "7.1.0", features = ["tokio", "tokio-rustls", "rustls"] }
+libunftp = { version = "0.22.0", features = ["experimental"] }
+suppaftp = { version = "8.0.1", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
 rcgen = "0.14.7"

 # Performance Analysis and Memory Profiling
@@ -282,7 +287,7 @@ tikv-jemallocator = { version = "0.6", features = ["profiling", "stats", "unpref
 # Used to control and obtain statistics for jemalloc at runtime
 tikv-jemalloc-ctl = { version = "0.6", features = ["use_std", "stats", "profiling"] }
 # Used to generate pprof-compatible memory profiling data and support symbolization and flame graphs
-jemalloc_pprof = { version = "0.8.1", features = ["symbolize", "flamegraph"] }
+jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] }
 # Used to generate CPU performance analysis data and flame diagrams
 pprof = { version = "0.15.0", features = ["flamegraph", "protobuf-codec"] }

@@ -0,0 +1,101 @@
+FROM ubuntu:24.04 AS build
+
+ARG TARGETARCH
+ARG RELEASE=latest
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /build
+
+RUN set -eux; \
+    case "$TARGETARCH" in \
+      amd64)  ARCH_SUBSTR="x86_64-gnu"  ;; \
+      arm64)  ARCH_SUBSTR="aarch64-gnu" ;; \
+      *) echo "Unsupported TARGETARCH=$TARGETARCH" >&2; exit 1 ;; \
+    esac; \
+    \
+    if [ "$RELEASE" = "latest" ]; then \
+      TAG="$(curl -fsSL https://api.github.com/repos/rustfs/rustfs/releases \
+              | grep -o '"tag_name": "[^"]*"' | cut -d'"' -f4 | head -n 1)"; \
+    else \
+      TAG="$RELEASE"; \
+    fi; \
+    \
+    URL="$(curl -fsSL "https://api.github.com/repos/rustfs/rustfs/releases/tags/$TAG" \
+           | grep -o "\"browser_download_url\": \"[^\"]*${ARCH_SUBSTR}[^\"]*\\.zip\"" \
+           | cut -d'"' -f4 | head -n 1)"; \
+    \
+    if [ -z "$URL" ]; then echo "Failed to locate release asset for $ARCH_SUBSTR at tag $TAG" >&2; exit 1; fi; \
+    \
+    curl -fL "$URL" -o rustfs.zip; \
+    unzip -q rustfs.zip -d /build; \
+    \
+    if [ ! -x /build/rustfs ]; then \
+      BIN_PATH="$(unzip -Z -1 rustfs.zip | grep -E '(^|/)rustfs$' | head -n 1 || true)"; \
+      if [ -n "$BIN_PATH" ]; then \
+        mkdir -p /build/.tmp && unzip -q rustfs.zip "$BIN_PATH" -d /build/.tmp && \
+        mv "/build/.tmp/$BIN_PATH" /build/rustfs; \
+      fi; \
+    fi; \
+    [ -x /build/rustfs ] || { echo "rustfs binary not found in asset" >&2; exit 1; }; \
+    chmod +x /build/rustfs; \
+    rm -rf rustfs.zip /build/.tmp || true
+
+FROM ubuntu:24.04
+
+ARG RELEASE=latest
+ARG BUILD_DATE
+ARG VCS_REF
+
+LABEL name="RustFS" \
+      vendor="RustFS Team" \
+      maintainer="RustFS Team <[email protected]>" \
+      version="v${RELEASE#v}" \
+      release="${RELEASE}" \
+      build-date="${BUILD_DATE}" \
+      vcs-ref="${VCS_REF}" \
+      summary="High-performance distributed object storage system (glibc)" \
+      url="https://rustfs.com" \
+      license="Apache-2.0"
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=build /build/rustfs /usr/bin/rustfs
+COPY entrypoint.sh /entrypoint.sh
+
+RUN chmod +x /usr/bin/rustfs /entrypoint.sh
+
+RUN groupadd -g 10001 rustfs && \
+    useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
+    mkdir -p /data /logs && \
+    chown -R rustfs:rustfs /data /logs && \
+    chmod 0750 /data /logs
+
+ENV RUSTFS_ADDRESS=":9000" \
+    RUSTFS_CONSOLE_ADDRESS=":9001" \
+    RUSTFS_ACCESS_KEY="rustfsadmin" \
+    RUSTFS_SECRET_KEY="rustfsadmin" \
+    RUSTFS_CONSOLE_ENABLE="true" \
+    RUSTFS_CORS_ALLOWED_ORIGINS="*" \
+    RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS="*" \
+    RUSTFS_VOLUMES="/data" \
+    RUSTFS_OBS_LOGGER_LEVEL=warn \
+    RUSTFS_OBS_LOG_DIRECTORY=/logs \
+    RUSTFS_OBS_ENVIRONMENT=production
+    
+EXPOSE 9000 9001
+
+VOLUME ["/data"]
+
+USER rustfs
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+CMD ["rustfs"]
@@ -37,8 +37,11 @@ datas = "datas"
 bre = "bre"
 abd = "abd"
 mak = "mak"
+gae = "gae"
+GAE = "GAE"
 # s3-tests original test names (cannot be changed)
 nonexisted = "nonexisted"
+consts = "consts"

 [files]
 extend-exclude = []
@@ -62,8 +62,8 @@ pub const DEFAULT_CONSOLE_ENABLE: bool = true;
 /// Default OBS configuration endpoint
 /// Environment variable: DEFAULT_OBS_ENDPOINT
 /// Command line argument: --obs-endpoint
-/// Example: DEFAULT_OBS_ENDPOINT="http://localost:4317"
-/// Example: --obs-endpoint http://localost:4317
+/// Example: DEFAULT_OBS_ENDPOINT="http://localost:4318"
+/// Example: --obs-endpoint http://localost:4318
 pub const DEFAULT_OBS_ENDPOINT: &str = "";

 /// Default TLS key for rustfs
@@ -98,6 +98,12 @@ pub const RUSTFS_HTTP_PREFIX: &str = "http://";
 /// Default value: https://
 pub const RUSTFS_HTTPS_PREFIX: &str = "https://";

+/// Environment variable for rustfs address
+/// This is the environment variable for rustfs address.
+/// It is used to bind the server to a specific address.
+/// Example: RUSTFS_ADDRESS=":9000"
+pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
+
 /// Default port for rustfs
 /// This is the default port for rustfs.
 /// This is used to bind the server to a specific port.
@@ -21,6 +21,7 @@ pub(crate) mod heal;
 pub(crate) mod object;
 pub(crate) mod profiler;
 pub(crate) mod protocols;
+pub(crate) mod proxy;
 pub(crate) mod quota;
 pub(crate) mod runtime;
 pub(crate) mod scanner;
@@ -14,27 +14,34 @@

 //! Protocol server configuration constants

-/// Default FTPS server bind address
-pub const DEFAULT_FTPS_ADDRESS: &str = "0.0.0.0:8021";
+/// Default FTP server bind address (non-encrypted)
+pub const DEFAULT_FTP_ADDRESS: &str = "0.0.0.0:8021";

-/// Default SFTP server bind address
-pub const DEFAULT_SFTP_ADDRESS: &str = "0.0.0.0:8022";
+/// Default FTPS server bind address (FTP over TLS)
+pub const DEFAULT_FTPS_ADDRESS: &str = "0.0.0.0:8022";
+
+/// Default FTP passive ports range (optional)
+pub const DEFAULT_FTP_PASSIVE_PORTS: Option<&str> = None;

 /// Default FTPS passive ports range (optional)
 pub const DEFAULT_FTPS_PASSIVE_PORTS: Option<&str> = None;

+/// Default FTP external IP (auto-detected)
+pub const DEFAULT_FTP_EXTERNAL_IP: Option<&str> = None;
+
 /// Default FTPS external IP (auto-detected)
 pub const DEFAULT_FTPS_EXTERNAL_IP: Option<&str> = None;

 /// Environment variable names
+pub const ENV_FTP_ENABLE: &str = "RUSTFS_FTP_ENABLE";
+pub const ENV_FTP_ADDRESS: &str = "RUSTFS_FTP_ADDRESS";
+pub const ENV_FTP_PASSIVE_PORTS: &str = "RUSTFS_FTP_PASSIVE_PORTS";
+pub const ENV_FTP_EXTERNAL_IP: &str = "RUSTFS_FTP_EXTERNAL_IP";
+
 pub const ENV_FTPS_ENABLE: &str = "RUSTFS_FTPS_ENABLE";
 pub const ENV_FTPS_ADDRESS: &str = "RUSTFS_FTPS_ADDRESS";
-pub const ENV_FTPS_CERTS_FILE: &str = "RUSTFS_FTPS_CERTS_FILE";
-pub const ENV_FTPS_KEY_FILE: &str = "RUSTFS_FTPS_KEY_FILE";
+pub const ENV_FTPS_TLS_ENABLED: &str = "RUSTFS_FTPS_TLS_ENABLED";
+pub const ENV_FTPS_CERTS_DIR: &str = "RUSTFS_FTPS_CERTS_DIR";
+pub const ENV_FTPS_CA_FILE: &str = "RUSTFS_FTPS_CA_FILE";
 pub const ENV_FTPS_PASSIVE_PORTS: &str = "RUSTFS_FTPS_PASSIVE_PORTS";
 pub const ENV_FTPS_EXTERNAL_IP: &str = "RUSTFS_FTPS_EXTERNAL_IP";
-
-pub const ENV_SFTP_ENABLE: &str = "RUSTFS_SFTP_ENABLE";
-pub const ENV_SFTP_ADDRESS: &str = "RUSTFS_SFTP_ADDRESS";
-pub const ENV_SFTP_HOST_KEY: &str = "RUSTFS_SFTP_HOST_KEY";
-pub const ENV_SFTP_AUTHORIZED_KEYS: &str = "RUSTFS_SFTP_AUTHORIZED_KEYS";
@@ -0,0 +1,125 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::DEFAULT_LOG_LEVEL;
+
+// ==================== Base Proxy Configuration ====================
+/// Environment variable to enable the trusted proxy middleware.
+pub const ENV_TRUSTED_PROXY_ENABLED: &str = "RUSTFS_TRUSTED_PROXY_ENABLED";
+/// Trusted proxy middleware is enabled by default.
+pub const DEFAULT_TRUSTED_PROXY_ENABLED: bool = true;
+
+/// Environment variable for the proxy validation mode.
+pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDATION_MODE";
+/// Default validation mode is "hop_by_hop".
+pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
+
+/// Environment variable to enable RFC 7239 "Forwarded" header support.
+pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
+/// RFC 7239 support is enabled by default.
+pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
+
+/// Environment variable for the maximum allowed proxy hops.
+pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";
+/// Default maximum hops is 10.
+pub const DEFAULT_TRUSTED_PROXY_MAX_HOPS: usize = 10;
+
+/// Environment variable to enable proxy chain continuity checks.
+pub const ENV_TRUSTED_PROXY_CHAIN_CONTINUITY_CHECK: &str = "RUSTFS_TRUSTED_PROXY_CHAIN_CONTINUITY_CHECK";
+/// Continuity checks are enabled by default.
+pub const DEFAULT_TRUSTED_PROXY_CHAIN_CONTINUITY_CHECK: bool = true;
+
+/// Environment variable to enable logging of failed proxy validations.
+pub const ENV_TRUSTED_PROXY_LOG_FAILED_VALIDATIONS: &str = "RUSTFS_TRUSTED_PROXY_LOG_FAILED_VALIDATIONS";
+/// Logging of failed validations is enabled by default.
+pub const DEFAULT_TRUSTED_PROXY_LOG_FAILED_VALIDATIONS: bool = true;
+
+// ==================== Trusted Proxy Networks ====================
+/// Environment variable for the list of trusted proxy networks (comma-separated IP/CIDR).
+pub const ENV_TRUSTED_PROXY_PROXIES: &str = "RUSTFS_TRUSTED_PROXY_NETWORKS";
+/// Default trusted networks include localhost and common private ranges.
+pub const DEFAULT_TRUSTED_PROXY_PROXIES: &str = "127.0.0.1,::1,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,fd00::/8";
+
+/// Environment variable for additional trusted proxy networks (production specific).
+pub const ENV_TRUSTED_PROXY_EXTRA_PROXIES: &str = "RUSTFS_TRUSTED_PROXY_EXTRA_NETWORKS";
+/// No extra trusted networks by default.
+pub const DEFAULT_TRUSTED_PROXY_EXTRA_PROXIES: &str = "";
+
+/// Environment variable for individual trusted proxy IPs.
+pub const ENV_TRUSTED_PROXY_IPS: &str = "RUSTFS_TRUSTED_PROXY_IPS";
+/// No individual trusted IPs by default.
+pub const DEFAULT_TRUSTED_PROXY_IPS: &str = "";
+
+/// Environment variable for private network ranges used in internal validation.
+pub const ENV_TRUSTED_PROXY_PRIVATE_NETWORKS: &str = "RUSTFS_TRUSTED_PROXY_PRIVATE_NETWORKS";
+/// Default private networks include common RFC 1918 and RFC 4193 ranges.
+pub const DEFAULT_TRUSTED_PROXY_PRIVATE_NETWORKS: &str = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,fd00::/8";
+
+// ==================== Cache Configuration ====================
+/// Environment variable for the proxy validation cache capacity.
+pub const ENV_TRUSTED_PROXY_CACHE_CAPACITY: &str = "RUSTFS_TRUSTED_PROXY_CACHE_CAPACITY";
+/// Default cache capacity is 10,000 entries.
+pub const DEFAULT_TRUSTED_PROXY_CACHE_CAPACITY: usize = 10_000;
+
+/// Environment variable for the cache entry time-to-live (TTL) in seconds.
+pub const ENV_TRUSTED_PROXY_CACHE_TTL_SECONDS: &str = "RUSTFS_TRUSTED_PROXY_CACHE_TTL_SECONDS";
+/// Default cache TTL is 300 seconds (5 minutes).
+pub const DEFAULT_TRUSTED_PROXY_CACHE_TTL_SECONDS: u64 = 300;
+
+/// Environment variable for the cache cleanup interval in seconds.
+pub const ENV_TRUSTED_PROXY_CACHE_CLEANUP_INTERVAL: &str = "RUSTFS_TRUSTED_PROXY_CACHE_CLEANUP_INTERVAL";
+/// Default cleanup interval is 60 seconds.
+pub const DEFAULT_TRUSTED_PROXY_CACHE_CLEANUP_INTERVAL: u64 = 60;
+
+// ==================== Monitoring Configuration ====================
+/// Environment variable to enable Prometheus metrics.
+pub const ENV_TRUSTED_PROXY_METRICS_ENABLED: &str = "RUSTFS_TRUSTED_PROXY_METRICS_ENABLED";
+/// Metrics are enabled by default.
+pub const DEFAULT_TRUSTED_PROXY_METRICS_ENABLED: bool = true;
+
+/// Environment variable for the application log level.
+pub const ENV_TRUSTED_PROXIES_LOG_LEVEL: &str = "RUSTFS_TRUSTED_PROXY_LOG_LEVEL";
+/// Default log level is "info".
+pub const DEFAULT_TRUSTED_PROXIES_LOG_LEVEL: &str = DEFAULT_LOG_LEVEL;
+
+/// Environment variable to enable structured JSON logging.
+pub const ENV_TRUSTED_PROXY_STRUCTURED_LOGGING: &str = "RUSTFS_TRUSTED_PROXY_STRUCTURED_LOGGING";
+/// Structured logging is disabled by default.
+pub const DEFAULT_TRUSTED_PROXY_STRUCTURED_LOGGING: bool = false;
+
+/// Environment variable to enable distributed tracing.
+pub const ENV_TRUSTED_PROXY_TRACING_ENABLED: &str = "RUSTFS_TRUSTED_PROXY_TRACING_ENABLED";
+/// Tracing is enabled by default.
+pub const DEFAULT_TRUSTED_PROXY_TRACING_ENABLED: bool = true;
+
+// ==================== Cloud Integration ====================
+/// Environment variable to enable automatic cloud metadata discovery.
+pub const ENV_TRUSTED_PROXY_CLOUD_METADATA_ENABLED: &str = "RUSTFS_TRUSTED_PROXY_CLOUD_METADATA_ENABLED";
+/// Cloud metadata discovery is disabled by default.
+pub const DEFAULT_TRUSTED_PROXY_CLOUD_METADATA_ENABLED: bool = false;
+
+/// Environment variable for the cloud metadata request timeout in seconds.
+pub const ENV_TRUSTED_PROXY_CLOUD_METADATA_TIMEOUT: &str = "RUSTFS_TRUSTED_PROXY_CLOUD_METADATA_TIMEOUT";
+/// Default cloud metadata timeout is 5 seconds.
+pub const DEFAULT_TRUSTED_PROXY_CLOUD_METADATA_TIMEOUT: u64 = 5;
+
+/// Environment variable to enable Cloudflare IP range integration.
+pub const ENV_TRUSTED_PROXY_CLOUDFLARE_IPS_ENABLED: &str = "RUSTFS_TRUSTED_PROXY_CLOUDFLARE_IPS_ENABLED";
+/// Cloudflare integration is disabled by default.
+pub const DEFAULT_TRUSTED_PROXY_CLOUDFLARE_IPS_ENABLED: bool = false;
+
+/// Environment variable to force a specific cloud provider (overrides auto-detection).
+pub const ENV_TRUSTED_PROXY_CLOUD_PROVIDER_FORCE: &str = "RUSTFS_TRUSTED_PROXY_CLOUD_PROVIDER_FORCE";
+/// No forced provider by default.
+pub const DEFAULT_TRUSTED_PROXY_CLOUD_PROVIDER_FORCE: &str = "";
@@ -33,6 +33,8 @@ pub use constants::profiler::*;
 #[cfg(feature = "constants")]
 pub use constants::protocols::*;
 #[cfg(feature = "constants")]
+pub use constants::proxy::*;
+#[cfg(feature = "constants")]
 pub use constants::quota::*;
 #[cfg(feature = "constants")]
 pub use constants::runtime::*;
@@ -13,7 +13,7 @@
 // limitations under the License.

 use crate::{DEFAULT_SECRET_KEY, ENV_RPC_SECRET, IAM_POLICY_CLAIM_NAME_SA, INHERITED_POLICY_TYPE};
-use rand::{Rng, RngCore};
+use rand::{Rng, RngExt};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use std::collections::HashMap;
@@ -58,7 +58,7 @@ fn encrypt<T: aes_gcm::aead::Aead>(
    use crate::error::Error;
    use aes_gcm::AeadCore;
    use aes_gcm::aead::array::Array;
-    use rand::RngCore;
+    use rand::Rng;

    let mut nonce: Array<u8, <T as AeadCore>::NonceSize> = Array::default();
    rand::rng().fill_bytes(&mut nonce);
@@ -40,11 +40,18 @@ impl ID {
    pub(crate) fn get_key(&self, password: &[u8], salt: &[u8]) -> Result<[u8; 32], crate::Error> {
        let mut key = [0u8; 32];
        match self {
-            ID::Pbkdf2AESGCM => pbkdf2_hmac::<Sha256>(password, salt, 8192, &mut key),
-            _ => {
-                let params = Params::new(64 * 1024, 1, 4, Some(32))?;
-                let argon_2id = Argon2::new(Algorithm::Argon2id, Version::V0x13, params);
-                argon_2id.hash_password_into(password, salt, &mut key)?;
+            ID::Pbkdf2AESGCM => {
+                pbkdf2_hmac::<Sha256>(password, salt, 8192, &mut key);
+            }
+            ID::Argon2idAESGCM | ID::Argon2idChaCHa20Poly1305 => {
+                const ARGON2_MEMORY: u32 = 64 * 1024;
+                const ARGON2_ITERATIONS: u32 = 1;
+                const ARGON2_PARALLELISM: u32 = 4;
+                const ARGON2_OUTPUT_LEN: usize = 32;
+
+                let params = Params::new(ARGON2_MEMORY, ARGON2_ITERATIONS, ARGON2_PARALLELISM, Some(ARGON2_OUTPUT_LEN))?;
+                let argon2 = Argon2::new(Algorithm::Argon2id, Version::V0x13, params);
+                argon2.hash_password_into(password, salt, &mut key)?;
            }
        }

@@ -106,7 +106,7 @@ fn test_encrypt_decrypt_binary_data() -> Result<(), crate::Error> {
 #[test]
 fn test_encrypt_decrypt_unicode_data() -> Result<(), crate::Error> {
    let unicode_strings = [
-        "Hello, 世界! 🌍",
+        "Hello, 世界！🌍",
        "Тест на русском языке",
        "العربية اختبار",
        "🚀🔐💻🌟⭐",
@@ -20,6 +20,12 @@ pub enum Error {
    #[error("invalid encryption algorithm ID: {0}")]
    ErrInvalidAlgID(u8),

+    #[error("invalid input: {0}")]
+    ErrInvalidInput(String),
+
+    #[error("invalid key length")]
+    ErrInvalidKeyLength,
+
    #[cfg(any(test, feature = "crypto"))]
    #[error("{0}")]
    ErrInvalidLength(#[from] sha2::digest::InvalidLength),
@@ -38,4 +44,13 @@ pub enum Error {

    #[error("jwt err: {0}")]
    ErrJwt(#[from] jsonwebtoken::errors::Error),
+
+    #[error("io error: {0}")]
+    ErrIo(#[from] std::io::Error),
+
+    #[error("invalid signature")]
+    ErrInvalidSignature,
+
+    #[error("invalid token")]
+    ErrInvalidToken,
 }
@@ -23,6 +23,10 @@ rust-version.workspace = true
 [lints]
 workspace = true

+[features]
+default = []
+ftps = []
+
 [dependencies]
 rustfs-ecstore.workspace = true
 rustfs-common.workspace = true
@@ -52,7 +56,7 @@ rand = { workspace = true }
 chrono = { workspace = true }
 md5 = { workspace = true }
 sha2 = { workspace = true }
-suppaftp.workspace = true
+suppaftp = { workspace = true, features = ["tokio", "rustls-aws-lc-rs"] }
 rcgen.workspace = true
 anyhow.workspace = true
 rustls.workspace = true
@@ -0,0 +1,241 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::common::RustFSTestClusterEnvironment;
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::SdkError;
+use bytes::Bytes;
+use serial_test::serial;
+use std::sync::Arc;
+use tokio::sync::Barrier;
+use tracing::{info, warn};
+
+const BUCKET: &str = "conditional-put-race-bucket";
+
+async fn cleanup_object(client: &Client, key: &str) {
+    if let Err(e) = client.delete_object().bucket(BUCKET).key(key).send().await {
+        warn!("Failed to delete object '{}' from bucket '{}' during cleanup: {:?}", key, BUCKET, e);
+    }
+}
+
+async fn conditional_put(
+    client: &Client,
+    key: &str,
+    data: &[u8],
+    client_id: usize,
+) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
+    let result = client
+        .put_object()
+        .bucket(BUCKET)
+        .key(key)
+        .body(Bytes::copy_from_slice(data).into())
+        .if_none_match("*")
+        .send()
+        .await;
+
+    match result {
+        Ok(resp) => {
+            info!("  Client {} SUCCEEDED - ETag: {}", client_id, resp.e_tag().unwrap_or("none"));
+            Ok(true)
+        }
+        Err(SdkError::ServiceError(e)) => {
+            let code = e.err().meta().code().unwrap_or("");
+            let message = e.err().meta().message().unwrap_or("");
+            if code == "PreconditionFailed" {
+                warn!("  Client {} got 412 PreconditionFailed", client_id);
+                Ok(false)
+            } else {
+                warn!("  Client {} got ServiceError: code={}, message={}", client_id, code, message);
+                Err(e.into_err().into())
+            }
+        }
+        Err(e) => {
+            warn!("  Client {} got non-ServiceError: {:?}", client_id, e);
+            Err(e.into())
+        }
+    }
+}
+
+async fn run_race_iteration(
+    clients: &[Client],
+    test_key: &str,
+    iteration: usize,
+) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
+    cleanup_object(&clients[0], test_key).await;
+    tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
+
+    let head_result = clients[0].head_object().bucket(BUCKET).key(test_key).send().await;
+
+    if head_result.is_ok() {
+        warn!("Warning: Object still exists after cleanup, skipping iteration {}", iteration);
+        return Ok(0);
+    }
+
+    info!("\n=== Iteration {} ===", iteration);
+    info!("Launching {} concurrent conditional PUTs to different nodes...", clients.len());
+
+    let barrier = Arc::new(Barrier::new(clients.len()));
+    let test_key = test_key.to_string();
+
+    let mut handles = vec![];
+    for (i, client) in clients.iter().enumerate() {
+        let client = client.clone();
+        let barrier = barrier.clone();
+        let key = test_key.clone();
+        let data = format!("data from client {}", i).into_bytes();
+
+        let handle = tokio::spawn(async move {
+            barrier.wait().await;
+            conditional_put(&client, &key, &data, i).await
+        });
+        handles.push(handle);
+    }
+
+    let mut success_count = 0;
+    let mut had_error = false;
+    for handle in handles {
+        match handle.await {
+            Ok(Ok(true)) => success_count += 1,
+            Ok(Ok(false)) => {}
+            Ok(Err(e)) => {
+                had_error = true;
+                info!("  Error: {}", e);
+            }
+            Err(e) => {
+                had_error = true;
+                info!("  Task error: {}", e);
+            }
+        }
+    }
+
+    info!("Result: {} out of {} succeeded", success_count, clients.len());
+
+    if success_count > 1 {
+        info!(">>> RACE CONDITION DETECTED!");
+    } else if success_count == 1 {
+        info!(">>> Correct behavior: exactly 1 writer succeeded.");
+    } else if had_error {
+        return Err("all conditional PUTs failed (e.g. cluster/bucket not ready)".into());
+    } else {
+        info!(">>> Unexpected: no writers succeeded.");
+    }
+
+    Ok(success_count)
+}
+
+#[tokio::test]
+#[serial]
+async fn test_conditional_put_race_cluster() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    crate::common::init_logging();
+    info!("Starting conditional PUT race test with auto cluster");
+
+    let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
+    cluster.start().await?;
+
+    cluster.create_test_bucket(BUCKET).await?;
+
+    let clients = cluster.create_all_clients()?;
+
+    let iterations = 5;
+    let mut races_detected = 0;
+    let mut correct_count = 0;
+    let mut error_count = 0;
+
+    for i in 1..=iterations {
+        let test_key = format!("race-test-{}-{}", std::process::id(), i);
+
+        match run_race_iteration(&clients, &test_key, i).await {
+            Ok(success_count) => {
+                if success_count > 1 {
+                    races_detected += 1;
+                } else if success_count == 1 {
+                    correct_count += 1;
+                }
+            }
+            Err(e) => {
+                error_count += 1;
+                warn!("Iteration {} failed (not a race; e.g. cluster/network): {}", i, e)
+            }
+        }
+
+        cleanup_object(&clients[0], &test_key).await;
+        tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
+    }
+
+    info!("\n====================================");
+    info!("SUMMARY");
+    info!("====================================");
+    info!("Total iterations:   {}", iterations);
+    info!("Correct (1 winner): {}", correct_count);
+    info!("Race conditions:    {}", races_detected);
+    info!("Errors (skipped):   {}", error_count);
+
+    assert_eq!(races_detected, 0, "Race conditions detected: {}/{}", races_detected, iterations);
+    assert_eq!(
+        error_count, 0,
+        "{} iteration(s) failed due to errors (e.g. cluster not ready)",
+        error_count
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    crate::common::init_logging();
+    info!("Starting basic conditional PUT test with auto cluster");
+
+    let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
+    cluster.start().await?;
+
+    cluster.create_test_bucket(BUCKET).await?;
+
+    let client = cluster.create_s3_client(0)?;
+    let test_key = "basic-conditional-put";
+    cleanup_object(&client, test_key).await;
+
+    let result = client
+        .put_object()
+        .bucket(BUCKET)
+        .key(test_key)
+        .body(Bytes::from("first write").into())
+        .if_none_match("*")
+        .send()
+        .await;
+    assert!(result.is_ok(), "First PUT with If-None-Match:* should succeed");
+
+    let result = client
+        .put_object()
+        .bucket(BUCKET)
+        .key(test_key)
+        .body(Bytes::from("second write").into())
+        .if_none_match("*")
+        .send()
+        .await;
+    assert!(result.is_err(), "Second PUT with If-None-Match:* should fail");
+
+    assert!(
+        matches!(result, Err(SdkError::ServiceError(_))),
+        "Expected ServiceError but got different error type"
+    );
+
+    if let Err(SdkError::ServiceError(e)) = result {
+        let code = e.err().meta().code().unwrap_or("");
+        assert_eq!(code, "PreconditionFailed");
+    }
+
+    cleanup_object(&client, test_key).await;
+    Ok(())
+}
@@ -81,6 +81,14 @@ fn build_rustfs_binary() {
    let mut cmd = Command::new("cargo");
    cmd.current_dir(&workspace).args(["build", "--bin", "rustfs"]);

+    // Read features from environment variable for e2e tests
+    if let Ok(features) = std::env::var("RUSTFS_BUILD_FEATURES")
+        && !features.is_empty()
+    {
+        cmd.arg("--features").arg(&features);
+        info!("Building with features: {}", features);
+    }
+
    if !cfg!(debug_assertions) {
        cmd.arg("--release");
    }
@@ -374,3 +382,276 @@ pub async fn awscurl_delete(
 ) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
    execute_awscurl(url, "DELETE", None, access_key, secret_key).await
 }
+
+/// Represents a single RustFS server instance in a test cluster.
+///
+/// Each `ClusterNode` tracks the node's network address, base URL for
+/// S3-compatible requests, on-disk data directory, and the underlying
+/// child process handle when the node is running.
+pub struct ClusterNode {
+    pub address: String,
+    pub url: String,
+    pub data_dir: String,
+    pub process: Option<Child>,
+}
+
+/// Test environment for managing a multi-node RustFS cluster.
+///
+/// `RustFSTestClusterEnvironment` is responsible for starting and stopping
+/// a group of `ClusterNode`s, managing their temporary storage directory,
+/// and providing the shared access and secret keys used by tests to
+/// interact with the cluster.
+pub struct RustFSTestClusterEnvironment {
+    pub nodes: Vec<ClusterNode>,
+    pub temp_dir: String,
+    pub access_key: String,
+    pub secret_key: String,
+}
+
+impl RustFSTestClusterEnvironment {
+    /// Create a new RustFS test cluster environment with the specified number of nodes.
+    ///
+    /// Generates a unique temporary root directory for the cluster, allocates an available TCP port
+    /// for each node, creates an independent data directory for every node, and initializes basic
+    /// cluster node configurations (node processes are not started at this stage).
+    ///
+    /// # Arguments
+    ///
+    /// * `node_count` - The number of nodes to create in the cluster, must be a positive integer
+    ///   (an empty cluster will cause errors in subsequent startup operations).
+    ///
+    /// # Returns
+    ///
+    /// * `Ok(Self)` - A new instance of `RustFSTestClusterEnvironment` with initialized node
+    ///   configurations and temporary directory info on success.
+    /// * `Err(Box<dyn Error + Send + Sync>)` - An error if any step fails, such as temporary
+    ///   directory creation failure or available port lookup failure.
+    pub async fn new(node_count: usize) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
+        if node_count == 0 {
+            return Err("Node count must be greater than zero".into());
+        }
+        let temp_dir = format!("/tmp/rustfs_cluster_test_{}", Uuid::new_v4());
+        fs::create_dir_all(&temp_dir).await?;
+
+        let mut nodes = Vec::with_capacity(node_count);
+        for i in 0..node_count {
+            let port = RustFSTestEnvironment::find_available_port().await?;
+            let address = format!("127.0.0.1:{}", port);
+            let url = format!("http://{}", address);
+            let data_dir = format!("{}/node{}", temp_dir, i);
+            fs::create_dir_all(&data_dir).await?;
+
+            nodes.push(ClusterNode {
+                address,
+                url,
+                data_dir,
+                process: None,
+            });
+        }
+
+        Ok(Self {
+            nodes,
+            temp_dir,
+            access_key: DEFAULT_ACCESS_KEY.to_string(),
+            secret_key: DEFAULT_SECRET_KEY.to_string(),
+        })
+    }
+
+    /// Build the volumes argument string for RustFS binary (internal helper method).
+    ///
+    /// Concatenates the address and data directory of all cluster nodes into a single string
+    /// used as the `RUSTFS_VOLUMES` environment variable for RustFS node processes.
+    fn build_volumes_arg(&self) -> String {
+        self.nodes
+            .iter()
+            .map(|n| format!("http://{}{}", n.address, n.data_dir))
+            .collect::<Vec<_>>()
+            .join(" ")
+    }
+
+    /// Start all node processes in the RustFS cluster and wait for the cluster service to be ready.
+    ///
+    /// Spawns a RustFS binary process for each node with necessary environment variable configurations,
+    /// first waits for each node's TCP port to be reachable, then verifies the cluster's S3-compatible
+    /// service availability via the S3 API.
+    ///
+    /// # Returns
+    ///
+    /// * `Ok(())` - All nodes start successfully and the cluster S3 service is ready for requests.
+    /// * `Err(Box<dyn Error + Send + Sync>)` - An error if process spawning fails, TCP port readiness
+    ///   times out, or cluster service readiness times out.
+    pub async fn start(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let binary_path = rustfs_binary_path();
+        let volumes_arg = self.build_volumes_arg();
+
+        for (i, node) in self.nodes.iter_mut().enumerate() {
+            info!("Starting cluster node {} on {}", i, node.address);
+
+            let process = Command::new(&binary_path)
+                .env("RUSTFS_VOLUMES", &volumes_arg)
+                .env("RUSTFS_ADDRESS", &node.address)
+                .env("RUSTFS_ACCESS_KEY", &self.access_key)
+                .env("RUSTFS_SECRET_KEY", &self.secret_key)
+                .env("RUSTFS_CONSOLE_ENABLE", "false")
+                .current_dir(&node.data_dir)
+                .spawn()?;
+
+            node.process = Some(process);
+        }
+
+        for (i, node) in self.nodes.iter().enumerate() {
+            self.wait_for_node_ready(&node.address, i).await?;
+        }
+
+        self.wait_for_service_ready().await?;
+
+        Ok(())
+    }
+
+    /// Wait for a single cluster node's TCP port to become reachable (internal helper method).
+    ///
+    /// Attempts to establish a TCP connection to the node's address, retries up to 60 times
+    /// with a 1-second interval between attempts. Fails if the port is unreachable after all retries.
+    async fn wait_for_node_ready(&self, address: &str, idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        for attempt in 0..60 {
+            if TcpStream::connect(address).await.is_ok() {
+                info!("Node {} ({}) TCP ready after {} attempts", idx, address, attempt + 1);
+                return Ok(());
+            }
+            sleep(Duration::from_secs(1)).await;
+        }
+        Err(format!("Node {} failed to become ready", idx).into())
+    }
+
+    /// Wait for the entire cluster's S3-compatible service to be ready (internal helper method).
+    ///
+    /// Verifies service availability by calling the S3 `list_buckets` API, retries up to 120 times
+    /// with a 1-second interval between attempts. Fails if the API call remains unsuccessful after all retries.
+    async fn wait_for_service_ready(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let client = self.create_s3_client(0)?;
+
+        for attempt in 0..120 {
+            match client.list_buckets().send().await {
+                Ok(_) => {
+                    info!("Cluster service ready after {} attempts", attempt + 1);
+                    return Ok(());
+                }
+                Err(_) => {
+                    sleep(Duration::from_secs(1)).await;
+                }
+            }
+        }
+        Err("Cluster service failed to become ready".into())
+    }
+
+    /// Create an S3 client configured to communicate with a specific cluster node.
+    ///
+    /// Configures the S3 client with the cluster's authentication credentials, a fixed `us-east-1` region,
+    /// the target node's endpoint URL, and enforces path-style access (required for RustFS S3 compatibility).
+    /// Performs a validity check on the node index before creating the client to avoid out-of-bounds errors.
+    ///
+    /// # Arguments
+    ///
+    /// * `node_idx` - The zero-based index of the target cluster node. Must be in the range `[0, total_nodes - 1]`.
+    ///
+    /// # Returns
+    ///
+    /// * `Ok(Client)` - A fully configured AWS S3 `Client` instance for the specified node on success.
+    /// * `Err(Box<dyn Error + Send + Sync>)` - An error if the node index is invalid, or if the S3 client configuration fails.
+    pub fn create_s3_client(&self, node_idx: usize) -> Result<Client, Box<dyn std::error::Error + Send + Sync>> {
+        if node_idx >= self.nodes.len() {
+            return Err("node_idx is invalid".into());
+        }
+        let credentials = Credentials::new(&self.access_key, &self.secret_key, None, None, "cluster-test");
+        let config = Config::builder()
+            .credentials_provider(credentials)
+            .region(Region::new("us-east-1"))
+            .endpoint_url(&self.nodes[node_idx].url)
+            .force_path_style(true)
+            .behavior_version_latest()
+            .build();
+        Ok(Client::from_conf(config))
+    }
+
+    /// Create S3 clients for all nodes in the RustFS cluster and collect them into a vector.
+    ///
+    /// Iterates over all cluster node indices, calls `create_s3_client` for each index, and aggregates
+    /// the resulting clients into a pre-allocated vector. Terminates immediately and returns an error
+    /// if any single node's S3 client creation fails (fails fast behavior).
+    ///
+    /// # Returns
+    ///
+    /// * `Ok(Vec<Client>)` - A vector of configured S3 `Client` instances (one per cluster node) on full success.
+    /// * `Err(Box<dyn Error + Send + Sync>)` - An error with a descriptive message if any client creation fails,
+    ///   including the underlying error from `create_s3_client`.
+    pub fn create_all_clients(&self) -> Result<Vec<Client>, Box<dyn std::error::Error + Send + Sync>> {
+        (0..self.nodes.len()).map(|i| self.create_s3_client(i)).try_fold(
+            Vec::with_capacity(self.nodes.len()),
+            |mut clients, result| match result {
+                Ok(client) => {
+                    clients.push(client);
+                    Ok(clients)
+                }
+                Err(e) => Err(format!("Failed to create S3 client for node: {}", e).into()),
+            },
+        )
+    }
+
+    /// Create a test S3 bucket in the RustFS cluster.
+    ///
+    /// Uses the S3 client of the first cluster node to call the S3 `create_bucket` API and
+    /// create a bucket with the specified name (follows S3 bucket naming conventions).
+    ///
+    /// # Arguments
+    ///
+    /// * `bucket_name` - The name of the bucket to create, must comply with S3 bucket naming
+    ///   rules (lowercase, no spaces, valid characters only).
+    ///
+    /// # Returns
+    ///
+    /// * `Ok(())` - The test bucket is created successfully via the S3 API.
+    /// * `Err(Box<dyn Error + Send + Sync>)` - An error if the S3 `create_bucket` API call fails,
+    ///   such as invalid bucket name, insufficient permissions, or an unready cluster.
+    pub async fn create_test_bucket(&self, bucket_name: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let client = self.create_s3_client(0)?;
+        client.create_bucket().bucket(bucket_name).send().await?;
+        info!("Created test bucket: {}", bucket_name);
+        Ok(())
+    }
+
+    /// Stop all running node processes in the RustFS cluster.
+    ///
+    /// Iterates over all cluster nodes, attempts to kill the spawned RustFS process (if running),
+    /// and waits for the process to exit. Logs an error if process termination or waiting fails,
+    /// but does not panic (fails gracefully).
+    ///
+    /// This method is automatically called by the `Drop` trait when the cluster environment
+    /// is destroyed, and can also be called manually to stop the cluster early.
+    pub fn stop(&mut self) {
+        for (i, node) in self.nodes.iter_mut().enumerate() {
+            if let Some(mut process) = node.process.take() {
+                info!("Stopping cluster node {}", i);
+                if let Err(e) = process.kill() {
+                    error!("Failed to kill cluster node {}: {}", i, e);
+                }
+                if let Err(e) = process.wait() {
+                    error!("Failed to wait for cluster node {} to exit: {}", i, e);
+                }
+            }
+        }
+    }
+}
+
+impl Drop for RustFSTestClusterEnvironment {
+    /// Clean up the RustFS test cluster environment when the instance is dropped.
+    ///
+    /// Automatically calls the `stop` method to terminate all running node processes, then
+    /// attempts to delete the cluster's temporary root directory and all its contents.
+    /// Logs a warning if directory deletion fails (does not affect program exit).
+    fn drop(&mut self) {
+        self.stop();
+        if let Err(e) = std::fs::remove_dir_all(&self.temp_dir) {
+            warn!("Failed to clean up cluster temp directory {}: {}", self.temp_dir, e);
+        }
+    }
+}
@@ -117,7 +117,7 @@ pub async fn create_default_key(

 /// Create a KMS key with a specific ID (by directly writing to the key directory)
 pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    use rand::RngCore;
+    use rand::Rng;
    use std::collections::HashMap;
    use tokio::fs;

@@ -61,6 +61,9 @@ mod protocols;
 #[cfg(test)]
 mod object_lock;

+#[cfg(test)]
+mod cluster_concurrency_test;
+
 // PutObject / MultipartUpload with checksum (Content-MD5, x-amz-checksum-*)
 #[cfg(test)]
 mod checksum_upload_test;
@@ -1,6 +1,6 @@
 # Protocol E2E Tests

-FTPS and SFTP protocol end-to-end tests for RustFS.
+FTPS protocol end-to-end tests for RustFS.

 ## Prerequisites

@@ -21,12 +21,7 @@ brew install sshpass openssh

 Run all protocol tests:
 ```bash
-cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
-```
-
-Run only FTPS tests:
-```bash
-cargo test --package e2e_test test_ftps_core_operations -- --test-threads=1 --nocapture
+RUSTFS_BUILD_FEATURES=ftps cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
 ```

 ## Test Coverage
@@ -41,4 +36,5 @@ cargo test --package e2e_test test_ftps_core_operations -- --test-threads=1 --no
 - cd nonexistent bucket (should fail)
 - delete object
 - cdup
- rmdir delete bucket
+- rmdir delete bucket
+
@@ -36,15 +36,26 @@ const FTPS_ADDRESS: &str = "127.0.0.1:9021";
 pub async fn test_ftps_core_operations() -> Result<()> {
    let env = ProtocolTestEnvironment::new().map_err(|e| anyhow::anyhow!("{}", e))?;

-    // Generate and write certificate
-    let cert = generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()])?;
-    let cert_path = PathBuf::from(&env.temp_dir).join("ftps.crt");
-    let key_path = PathBuf::from(&env.temp_dir).join("ftps.key");
+    let cert_dir = PathBuf::from(&env.temp_dir).join("ftps_certs");
+    tokio::fs::create_dir_all(&cert_dir).await?;

-    let cert_pem = cert.cert.pem();
-    let key_pem = cert.signing_key.serialize_pem();
-    tokio::fs::write(&cert_path, &cert_pem).await?;
-    tokio::fs::write(&key_path, &key_pem).await?;
+    // Generate default certificate for root directory
+    let default_cert = generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()])?;
+    let default_cert_path = cert_dir.join("rustfs_cert.pem");
+    let default_key_path = cert_dir.join("rustfs_key.pem");
+    tokio::fs::write(&default_cert_path, default_cert.cert.pem()).await?;
+    tokio::fs::write(&default_key_path, default_cert.signing_key.serialize_pem()).await?;
+
+    // Create subdirectory for domain-specific certificate
+    let example_domain_dir = cert_dir.join("example1.com");
+    tokio::fs::create_dir_all(&example_domain_dir).await?;
+    let domain_cert = generate_simple_self_signed(vec!["example1.com".to_string()])?;
+    let domain_cert_path = example_domain_dir.join("rustfs_cert.pem");
+    let domain_key_path = example_domain_dir.join("rustfs_key.pem");
+    tokio::fs::write(&domain_cert_path, domain_cert.cert.pem()).await?;
+    tokio::fs::write(&domain_key_path, domain_cert.signing_key.serialize_pem()).await?;
+
+    info!("Generated 2 certificates in {:?}", cert_dir);

    // Start server manually
    info!("Starting FTPS server on {}", FTPS_ADDRESS);
@@ -52,8 +63,7 @@ pub async fn test_ftps_core_operations() -> Result<()> {
    let mut server_process = Command::new(&binary_path)
        .env("RUSTFS_FTPS_ENABLE", "true")
        .env("RUSTFS_FTPS_ADDRESS", FTPS_ADDRESS)
-        .env("RUSTFS_FTPS_CERTS_FILE", cert_path.to_str().unwrap())
-        .env("RUSTFS_FTPS_KEY_FILE", key_path.to_str().unwrap())
+        .env("RUSTFS_FTPS_CERTS_DIR", cert_dir.to_str().unwrap())
        .arg(&env.temp_dir)
        .spawn()?;

@@ -73,7 +83,7 @@ pub async fn test_ftps_core_operations() -> Result<()> {
        let mut root_store = RootCertStore::empty();
        // Add the self-signed certificate to the trust store for e2e
        // Note: In a real environment, you'd use proper root certificates
-        let cert_pem = cert.cert.pem();
+        let cert_pem = default_cert.cert.pem();
        let cert_der = rustls_pemfile::certs(&mut Cursor::new(cert_pem))
            .collect::<Result<Vec<_>, _>>()
            .map_err(|e| anyhow::anyhow!("Failed to parse cert: {}", e))?;
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-//! Protocol tests for FTPS and SFTP
+//! Protocol tests for FTPS

 pub mod ftps_core;
 pub mod test_env;
@@ -59,12 +59,9 @@ struct TestDefinition {
 impl ProtocolTestSuite {
    /// Create default test suite
    pub fn new() -> Self {
-        let tests = vec![
-            TestDefinition {
-                name: "test_ftps_core_operations".to_string(),
-            },
-            // TestDefinition { name: "test_sftp_core_operations".to_string() },
-        ];
+        let tests = vec![TestDefinition {
+            name: "test_ftps_core_operations".to_string(),
+        }];

        Self { tests }
    }
@@ -86,10 +83,6 @@ impl ProtocolTestSuite {
                    info!("=== Starting FTPS Module Test ===");
                    "FTPS core operations (put, ls, mkdir, rmdir, delete)"
                }
-                "test_sftp_core_operations" => {
-                    info!("=== Starting SFTP Module Test ===");
-                    "SFTP core operations (put, ls, mkdir, rmdir, delete)"
-                }
                _ => "",
            };

@@ -128,7 +121,6 @@ impl ProtocolTestSuite {
    async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        match test_def.name.as_str() {
            "test_ftps_core_operations" => test_ftps_core_operations().await.map_err(|e| e.into()),
-            // "test_sftp_core_operations" => test_sftp_core_operations().await.map_err(|e| e.into()),
            _ => Err(format!("Test {} not implemented", test_def.name).into()),
        }
    }
@@ -422,7 +422,7 @@ mod integration_tests {
        // Check if we can upload 1KB (should succeed - we haven't used the full quota yet)
        let check_result = env.check_bucket_quota("PUT", 1024).await?;
        assert!(check_result.get("allowed").unwrap().as_bool().unwrap());
-        assert_eq!(check_result.get("remaining_quota").unwrap().as_u64().unwrap(), 512 * 1024 - 1024);
+        assert_eq!(check_result.get("remaining_quota").unwrap().as_u64().unwrap(), 523264); // 511 * 1024

        // Check if we can upload 600KB (should fail - would exceed quota)
        let check_result = env.check_bucket_quota("PUT", 600 * 1024).await?;
@@ -570,6 +570,60 @@ mod integration_tests {
        Ok(())
    }

+    /// Test that a normal user with `readwrite` policy can read quota but cannot set/clear quota.
+    #[tokio::test]
+    #[serial]
+    async fn test_quota_normal_user_permissions() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        init_logging();
+        let env = QuotaTestEnv::new().await?;
+        env.create_bucket().await?;
+
+        // Admin sets quota first
+        env.set_bucket_quota(1024 * 1024).await?;
+
+        // Create a normal user via admin API
+        let normal_ak = "normaluser";
+        let normal_sk = "normaluser123";
+        let add_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.env.url, normal_ak);
+        let user_body = serde_json::json!({ "secretKey": normal_sk, "status": "enabled" }).to_string();
+        awscurl_put(&add_user_url, &user_body, &env.env.access_key, &env.env.secret_key).await?;
+
+        // Attach `readwrite` policy to the normal user
+        let policy_url = format!(
+            "{}/rustfs/admin/v3/set-user-or-group-policy?policyName=readwrite&userOrGroup={}&isGroup=false",
+            env.env.url, normal_ak
+        );
+        awscurl_put(&policy_url, "", &env.env.access_key, &env.env.secret_key).await?;
+
+        // Normal user reads quota — should succeed
+        let get_url = format!("{}/rustfs/admin/v3/quota/{}", env.env.url, env.bucket_name);
+        let resp = awscurl_get(&get_url, normal_ak, normal_sk).await?;
+        let quota_info: serde_json::Value = serde_json::from_str(&resp)?;
+        assert_eq!(quota_info.get("quota").and_then(|v| v.as_u64()), Some(1024 * 1024));
+
+        // Normal user reads quota stats — should succeed
+        let stats_url = format!("{}/rustfs/admin/v3/quota-stats/{}", env.env.url, env.bucket_name);
+        let resp = awscurl_get(&stats_url, normal_ak, normal_sk).await?;
+        assert!(resp.contains("quota_limit"));
+
+        // Normal user sets quota — should be denied
+        let set_resp = awscurl_put(
+            &get_url,
+            &serde_json::json!({"quota": 2048, "quota_type": "HARD"}).to_string(),
+            normal_ak,
+            normal_sk,
+        )
+        .await;
+        assert!(set_resp.is_err(), "normal user should not be able to set quota");
+
+        // Normal user clears quota — should be denied
+        let del_resp = awscurl_delete(&get_url, normal_ak, normal_sk).await;
+        assert!(del_resp.is_err(), "normal user should not be able to clear quota");
+
+        env.cleanup_bucket().await?;
+        Ok(())
+    }
+
    #[tokio::test]
    #[serial]
    async fn test_quota_copy_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -67,6 +67,22 @@ mod tests {
        Ok(())
    }

+    async fn suspend_versioning(client: &Client, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let versioning_config = VersioningConfiguration::builder()
+            .status(BucketVersioningStatus::Suspended)
+            .build();
+
+        client
+            .put_bucket_versioning()
+            .bucket(bucket)
+            .versioning_configuration(versioning_config)
+            .send()
+            .await?;
+
+        info!("✅ Versioning suspended for bucket {}", bucket);
+        Ok(())
+    }
+
    /// Test 1: PutObject should return version_id when versioning is enabled
    /// This directly addresses the Veeam issue from #1066
    #[tokio::test]
@@ -395,4 +411,91 @@ mod tests {

        info!("✅ PASSED: Veeam backup workflow simulation completed successfully");
    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_terraform_put_after_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        init_logging();
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        // Use a versioned bucket for this test
+        let bucket = "terraform";
+
+        let client = env.create_s3_client();
+        env.create_test_bucket(bucket).await?;
+
+        let key = "terraform.tfstate";
+        let response = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from(b"v1".to_vec()))
+            .send()
+            .await;
+        assert!(response.is_ok());
+
+        client.delete_object().bucket(bucket).key(key).send().await?;
+
+        let response = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from(b"v1".to_vec()))
+            .send()
+            .await;
+
+        assert!(response.is_ok());
+
+        let get_response = client.get_object().bucket(bucket).key(key).send().await;
+        assert!(get_response.is_ok(), "Object should exist after PUT");
+
+        Ok(())
+    }
+
+    /// Test 7: PutObject should return "null" version_id when versioning is Suspended
+    #[tokio::test]
+    #[serial]
+    async fn test_put_object_returns_null_version_id_with_suspended_versioning() {
+        init_logging();
+        info!("🧪 TEST: PutObject returns null version_id with versioning suspended");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-suspended-version-id";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+        suspend_versioning(&client, bucket)
+            .await
+            .expect("Failed to suspend versioning");
+
+        let key = "test-file-suspended.txt";
+        let content = b"Test content for suspended version ID test";
+
+        info!("📤 Uploading object to suspended versioning bucket");
+        let result = client
+            .put_object()
+            .bucket(bucket)
+            .key(key)
+            .body(ByteStream::from_static(content))
+            .send()
+            .await;
+
+        assert!(result.is_ok(), "PutObject failed: {:?}", result.err());
+        let output = result.unwrap();
+
+        info!("📥 PutObject response - version_id: {:?}", output.version_id);
+
+        // When suspended, version_id must be "null"
+        assert_eq!(
+            output.version_id.as_deref(),
+            Some("null"),
+            "❌ FAILED: version_id should be 'null' when versioning is suspended"
+        );
+
+        info!("✅ PASSED: PutObject correctly returns 'null' version_id");
+    }
 }
@@ -52,6 +52,7 @@ glob = { workspace = true }
 thiserror.workspace = true
 flatbuffers.workspace = true
 futures.workspace = true
+futures-util.workspace = true
 tracing.workspace = true
 serde.workspace = true
 time.workspace = true
@@ -60,6 +61,8 @@ serde_json.workspace = true
 quick-xml = { workspace = true, features = ["serialize", "async-tokio"] }
 s3s.workspace = true
 http.workspace = true
+http-body = { workspace = true }
+http-body-util.workspace = true
 url.workspace = true
 uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] }
 reed-solomon-simd = { workspace = true }
@@ -64,6 +64,8 @@ use uuid::Uuid;
 const DEFAULT_HEALTH_CHECK_DURATION: Duration = Duration::from_secs(5);
 const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);

+const REPLICATION_REQUEST_TRUE: HeaderValue = HeaderValue::from_static("true");
+
 pub static GLOBAL_BUCKET_TARGET_SYS: OnceLock<BucketTargetSys> = OnceLock::new();

 #[derive(Debug, Clone)]
@@ -999,10 +1001,14 @@ impl PutObjectOptions {
        if self.internal.source_mtime.unix_timestamp() != 0 {
            header.insert(
                RUSTFS_BUCKET_SOURCE_MTIME,
-                HeaderValue::from_str(&self.internal.source_mtime.unix_timestamp().to_string()).expect("err"),
+                HeaderValue::from_str(&self.internal.source_mtime.format(&Rfc3339).unwrap_or_default()).expect("err"),
            );
        }

+        if self.internal.replication_request {
+            header.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE);
+        }
+
        header
    }

@@ -1211,6 +1217,9 @@ impl TargetClient {
        {
            headers.insert(RUSTFS_BUCKET_SOURCE_VERSION_ID, header_value);
        }
+        if opts.internal.replication_request {
+            headers.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE);
+        }

        match self
            .client
@@ -18,12 +18,10 @@
 #![allow(unused_must_use)]
 #![allow(clippy::all)]

-use crate::bucket::lifecycle::rule::TransitionOps;
-use crate::store_api::ObjectInfo;
 use rustfs_filemeta::{ReplicationStatusType, VersionPurgeStatusType};
 use s3s::dto::{
    BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionTransition,
-    ObjectLockConfiguration, ObjectLockEnabled, RestoreRequest, Transition,
+    ObjectLockConfiguration, ObjectLockEnabled, RestoreRequest, Transition, TransitionStorageClass,
 };
 use std::cmp::Ordering;
 use std::collections::HashMap;
@@ -35,6 +33,9 @@ use time::{self, Duration, OffsetDateTime};
 use tracing::info;
 use uuid::Uuid;

+use crate::bucket::lifecycle::rule::TransitionOps;
+use crate::store_api::ObjectInfo;
+
 pub const TRANSITION_COMPLETE: &str = "complete";
 pub const TRANSITION_PENDING: &str = "pending";
 const ERR_LIFECYCLE_NO_RULE: &str = "Lifecycle configuration should have at least one rule";
@@ -171,44 +172,51 @@ impl Lifecycle for BucketLifecycleConfiguration {
                continue;
            }

-            let rule_prefix = rule.prefix.as_ref().expect("err!");
+            let rule_prefix = &rule.prefix.clone().unwrap_or_default();
            if prefix.len() > 0 && rule_prefix.len() > 0 && !prefix.starts_with(rule_prefix) && !rule_prefix.starts_with(&prefix)
            {
                continue;
            }

-            let rule_noncurrent_version_expiration = rule.noncurrent_version_expiration.as_ref().expect("err!");
-            if rule_noncurrent_version_expiration.noncurrent_days.expect("err!") > 0 {
+            if let Some(rule_noncurrent_version_expiration) = &rule.noncurrent_version_expiration {
+                if let Some(noncurrent_days) = rule_noncurrent_version_expiration.noncurrent_days {
+                    if noncurrent_days > 0 {
+                        return true;
+                    }
+                }
+                if let Some(newer_noncurrent_versions) = rule_noncurrent_version_expiration.newer_noncurrent_versions {
+                    if newer_noncurrent_versions > 0 {
+                        return true;
+                    }
+                }
+            }
+            if rule.noncurrent_version_transitions.is_some() {
                return true;
            }
-            if rule_noncurrent_version_expiration.newer_noncurrent_versions.expect("err!") > 0 {
-                return true;
+            if let Some(rule_expiration) = &rule.expiration {
+                if let Some(date1) = rule_expiration.date.clone() {
+                    if OffsetDateTime::from(date1).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() {
+                        return true;
+                    }
+                }
+                if rule_expiration.date.is_some() {
+                    return true;
+                }
+                if let Some(expired_object_delete_marker) = rule_expiration.expired_object_delete_marker
+                    && expired_object_delete_marker
+                {
+                    return true;
+                }
            }
-            if !rule.noncurrent_version_transitions.is_none() {
-                return true;
+            if let Some(rule_transitions) = &rule.transitions {
+                let rule_transitions_0 = rule_transitions[0].clone();
+                if let Some(date1) = rule_transitions_0.date {
+                    if OffsetDateTime::from(date1).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp() {
+                        return true;
+                    }
+                }
            }
-            let rule_expiration = rule.expiration.as_ref().expect("err!");
-            if !rule_expiration.date.is_none()
-                && OffsetDateTime::from(rule_expiration.date.clone().expect("err!")).unix_timestamp()
-                    < OffsetDateTime::now_utc().unix_timestamp()
-            {
-                return true;
-            }
-            if !rule_expiration.date.is_none() {
-                return true;
-            }
-            if rule_expiration.expired_object_delete_marker.expect("err!") {
-                return true;
-            }
-            let rule_transitions: &[Transition] = &rule.transitions.as_ref().expect("err!");
-            let rule_transitions_0 = rule_transitions[0].clone();
-            if !rule_transitions_0.date.is_none()
-                && OffsetDateTime::from(rule_transitions_0.date.expect("err!")).unix_timestamp()
-                    < OffsetDateTime::now_utc().unix_timestamp()
-            {
-                return true;
-            }
-            if !rule.transitions.is_none() {
+            if rule.transitions.is_some() {
                return true;
            }
        }
@@ -232,7 +240,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                            return Err(std::io::Error::other(ERR_LIFECYCLE_BUCKET_LOCKED));
                        }
                    }
-                        }
+                }
            }*/
        }
        for (i, _) in self.rules.iter().enumerate() {
@@ -325,7 +333,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                        if let Some(expired_object_delete_marker) = expiration.expired_object_delete_marker {
                            events.push(Event {
                                action: IlmAction::DeleteVersionAction,
-                                rule_id: rule.id.clone().expect("err!"),
+                                rule_id: rule.id.clone().unwrap_or_default(),
                                due: Some(now),
                                noncurrent_days: 0,
                                newer_noncurrent_versions: 0,
@@ -339,7 +347,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                            if now.unix_timestamp() >= expected_expiry.unix_timestamp() {
                                events.push(Event {
                                    action: IlmAction::DeleteVersionAction,
-                                    rule_id: rule.id.clone().expect("err!"),
+                                    rule_id: rule.id.clone().unwrap_or_default(),
                                    due: Some(expected_expiry),
                                    noncurrent_days: 0,
                                    newer_noncurrent_versions: 0,
@@ -360,7 +368,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                                    if now.unix_timestamp() >= due.unix_timestamp() {
                                        events.push(Event {
                                            action: IlmAction::DelMarkerDeleteAllVersionsAction,
-                                            rule_id: rule.id.clone().expect("err!"),
+                                            rule_id: rule.id.clone().unwrap_or_default(),
                                            due: Some(due),
                                            noncurrent_days: 0,
                                            newer_noncurrent_versions: 0,
@@ -393,7 +401,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                                    if now.unix_timestamp() >= expected_expiry.unix_timestamp() {
                                        events.push(Event {
                                            action: IlmAction::DeleteVersionAction,
-                                            rule_id: rule.id.clone().expect("err!"),
+                                            rule_id: rule.id.clone().unwrap_or_default(),
                                            due: Some(expected_expiry),
                                            noncurrent_days: 0,
                                            newer_noncurrent_versions: 0,
@@ -416,7 +424,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                                    if now.unix_timestamp() == 0 || now.unix_timestamp() > due0.unix_timestamp() {
                                        events.push(Event {
                                            action: IlmAction::TransitionVersionAction,
-                                            rule_id: rule.id.clone().expect("err!"),
+                                            rule_id: rule.id.clone().unwrap_or_default(),
                                            due,
                                            storage_class: rule.noncurrent_version_transitions.as_ref().unwrap()[0]
                                                .storage_class
@@ -450,7 +458,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                                info!("eval_inner: expiration by date - date0={:?}", date0);
                                events.push(Event {
                                    action: IlmAction::DeleteAction,
-                                    rule_id: rule.id.clone().expect("err!"),
+                                    rule_id: rule.id.clone().unwrap_or_default(),
                                    due: Some(date0),
                                    noncurrent_days: 0,
                                    newer_noncurrent_versions: 0,
@@ -471,7 +479,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
                                info!("eval_inner: object should expire, adding DeleteAction");
                                let mut event = Event {
                                    action: IlmAction::DeleteAction,
-                                    rule_id: rule.id.clone().expect("err!"),
+                                    rule_id: rule.id.clone().unwrap_or_default(),
                                    due: Some(expected_expiry),
                                    noncurrent_days: 0,
                                    newer_noncurrent_versions: 0,
@@ -496,9 +504,14 @@ impl Lifecycle for BucketLifecycleConfiguration {
                                if now.unix_timestamp() == 0 || now.unix_timestamp() > due0.unix_timestamp() {
                                    events.push(Event {
                                        action: IlmAction::TransitionAction,
-                                        rule_id: rule.id.clone().expect("err!"),
+                                        rule_id: rule.id.clone().unwrap_or_default(),
                                        due,
-                                        storage_class: transitions[0].storage_class.clone().expect("err!").as_str().to_string(),
+                                        storage_class: transitions[0]
+                                            .storage_class
+                                            .clone()
+                                            .unwrap_or(TransitionStorageClass::from_static(""))
+                                            .as_str()
+                                            .to_string(),
                                        noncurrent_days: 0,
                                        newer_noncurrent_versions: 0,
                                    });
@@ -512,9 +525,10 @@ impl Lifecycle for BucketLifecycleConfiguration {

        if events.len() > 0 {
            events.sort_by(|a, b| {
-                if now.unix_timestamp() > a.due.expect("err!").unix_timestamp()
-                    && now.unix_timestamp() > b.due.expect("err").unix_timestamp()
-                    || a.due.expect("err").unix_timestamp() == b.due.expect("err").unix_timestamp()
+                if now.unix_timestamp() > a.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp()
+                    && now.unix_timestamp() > b.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp()
+                    || a.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp()
+                        == b.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp()
                {
                    match a.action {
                        IlmAction::DeleteAllVersionsAction
@@ -537,7 +551,9 @@ impl Lifecycle for BucketLifecycleConfiguration {
                    return Ordering::Less;
                }

-                if a.due.expect("err").unix_timestamp() < b.due.expect("err").unix_timestamp() {
+                if a.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp()
+                    < b.due.unwrap_or_else(|| OffsetDateTime::UNIX_EPOCH).unix_timestamp()
+                {
                    return Ordering::Less;
                }
                return Ordering::Greater;
@@ -558,8 +574,8 @@ impl Lifecycle for BucketLifecycleConfiguration {
                        }
                        return Event {
                            action: IlmAction::DeleteVersionAction,
-                            rule_id: rule.id.clone().expect("err"),
-                            noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
+                            rule_id: rule.id.clone().unwrap_or_default(),
+                            noncurrent_days: noncurrent_version_expiration.noncurrent_days.unwrap_or(0) as u32,
                            newer_noncurrent_versions: newer_noncurrent_versions as usize,
                            due: Some(OffsetDateTime::UNIX_EPOCH),
                            storage_class: "".into(),
@@ -567,8 +583,8 @@ impl Lifecycle for BucketLifecycleConfiguration {
                    } else {
                        return Event {
                            action: IlmAction::DeleteVersionAction,
-                            rule_id: rule.id.clone().expect("err"),
-                            noncurrent_days: noncurrent_version_expiration.noncurrent_days.expect("noncurrent_days err.") as u32,
+                            rule_id: rule.id.clone().unwrap_or_default(),
+                            noncurrent_days: noncurrent_version_expiration.noncurrent_days.unwrap_or(0) as u32,
                            newer_noncurrent_versions: 0,
                            due: Some(OffsetDateTime::UNIX_EPOCH),
                            storage_class: "".into(),
@@ -106,7 +106,7 @@ pub fn check_retention_for_modification(
    None
 }

-pub(crate) fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
+pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
    let target_year = dt.year() + years;
    dt.replace_year(target_year)
        .or_else(|_| {
@@ -36,6 +36,18 @@ impl QuotaChecker {
        bucket: &str,
        operation: QuotaOperation,
        operation_size: u64,
+    ) -> Result<QuotaCheckResult, QuotaError> {
+        self.check_quota_with_usage_reporting(bucket, operation, operation_size, false)
+            .await
+    }
+
+    /// Check quota with option to force usage calculation even when no quota is configured
+    pub async fn check_quota_with_usage_reporting(
+        &self,
+        bucket: &str,
+        operation: QuotaOperation,
+        operation_size: u64,
+        force_usage_calculation: bool,
    ) -> Result<QuotaCheckResult, QuotaError> {
        let start_time = Instant::now();
        let quota_config = self.get_quota_config(bucket).await?;
@@ -43,7 +55,11 @@ impl QuotaChecker {
        // If no quota limit is set, allow operation
        let quota_limit = match quota_config.quota {
            None => {
-                let current_usage = self.get_real_time_usage(bucket).await?;
+                let current_usage = if force_usage_calculation {
+                    Some(self.get_real_time_usage(bucket).await?)
+                } else {
+                    None // Skip expensive usage calculation when no quota and not forced for performance
+                };
                return Ok(QuotaCheckResult {
                    allowed: true,
                    current_usage,
@@ -58,12 +74,12 @@ impl QuotaChecker {
        let current_usage = self.get_real_time_usage(bucket).await?;

        let expected_usage = match operation {
-            QuotaOperation::PutObject | QuotaOperation::CopyObject => current_usage + operation_size,
+            QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => current_usage + operation_size,
            QuotaOperation::DeleteObject => current_usage.saturating_sub(operation_size),
        };

        let allowed = match operation {
-            QuotaOperation::PutObject | QuotaOperation::CopyObject => {
+            QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
                quota_config.check_operation_allowed(current_usage, operation_size)
            }
            QuotaOperation::DeleteObject => true,
@@ -84,7 +100,7 @@ impl QuotaChecker {

        let result = QuotaCheckResult {
            allowed,
-            current_usage,
+            current_usage: Some(current_usage),
            quota_limit: Some(quota_limit),
            operation_size,
            remaining,
@@ -165,7 +181,7 @@ mod tests {
    async fn test_quota_check_no_limit() {
        let result = QuotaCheckResult {
            allowed: true,
-            current_usage: 0,
+            current_usage: None,
            quota_limit: None,
            operation_size: 1024,
            remaining: None,
@@ -80,7 +80,8 @@ impl BucketQuota {
 #[derive(Debug)]
 pub struct QuotaCheckResult {
    pub allowed: bool,
-    pub current_usage: u64,
+    /// current_usage: None when skipped for performance (no quota configured)
+    pub current_usage: Option<u64>,
    /// quota_limit: None means unlimited
    pub quota_limit: Option<u64>,
    pub operation_size: u64,
@@ -90,6 +91,7 @@ pub struct QuotaCheckResult {
 #[derive(Debug)]
 pub enum QuotaOperation {
    PutObject,
+    PostObject,
    CopyObject,
    DeleteObject,
 }
@@ -782,7 +782,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
            }

            // Generate random duration between 0 and 1 minute
-            use rand::Rng;
+            use rand::RngExt;
            let duration_millis = rand::rng().random_range(0..60_000);
            let mut duration = Duration::from_millis(duration_millis);

@@ -28,6 +28,7 @@ use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_fo
 use crate::event::name::EventName;
 use crate::event_notification::{EventArgs, send_event};
 use crate::global::GLOBAL_LocalNodeName;
+use crate::set_disk::get_lock_acquire_timeout;
 use crate::store_api::{DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, WalkOptions};
 use crate::{StorageAPI, new_object_layer_fn};
 use aws_sdk_s3::error::SdkError;
@@ -1273,7 +1274,58 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
        }
    };

-    //TODO: nslock
+    let ns_lock = match storage
+        .new_ns_lock(&bucket, format!("/[replicate]/{}", dobj.delete_object.object_name).as_str())
+        .await
+    {
+        Ok(ns_lock) => ns_lock,
+        Err(e) => {
+            warn!(
+                "failed to get ns lock for bucket:{} object:{} error:{}",
+                bucket, dobj.delete_object.object_name, e
+            );
+            send_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: dobj.delete_object.object_name.clone(),
+                    version_id,
+                    delete_marker: dobj.delete_object.delete_marker,
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                host: GLOBAL_LocalNodeName.to_string(),
+                ..Default::default()
+            });
+            return;
+        }
+    };
+
+    let _lock_guard = match ns_lock.get_write_lock(get_lock_acquire_timeout()).await {
+        Ok(lock_guard) => lock_guard,
+        Err(e) => {
+            warn!(
+                "failed to get write lock for bucket:{} object:{} error:{}",
+                bucket, dobj.delete_object.object_name, e
+            );
+            send_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.as_ref().to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: dobj.delete_object.object_name.clone(),
+                    version_id,
+                    delete_marker: dobj.delete_object.delete_marker,
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                host: GLOBAL_LocalNodeName.to_string(),
+                ..Default::default()
+            });
+            return;
+        }
+    };

    // Initialize replicated infos
    let mut rinfos = ReplicatedInfos {
@@ -1379,7 +1431,7 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
        dobj.delete_object.version_id.map(|v| v.to_string()),
    );
    if replication_status != prev_status {
-        drs.replica_timestamp = Some(OffsetDateTime::now_utc());
+        drs.replication_timestamp = Some(OffsetDateTime::now_utc());
    }

    let event_name = if replication_status == ReplicationStatusType::Completed {
@@ -1521,6 +1573,13 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
        }
    }

+    if rinfo.replication_status == ReplicationStatusType::Completed
+        && !tgt_client.reset_id.is_empty()
+        && dobj.op_type == ReplicationType::ExistingObject
+    {
+        rinfo.resync_timestamp = format!("{};{}", OffsetDateTime::now_utc().format(&Rfc3339).unwrap(), tgt_client.reset_id);
+    }
+
    rinfo
 }

@@ -1621,8 +1680,13 @@ pub async fn replicate_object<S: StorageAPI>(roi: ReplicateObjectInfo, storage:
    let mut object_info = roi.to_object_info();

    if roi.replication_status_internal != new_replication_internal || rinfos.replication_resynced() {
+        let mut eval_metadata = HashMap::new();
+        if let Some(ref s) = new_replication_internal {
+            eval_metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}replication-status"), s.clone());
+        }
        let popts = ObjectOptions {
            version_id: roi.version_id.map(|v| v.to_string()),
+            eval_metadata: Some(eval_metadata),
            ..Default::default()
        };

@@ -1772,6 +1836,36 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
            return rinfo;
        }

+        let mut replication_action = replication_action;
+        match tgt_client
+            .head_object(&tgt_client.bucket, &object, self.version_id.map(|v| v.to_string()))
+            .await
+        {
+            Ok(oi) => {
+                replication_action = get_replication_action(&object_info, &oi, self.op_type);
+                if replication_action == ReplicationAction::None {
+                    rinfo.replication_status = ReplicationStatusType::Completed;
+                    rinfo.replication_resynced = true;
+                    rinfo.replication_action = ReplicationAction::None;
+                    rinfo.size = size;
+                    return rinfo;
+                }
+            }
+            Err(e) => {
+                if let Some(se) = e.as_service_error() {
+                    if !se.is_not_found() {
+                        rinfo.error = Some(e.to_string());
+                        warn!("replication head_object failed bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
+                        return rinfo;
+                    }
+                } else {
+                    rinfo.error = Some(e.to_string());
+                    warn!("replication head_object failed bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
+                    return rinfo;
+                }
+            }
+        }
+
        rinfo.replication_status = ReplicationStatusType::Completed;
        rinfo.replication_resynced = true;
        rinfo.size = size;
@@ -1830,6 +1924,10 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
        } {
            rinfo.replication_status = ReplicationStatusType::Failed;
            rinfo.error = Some(err.to_string());
+            warn!(
+                "replication put_object failed src_bucket={} dest_bucket={} object={} err={:?}",
+                bucket, tgt_client.bucket, object, err
+            );

            // TODO: check offline
            return rinfo;
@@ -36,10 +36,7 @@ impl VersioningApi for VersioningConfiguration {
            return true;
        }

-        if let Some(exclude_folders) = self.exclude_folders
-            && exclude_folders
-            && prefix.ends_with('/')
-        {
+        if self.exclude_folders.unwrap_or(false) && prefix.ends_with('/') {
            return false;
        }

@@ -18,8 +18,10 @@
 #![allow(unused_must_use)]
 #![allow(clippy::all)]

-use bytes::Bytes;
 use http::{HeaderMap, StatusCode};
+use http_body_util::BodyExt;
+use hyper::body::Body;
+use hyper::body::Bytes;
 use std::collections::HashMap;

 use crate::client::{
@@ -61,9 +63,19 @@ impl TransitionClient {

        let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
        //defer closeResponse(resp)
+
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
        //if resp != nil {
-        if resp.status() != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK {
-            return Err(std::io::Error::other(http_resp_to_error_response(&resp, vec![], bucket_name, "")));
+        if resp_status != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(
+                resp_status,
+                &h,
+                vec![],
+                bucket_name,
+                "",
+            )));
        }
        //}
        Ok(())
@@ -97,8 +109,17 @@ impl TransitionClient {
            .await?;
        //defer closeResponse(resp)

-        if resp.status() != StatusCode::NO_CONTENT {
-            return Err(std::io::Error::other(http_resp_to_error_response(&resp, vec![], bucket_name, "")));
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
+        if resp_status != StatusCode::NO_CONTENT {
+            return Err(std::io::Error::other(http_resp_to_error_response(
+                resp_status,
+                &h,
+                vec![],
+                bucket_name,
+                "",
+            )));
        }

        Ok(())
@@ -136,7 +157,15 @@ impl TransitionClient {
            )
            .await?;

-        let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string();
+        let mut body_vec = Vec::new();
+        let mut body = resp.into_body();
+        while let Some(frame) = body.frame().await {
+            let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+            if let Some(data) = frame.data_ref() {
+                body_vec.extend_from_slice(data);
+            }
+        }
+        let policy = String::from_utf8_lossy(&body_vec).to_string();
        Ok(policy)
    }
 }
@@ -18,12 +18,11 @@
 #![allow(unused_must_use)]
 #![allow(clippy::all)]

-use http::StatusCode;
+use http::{HeaderMap, StatusCode};
 use serde::{Deserialize, Serialize};
 use serde::{de::Deserializer, ser::Serializer};
 use std::fmt::Display;

-use s3s::Body;
 use s3s::S3ErrorCode;

 const _REPORT_ISSUE: &str = "Please report this issue at https://github.com/rustfs/rustfs/issues.";
@@ -95,20 +94,31 @@ pub fn to_error_response(err: &std::io::Error) -> ErrorResponse {
 }

 pub fn http_resp_to_error_response(
-    resp: &http::Response<Body>,
+    resp_status: StatusCode,
+    h: &HeaderMap,
    b: Vec<u8>,
    bucket_name: &str,
    object_name: &str,
 ) -> ErrorResponse {
    let err_body = String::from_utf8(b).unwrap();
+    if h.is_empty() || resp_status.is_client_error() || resp_status.is_server_error() {
+        return ErrorResponse {
+            status_code: resp_status,
+            code: S3ErrorCode::ResponseInterrupted,
+            message: "Invalid HTTP response.".to_string(),
+            bucket_name: bucket_name.to_string(),
+            key: object_name.to_string(),
+            ..Default::default()
+        };
+    }
    let err_resp_ = quick_xml::de::from_str::<ErrorResponse>(&err_body);
    let mut err_resp = ErrorResponse::default();
    if err_resp_.is_err() {
-        match resp.status() {
+        match resp_status {
            StatusCode::NOT_FOUND => {
                if object_name == "" {
                    err_resp = ErrorResponse {
-                        status_code: resp.status(),
+                        status_code: resp_status,
                        code: S3ErrorCode::NoSuchBucket,
                        message: "The specified bucket does not exist.".to_string(),
                        bucket_name: bucket_name.to_string(),
@@ -116,7 +126,7 @@ pub fn http_resp_to_error_response(
                    };
                } else {
                    err_resp = ErrorResponse {
-                        status_code: resp.status(),
+                        status_code: resp_status,
                        code: S3ErrorCode::NoSuchKey,
                        message: "The specified key does not exist.".to_string(),
                        bucket_name: bucket_name.to_string(),
@@ -127,7 +137,7 @@ pub fn http_resp_to_error_response(
            }
            StatusCode::FORBIDDEN => {
                err_resp = ErrorResponse {
-                    status_code: resp.status(),
+                    status_code: resp_status,
                    code: S3ErrorCode::AccessDenied,
                    message: "Access Denied.".to_string(),
                    bucket_name: bucket_name.to_string(),
@@ -137,7 +147,7 @@ pub fn http_resp_to_error_response(
            }
            StatusCode::CONFLICT => {
                err_resp = ErrorResponse {
-                    status_code: resp.status(),
+                    status_code: resp_status,
                    code: S3ErrorCode::BucketNotEmpty,
                    message: "Bucket not empty.".to_string(),
                    bucket_name: bucket_name.to_string(),
@@ -146,7 +156,7 @@ pub fn http_resp_to_error_response(
            }
            StatusCode::PRECONDITION_FAILED => {
                err_resp = ErrorResponse {
-                    status_code: resp.status(),
+                    status_code: resp_status,
                    code: S3ErrorCode::PreconditionFailed,
                    message: "Pre condition failed.".to_string(),
                    bucket_name: bucket_name.to_string(),
@@ -155,13 +165,13 @@ pub fn http_resp_to_error_response(
                };
            }
            _ => {
-                let mut msg = resp.status().to_string();
+                let mut msg = resp_status.to_string();
                if err_body.len() > 0 {
                    msg = err_body;
                }
                err_resp = ErrorResponse {
-                    status_code: resp.status(),
-                    code: S3ErrorCode::Custom(resp.status().to_string().into()),
+                    status_code: resp_status,
+                    code: S3ErrorCode::Custom(resp_status.to_string().into()),
                    message: msg,
                    bucket_name: bucket_name.to_string(),
                    ..Default::default()
@@ -171,32 +181,32 @@ pub fn http_resp_to_error_response(
    } else {
        err_resp = err_resp_.unwrap();
    }
-    err_resp.status_code = resp.status();
-    if let Some(server_name) = resp.headers().get("Server") {
+    err_resp.status_code = resp_status;
+    if let Some(server_name) = h.get("Server") {
        err_resp.server = server_name.to_str().expect("err").to_string();
    }

-    let code = resp.headers().get("x-minio-error-code");
+    let code = h.get("x-minio-error-code");
    if code.is_some() {
        err_resp.code = S3ErrorCode::Custom(code.expect("err").to_str().expect("err").into());
    }
-    let desc = resp.headers().get("x-minio-error-desc");
+    let desc = h.get("x-minio-error-desc");
    if desc.is_some() {
        err_resp.message = desc.expect("err").to_str().expect("err").trim_matches('"').to_string();
    }

    if err_resp.request_id == "" {
-        if let Some(x_amz_request_id) = resp.headers().get("x-amz-request-id") {
+        if let Some(x_amz_request_id) = h.get("x-amz-request-id") {
            err_resp.request_id = x_amz_request_id.to_str().expect("err").to_string();
        }
    }
    if err_resp.host_id == "" {
-        if let Some(x_amz_id_2) = resp.headers().get("x-amz-id-2") {
+        if let Some(x_amz_id_2) = h.get("x-amz-id-2") {
            err_resp.host_id = x_amz_id_2.to_str().expect("err").to_string();
        }
    }
    if err_resp.region == "" {
-        if let Some(x_amz_bucket_region) = resp.headers().get("x-amz-bucket-region") {
+        if let Some(x_amz_bucket_region) = h.get("x-amz-bucket-region") {
            err_resp.region = x_amz_bucket_region.to_str().expect("err").to_string();
        }
    }
@@ -19,17 +19,25 @@
 #![allow(unused_must_use)]
 #![allow(clippy::all)]

-use bytes::Bytes;
+use futures_util::ready;
 use http::HeaderMap;
-use std::io::Cursor;
+use std::io::{Cursor, Error as IoError, ErrorKind as IoErrorKind, Read};
+use std::pin::Pin;
+use std::task::{Context, Poll};
 use tokio::io::BufReader;
+use tokio_util::io::StreamReader;

 use crate::client::{
    api_error_response::err_invalid_argument,
    api_get_options::GetObjectOptions,
    transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
 };
+use futures_util::StreamExt;
+use http_body_util::BodyExt;
+use hyper::body::Body;
+use hyper::body::Bytes;
 use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+use tokio_util::io::ReaderStream;

 impl TransitionClient {
    pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<Object, std::io::Error> {
@@ -65,11 +73,19 @@ impl TransitionClient {
            )
            .await?;

-        let resp = &resp;
        let object_stat = to_object_info(bucket_name, object_name, resp.headers())?;

-        let b = resp.body().bytes().expect("err").to_vec();
-        Ok((object_stat, resp.headers().clone(), BufReader::new(Cursor::new(b))))
+        let h = resp.headers().clone();
+
+        let mut body_vec = Vec::new();
+        let mut body = resp.into_body();
+        while let Some(frame) = body.frame().await {
+            let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+            if let Some(data) = frame.data_ref() {
+                body_vec.extend_from_slice(data);
+            }
+        }
+        Ok((object_stat, h, BufReader::new(Cursor::new(body_vec))))
    }
 }

@@ -25,6 +25,7 @@ use crate::client::{
 };
 use bytes::Bytes;
 use http::{HeaderMap, HeaderValue};
+use http_body_util::BodyExt;
 use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
 use rustfs_utils::EMPTY_STRING_SHA256_HASH;
 use s3s::dto::Owner;
@@ -83,18 +84,29 @@ impl TransitionClient {
            )
            .await?;

-        if resp.status() != http::StatusCode::OK {
-            let b = resp.body().bytes().expect("err").to_vec();
-            return Err(std::io::Error::other(http_resp_to_error_response(&resp, b, bucket_name, object_name)));
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
+        let mut body_vec = Vec::new();
+        let mut body = resp.into_body();
+        while let Some(frame) = body.frame().await {
+            let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+            if let Some(data) = frame.data_ref() {
+                body_vec.extend_from_slice(data);
+            }
        }

-        let b = resp
-            .body_mut()
-            .store_all_limited(MAX_S3_CLIENT_RESPONSE_SIZE)
-            .await
-            .unwrap()
-            .to_vec();
-        let mut res = match quick_xml::de::from_str::<AccessControlPolicy>(&String::from_utf8(b).unwrap()) {
+        if resp_status != http::StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(
+                resp_status,
+                &h,
+                body_vec,
+                bucket_name,
+                object_name,
+            )));
+        }
+
+        let mut res = match quick_xml::de::from_str::<AccessControlPolicy>(&String::from_utf8(body_vec).unwrap()) {
            Ok(result) => result,
            Err(err) => {
                return Err(std::io::Error::other(err.to_string()));
@@ -18,7 +18,6 @@
 #![allow(unused_must_use)]
 #![allow(clippy::all)]

-use bytes::Bytes;
 use http::{HeaderMap, HeaderValue};
 use std::collections::HashMap;
 use time::OffsetDateTime;
@@ -28,9 +27,12 @@ use crate::client::{
    api_get_object_acl::AccessControlPolicy,
    transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
 };
+use http_body_util::BodyExt;
+use hyper::body::Body;
+use hyper::body::Bytes;
+use hyper::body::Incoming;
 use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
 use rustfs_utils::EMPTY_STRING_SHA256_HASH;
-use s3s::Body;
 use s3s::header::{X_AMZ_MAX_PARTS, X_AMZ_OBJECT_ATTRIBUTES, X_AMZ_PART_NUMBER_MARKER, X_AMZ_VERSION_ID};

 pub struct ObjectAttributesOptions {
@@ -130,19 +132,12 @@ struct ObjectAttributePart {
 }

 impl ObjectAttributes {
-    pub async fn parse_response(&mut self, resp: &mut http::Response<Body>) -> Result<(), std::io::Error> {
-        let h = resp.headers();
+    pub async fn parse_response(&mut self, h: &HeaderMap, body_vec: Vec<u8>) -> Result<(), std::io::Error> {
        let mod_time = OffsetDateTime::parse(h.get("Last-Modified").unwrap().to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(); //RFC7231Time
        self.last_modified = mod_time;
        self.version_id = h.get(X_AMZ_VERSION_ID).unwrap().to_str().unwrap().to_string();

-        let b = resp
-            .body_mut()
-            .store_all_limited(MAX_S3_CLIENT_RESPONSE_SIZE)
-            .await
-            .unwrap()
-            .to_vec();
-        let mut response = match quick_xml::de::from_str::<ObjectAttributesResponse>(&String::from_utf8(b).unwrap()) {
+        let mut response = match quick_xml::de::from_str::<ObjectAttributesResponse>(&String::from_utf8(body_vec).unwrap()) {
            Ok(result) => result,
            Err(err) => {
                return Err(std::io::Error::other(err.to_string()));
@@ -213,7 +208,8 @@ impl TransitionClient {
            )
            .await?;

-        let h = resp.headers();
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
        let has_etag = h.get("ETag").unwrap().to_str().unwrap();
        if !has_etag.is_empty() {
            return Err(std::io::Error::other(
@@ -221,14 +217,17 @@ impl TransitionClient {
            ));
        }

-        if resp.status() != http::StatusCode::OK {
-            let b = resp
-                .body_mut()
-                .store_all_limited(MAX_S3_CLIENT_RESPONSE_SIZE)
-                .await
-                .unwrap()
-                .to_vec();
-            let err_body = String::from_utf8(b).unwrap();
+        let mut body_vec = Vec::new();
+        let mut body = resp.into_body();
+        while let Some(frame) = body.frame().await {
+            let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+            if let Some(data) = frame.data_ref() {
+                body_vec.extend_from_slice(data);
+            }
+        }
+
+        if resp_status != http::StatusCode::OK {
+            let err_body = String::from_utf8(body_vec).unwrap();
            let mut er = match quick_xml::de::from_str::<AccessControlPolicy>(&err_body) {
                Ok(result) => result,
                Err(err) => {
@@ -240,7 +239,7 @@ impl TransitionClient {
        }

        let mut oa = ObjectAttributes::new();
-        oa.parse_response(&mut resp).await?;
+        oa.parse_response(&h, body_vec).await?;

        Ok(oa)
    }
@@ -27,8 +27,10 @@ use crate::client::{
    transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
 };
 use crate::store_api::BucketInfo;
-use bytes::Bytes;
 use http::{HeaderMap, StatusCode};
+use http_body_util::BodyExt;
+use hyper::body::Body;
+use hyper::body::Bytes;
 use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
 use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
 use std::collections::HashMap;
@@ -97,18 +99,30 @@ impl TransitionClient {
                },
            )
            .await?;
+
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
        if resp.status() != StatusCode::OK {
-            return Err(std::io::Error::other(http_resp_to_error_response(&resp, vec![], bucket_name, "")));
+            return Err(std::io::Error::other(http_resp_to_error_response(
+                resp_status,
+                &h,
+                vec![],
+                bucket_name,
+                "",
+            )));
        }

        //let mut list_bucket_result = ListBucketV2Result::default();
-        let b = resp
-            .body_mut()
-            .store_all_limited(MAX_S3_CLIENT_RESPONSE_SIZE)
-            .await
-            .unwrap()
-            .to_vec();
-        let mut list_bucket_result = match quick_xml::de::from_str::<ListBucketV2Result>(&String::from_utf8(b).unwrap()) {
+        let mut body_vec = Vec::new();
+        let mut body = resp.into_body();
+        while let Some(frame) = body.frame().await {
+            let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+            if let Some(data) = frame.data_ref() {
+                body_vec.extend_from_slice(data);
+            }
+        }
+        let mut list_bucket_result = match quick_xml::de::from_str::<ListBucketV2Result>(&String::from_utf8(body_vec).unwrap()) {
            Ok(result) => result,
            Err(err) => {
                return Err(std::io::Error::other(err.to_string()));
@@ -87,14 +87,14 @@ pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<
            object_size = configured_part_size as i64 * MAX_PARTS_COUNT;
        }
    } else {
-        let mut configured_part_size = configured_part_size;
-        configured_part_size = MIN_PART_SIZE as u64;
-        part_size_flt = (object_size / MAX_PARTS_COUNT) as f64;
-        part_size_flt = (part_size_flt / configured_part_size as f64) * configured_part_size as f64;
+        let min_part = MIN_PART_SIZE as f64;
+        part_size_flt = (object_size as f64 / MAX_PARTS_COUNT as f64).ceil();
+        part_size_flt = part_size_flt.max(min_part);
+        part_size_flt = (part_size_flt / min_part).ceil() * min_part;
    }

    let total_parts_count = (object_size as f64 / part_size_flt).ceil() as i64;
-    let part_size = part_size_flt.ceil() as i64;
+    let part_size = part_size_flt as i64;
    let last_part_size = object_size - (total_parts_count - 1) * part_size;
    Ok((total_parts_count, part_size, last_part_size))
 }
@@ -17,8 +17,8 @@
 #![allow(unused_must_use)]
 #![allow(clippy::all)]

-use bytes::Bytes;
 use http::{HeaderMap, HeaderName, StatusCode};
+use hyper::body::Bytes;
 use s3s::S3ErrorCode;
 use std::collections::HashMap;
 use time::OffsetDateTime;
@@ -225,10 +225,15 @@ impl TransitionClient {
        };

        let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
+
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
        //if resp.is_none() {
        if resp.status() != StatusCode::OK {
            return Err(std::io::Error::other(http_resp_to_error_response(
-                &resp,
+                resp_status,
+                &h,
                vec![],
                bucket_name,
                object_name,
@@ -287,9 +292,14 @@ impl TransitionClient {
        };

        let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
+
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
        if resp.status() != StatusCode::OK {
            return Err(std::io::Error::other(http_resp_to_error_response(
-                &resp,
+                resp_status,
+                &h,
                vec![],
                &p.bucket_name.clone(),
                &p.object_name,
@@ -370,7 +380,8 @@ impl TransitionClient {

        let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;

-        let b = resp.body().bytes().expect("err").to_vec();
+        let h = resp.headers().clone();
+
        let complete_multipart_upload_result: CompleteMultipartUploadResult = CompleteMultipartUploadResult::default();

        let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
@@ -382,7 +393,6 @@ impl TransitionClient {
            (OffsetDateTime::now_utc(), "".to_string())
        };

-        let h = resp.headers();
        Ok(UploadInfo {
            bucket: complete_multipart_upload_result.bucket,
            key: complete_multipart_upload_result.key,
@@ -479,9 +479,13 @@ impl TransitionClient {

        let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;

+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
        if resp.status() != StatusCode::OK {
            return Err(std::io::Error::other(http_resp_to_error_response(
-                &resp,
+                resp_status,
+                &h,
                vec![],
                bucket_name,
                object_name,
@@ -18,8 +18,10 @@
 #![allow(unused_must_use)]
 #![allow(clippy::all)]

-use bytes::Bytes;
 use http::{HeaderMap, HeaderValue, Method, StatusCode};
+use http_body_util::BodyExt;
+use hyper::body::Body;
+use hyper::body::Bytes;
 use rustfs_utils::HashAlgorithm;
 use s3s::S3ErrorCode;
 use s3s::dto::ReplicationStatus;
@@ -344,8 +346,15 @@ impl TransitionClient {
                )
                .await?;

-            let body_bytes: Vec<u8> = resp.body().bytes().expect("err").to_vec();
-            process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_bytes)), result_tx.clone());
+            let mut body_vec = Vec::new();
+            let mut body = resp.into_body();
+            while let Some(frame) = body.frame().await {
+                let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+                if let Some(data) = frame.data_ref() {
+                    body_vec.extend_from_slice(data);
+                }
+            }
+            process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_vec)), result_tx.clone());
        }
        Ok(())
    }
@@ -390,6 +399,10 @@ impl TransitionClient {
                },
            )
            .await?;
+
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
        //if resp.is_some() {
        if resp.status() != StatusCode::NO_CONTENT {
            let error_response: ErrorResponse;
@@ -426,7 +439,8 @@ impl TransitionClient {
                }
                _ => {
                    return Err(std::io::Error::other(http_resp_to_error_response(
-                        &resp,
+                        resp_status,
+                        &h,
                        vec![],
                        bucket_name,
                        object_name,
@@ -24,8 +24,10 @@ use crate::client::{
    api_get_options::GetObjectOptions,
    transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
 };
-use bytes::Bytes;
 use http::HeaderMap;
+use http_body_util::BodyExt;
+use hyper::body::Body;
+use hyper::body::Bytes;
 use s3s::dto::RestoreRequest;
 use std::collections::HashMap;
 use std::io::Cursor;
@@ -107,9 +109,25 @@ impl TransitionClient {
            )
            .await?;

-        let b = resp.body().bytes().expect("err").to_vec();
-        if resp.status() != http::StatusCode::ACCEPTED && resp.status() != http::StatusCode::OK {
-            return Err(std::io::Error::other(http_resp_to_error_response(&resp, b, bucket_name, "")));
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
+        let mut body_vec = Vec::new();
+        let mut body = resp.into_body();
+        while let Some(frame) = body.frame().await {
+            let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+            if let Some(data) = frame.data_ref() {
+                body_vec.extend_from_slice(data);
+            }
+        }
+        if resp_status != http::StatusCode::ACCEPTED && resp_status != http::StatusCode::OK {
+            return Err(std::io::Error::other(http_resp_to_error_response(
+                resp_status,
+                &h,
+                body_vec,
+                bucket_name,
+                "",
+            )));
        }
        Ok(())
    }
@@ -18,8 +18,10 @@
 #![allow(unused_must_use)]
 #![allow(clippy::all)]

-use bytes::Bytes;
 use http::{HeaderMap, HeaderValue};
+use http_body_util::BodyExt;
+use hyper::body::Body;
+use hyper::body::Bytes;
 use rustfs_utils::EMPTY_STRING_SHA256_HASH;
 use std::{collections::HashMap, str::FromStr};
 use tokio::io::BufReader;
@@ -66,10 +68,20 @@ impl TransitionClient {
                return Ok(false);
            }

-            let b = resp.body().bytes().expect("err").to_vec();
-            let resperr = http_resp_to_error_response(&resp, b, bucket_name, "");
+            let resp_status = resp.status();
+            let h = resp.headers().clone();

-            warn!("bucket exists, resp: {:?}, resperr: {:?}", resp, resperr);
+            let mut body_vec = Vec::new();
+            let mut body = resp.into_body();
+            while let Some(frame) = body.frame().await {
+                let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+                if let Some(data) = frame.data_ref() {
+                    body_vec.extend_from_slice(data);
+                }
+            }
+            let resperr = http_resp_to_error_response(resp_status, &h, body_vec, bucket_name, "");
+
+            warn!("bucket exists, resperr: {:?}", resperr);
            /*if to_error_response(resperr).code == "NoSuchBucket" {
                return Ok(false);
            }
@@ -108,10 +120,20 @@ impl TransitionClient {

        match resp {
            Ok(resp) => {
-                let b = resp.body().bytes().expect("get bucket versioning err").to_vec();
-                let resperr = http_resp_to_error_response(&resp, b, bucket_name, "");
+                let resp_status = resp.status();
+                let h = resp.headers().clone();

-                warn!("get bucket versioning, resp: {:?}, resperr: {:?}", resp, resperr);
+                let mut body_vec = Vec::new();
+                let mut body = resp.into_body();
+                while let Some(frame) = body.frame().await {
+                    let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+                    if let Some(data) = frame.data_ref() {
+                        body_vec.extend_from_slice(data);
+                    }
+                }
+                let resperr = http_resp_to_error_response(resp_status, &h, body_vec, bucket_name, "");
+
+                warn!("get bucket versioning, resperr: {:?}", resperr);

                Ok(VersioningConfiguration::default())
            }
@@ -25,10 +25,13 @@ use crate::client::{
    transition_api::{CreateBucketConfiguration, LocationConstraint, TransitionClient},
 };
 use http::Request;
+use http_body_util::BodyExt;
 use hyper::StatusCode;
+use hyper::body::Body;
+use hyper::body::Bytes;
+use hyper::body::Incoming;
 use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
 use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
-use s3s::Body;
 use s3s::S3ErrorCode;
 use std::collections::HashMap;

@@ -86,7 +89,7 @@ impl TransitionClient {
        Ok(location)
    }

-    fn get_bucket_location_request(&self, bucket_name: &str) -> Result<http::Request<Body>, std::io::Error> {
+    fn get_bucket_location_request(&self, bucket_name: &str) -> Result<http::Request<s3s::Body>, std::io::Error> {
        let mut url_values = HashMap::new();
        url_values.insert("location".to_string(), "".to_string());

@@ -120,7 +123,11 @@ impl TransitionClient {
            url_str = target_url.to_string();
        }

-        let Ok(mut req) = Request::builder().method(http::Method::GET).uri(url_str).body(Body::empty()) else {
+        let Ok(mut req) = Request::builder()
+            .method(http::Method::GET)
+            .uri(url_str)
+            .body(s3s::Body::empty())
+        else {
            return Err(std::io::Error::other("create request error"));
        };

@@ -172,13 +179,16 @@ impl TransitionClient {
 }

 async fn process_bucket_location_response(
-    mut resp: http::Response<Body>,
+    mut resp: http::Response<Incoming>,
    bucket_name: &str,
    tier_type: &str,
 ) -> Result<String, std::io::Error> {
    //if resp != nil {
    if resp.status() != StatusCode::OK {
-        let err_resp = http_resp_to_error_response(&resp, vec![], bucket_name, "");
+        let resp_status = resp.status();
+        let h = resp.headers().clone();
+
+        let err_resp = http_resp_to_error_response(resp_status, &h, vec![], bucket_name, "");
        match err_resp.code {
                S3ErrorCode::NotImplemented => {
                    match err_resp.server.as_str() {
@@ -208,18 +218,22 @@ async fn process_bucket_location_response(
    }
    //}

-    let b = resp
-        .body_mut()
-        .store_all_limited(MAX_S3_CLIENT_RESPONSE_SIZE)
-        .await
-        .unwrap()
-        .to_vec();
+    let mut body_vec = Vec::new();
+    let mut body = resp.into_body();
+    while let Some(frame) = body.frame().await {
+        let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+        if let Some(data) = frame.data_ref() {
+            body_vec.extend_from_slice(data);
+        }
+    }
    let mut location = "".to_string();
    if tier_type == "huaweicloud" {
-        let d = quick_xml::de::from_str::<CreateBucketConfiguration>(&String::from_utf8(b).unwrap()).unwrap();
+        let d = quick_xml::de::from_str::<CreateBucketConfiguration>(&String::from_utf8(body_vec).unwrap()).unwrap();
        location = d.location_constraint;
    } else {
-        if let Ok(LocationConstraint { field }) = quick_xml::de::from_str::<LocationConstraint>(&String::from_utf8(b).unwrap()) {
+        if let Ok(LocationConstraint { field }) =
+            quick_xml::de::from_str::<LocationConstraint>(&String::from_utf8(body_vec).unwrap())
+        {
            location = field;
        }
    }
@@ -118,38 +118,51 @@ pub fn new_getobjectreader<'a>(
    let mut is_encrypted = false;
    let is_compressed = false; //oi.is_compressed_ok();

-    let mut rs_ = None;
+    let rs_;
    if rs.is_none() && opts.part_number.is_some() && opts.part_number.unwrap() > 0 {
        rs_ = part_number_to_rangespec(oi.clone(), opts.part_number.unwrap());
+    } else {
+        rs_ = rs.clone();
    }

    let mut get_fn: ObjReaderFn;

-    let (off, length) = match rs_.unwrap().get_offset_length(oi.size) {
-        Ok(x) => x,
-        Err(err) => {
-            return Err(ErrorResponse {
-                code: S3ErrorCode::InvalidRange,
-                message: err.to_string(),
-                key: None,
-                bucket_name: None,
-                region: None,
-                request_id: None,
-                host_id: "".to_string(),
-            });
-        }
-    };
-    get_fn = Arc::new(move |input_reader: BufReader<Cursor<Vec<u8>>>, _: HeaderMap| {
-        //Box::pin({
-        let r = GetObjectReader {
-            object_info: oi.clone(),
-            stream: Box::new(input_reader),
+    if let Some(rs_) = rs_ {
+        let (off, length) = match rs_.get_offset_length(oi.size) {
+            Ok(x) => x,
+            Err(err) => {
+                return Err(ErrorResponse {
+                    code: S3ErrorCode::InvalidRange,
+                    message: err.to_string(),
+                    key: None,
+                    bucket_name: None,
+                    region: None,
+                    request_id: None,
+                    host_id: "".to_string(),
+                });
+            }
        };
-        r
-        //})
-    });
+        get_fn = Arc::new(move |input_reader: BufReader<Cursor<Vec<u8>>>, _: HeaderMap| {
+            //Box::pin({
+            let r = GetObjectReader {
+                object_info: oi.clone(),
+                stream: Box::new(input_reader),
+            };
+            r
+            //})
+        });

-    Ok((get_fn, off as i64, length as i64))
+        return Ok((get_fn, off as i64, length as i64));
+    }
+    Err(ErrorResponse {
+        code: S3ErrorCode::InvalidRange,
+        message: "Invalid range".to_string(),
+        key: Some(oi.name.clone()),
+        bucket_name: Some(oi.bucket.clone()),
+        region: Some("".to_string()),
+        request_id: None,
+        host_id: "".to_string(),
+    })
 }

 /// Convert a raw stored ETag into the strongly-typed `s3s::dto::ETag`.
@@ -20,6 +20,7 @@

 use crate::client::bucket_cache::BucketLocationCache;
 use crate::client::{
+    api_error_response::ErrorResponse,
    api_error_response::{err_invalid_argument, http_resp_to_error_response, to_error_response},
    api_get_options::GetObjectOptions,
    api_put_object::PutObjectOptions,
@@ -32,18 +33,21 @@ use crate::client::{
    credentials::{CredContext, Credentials, SignatureType, Static},
 };
 use crate::{client::checksum::ChecksumMode, store_api::GetObjectReader};
-use bytes::Bytes;
 use futures::{Future, StreamExt};
 use http::{HeaderMap, HeaderName};
 use http::{
    HeaderValue, Response, StatusCode,
    request::{Builder, Request},
 };
+use http_body::Body;
+use http_body_util::BodyExt;
+use hyper::body::Bytes;
+use hyper::body::Incoming;
 use hyper_rustls::{ConfigBuilderExt, HttpsConnector};
 use hyper_util::{client::legacy::Client, client::legacy::connect::HttpConnector, rt::TokioExecutor};
 use md5::Digest;
 use md5::Md5;
-use rand::Rng;
+use rand::{Rng, RngExt};
 use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
 use rustfs_rio::HashReader;
 use rustfs_utils::HashAlgorithm;
@@ -54,8 +58,8 @@ use rustfs_utils::{
    },
 };
 use s3s::S3ErrorCode;
+use s3s::dto::Owner;
 use s3s::dto::ReplicationStatus;
-use s3s::{Body, dto::Owner};
 use serde::{Deserialize, Serialize};
 use sha2::Sha256;
 use std::io::Cursor;
@@ -95,7 +99,7 @@ pub struct TransitionClient {
    pub creds_provider: Arc<Mutex<Credentials<Static>>>,
    pub override_signer_type: SignatureType,
    pub secure: bool,
-    pub http_client: Client<HttpsConnector<HttpConnector>, Body>,
+    pub http_client: Client<HttpsConnector<HttpConnector>, s3s::Body>,
    pub bucket_loc_cache: Arc<Mutex<BucketLocationCache>>,
    pub is_trace_enabled: Arc<Mutex<bool>>,
    pub trace_errors_only: Arc<Mutex<bool>>,
@@ -271,7 +275,7 @@ impl TransitionClient {
        todo!();
    }

-    fn dump_http(&self, req: &http::Request<Body>, resp: &http::Response<Body>) -> Result<(), std::io::Error> {
+    fn dump_http(&self, req: &http::Request<s3s::Body>, resp: &http::Response<Incoming>) -> Result<(), std::io::Error> {
        let mut resp_trace: Vec<u8>;

        //info!("{}{}", self.trace_output, "---------BEGIN-HTTP---------");
@@ -280,7 +284,7 @@ impl TransitionClient {
        Ok(())
    }

-    pub async fn doit(&self, req: http::Request<Body>) -> Result<http::Response<Body>, std::io::Error> {
+    pub async fn doit(&self, req: http::Request<s3s::Body>) -> Result<http::Response<Incoming>, std::io::Error> {
        let req_method;
        let req_uri;
        let req_headers;
@@ -295,9 +299,7 @@ impl TransitionClient {
            debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string());
            resp = http_client.request(req);
        }
-        let resp = resp
-            .await /*.map_err(Into::into)*/
-            .map(|res| res.map(Body::from));
+        let resp = resp.await;
        debug!("http_client url: {} {}", req_method, req_uri);
        debug!("http_client headers: {:?}", req_headers);
        if let Err(err) = resp {
@@ -305,7 +307,7 @@ impl TransitionClient {
            return Err(std::io::Error::other(err));
        }

-        let mut resp = resp.unwrap();
+        let resp = resp.unwrap();
        debug!("http_resp: {:?}", resp);

        //let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
@@ -314,23 +316,27 @@ impl TransitionClient {
        //if self.is_trace_enabled && !(self.trace_errors_only && resp.status() == StatusCode::OK) {
        if resp.status() != StatusCode::OK {
            //self.dump_http(&cloned_req, &resp)?;
-            let b = resp
-                .body_mut()
-                .store_all_limited(MAX_S3_CLIENT_RESPONSE_SIZE)
-                .await
-                .unwrap()
-                .to_vec();
-            warn!("err_body: {}", String::from_utf8(b).unwrap());
+            let mut body_vec = Vec::new();
+            let mut body = resp.into_body();
+            while let Some(frame) = body.frame().await {
+                let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+                if let Some(data) = frame.data_ref() {
+                    body_vec.extend_from_slice(data);
+                }
+            }
+            let body_str = String::from_utf8_lossy(&body_vec);
+            warn!("err_body: {}", body_str);
+            Err(std::io::Error::other(format!("http_client call error: {}", body_str)))
+        } else {
+            Ok(resp)
        }
-
-        Ok(resp)
    }

    pub async fn execute_method(
        &self,
        method: http::Method,
        metadata: &mut RequestMetadata,
-    ) -> Result<http::Response<Body>, std::io::Error> {
+    ) -> Result<http::Response<Incoming>, std::io::Error> {
        if self.is_offline() {
            let mut s = self.endpoint_url.to_string();
            s.push_str(" is offline.");
@@ -340,7 +346,7 @@ impl TransitionClient {
        let retryable: bool;
        //let mut body_seeker: BufferReader;
        let mut req_retry = self.max_retries;
-        let mut resp: http::Response<Body>;
+        let mut resp: http::Response<Incoming>;

        //if metadata.content_body != nil {
        //body_seeker = BufferReader::new(metadata.content_body.read_all().await?);
@@ -362,13 +368,19 @@ impl TransitionClient {
                }
            }

-            let b = resp
-                .body_mut()
-                .store_all_limited(MAX_S3_CLIENT_RESPONSE_SIZE)
-                .await
-                .unwrap()
-                .to_vec();
-            let mut err_response = http_resp_to_error_response(&resp, b.clone(), &metadata.bucket_name, &metadata.object_name);
+            let resp_status = resp.status();
+            let h = resp.headers().clone();
+
+            let mut body_vec = Vec::new();
+            let mut body = resp.into_body();
+            while let Some(frame) = body.frame().await {
+                let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+                if let Some(data) = frame.data_ref() {
+                    body_vec.extend_from_slice(data);
+                }
+            }
+            let mut err_response =
+                http_resp_to_error_response(resp_status, &h, body_vec.clone(), &metadata.bucket_name, &metadata.object_name);
            err_response.message = format!("remote tier error: {}", err_response.message);

            if self.region == "" {
@@ -404,7 +416,7 @@ impl TransitionClient {
                continue;
            }

-            if is_http_status_retryable(&resp.status()) {
+            if is_http_status_retryable(&resp_status) {
                continue;
            }

@@ -418,7 +430,7 @@ impl TransitionClient {
        &self,
        method: &http::Method,
        metadata: &mut RequestMetadata,
-    ) -> Result<http::Request<Body>, std::io::Error> {
+    ) -> Result<http::Request<s3s::Body>, std::io::Error> {
        let mut location = metadata.bucket_location.clone();
        if location == "" && metadata.bucket_name != "" {
            location = self.get_bucket_location(&metadata.bucket_name).await?;
@@ -438,7 +450,7 @@ impl TransitionClient {
        let Ok(mut req) = Request::builder()
            .method(method)
            .uri(target_url.to_string())
-            .body(Body::empty())
+            .body(s3s::Body::empty())
        else {
            return Err(std::io::Error::other("create request error"));
        };
@@ -550,10 +562,10 @@ impl TransitionClient {
        if metadata.content_length > 0 {
            match &mut metadata.content_body {
                ReaderImpl::Body(content_body) => {
-                    *req.body_mut() = Body::from(content_body.clone());
+                    *req.body_mut() = s3s::Body::from(content_body.clone());
                }
                ReaderImpl::ObjectBody(content_body) => {
-                    *req.body_mut() = Body::from(content_body.read_all().await?);
+                    *req.body_mut() = s3s::Body::from(content_body.read_all().await?);
                }
            }
        }
@@ -561,7 +573,7 @@ impl TransitionClient {
        Ok(req)
    }

-    pub fn set_user_agent(&self, req: &mut Request<Body>) {
+    pub fn set_user_agent(&self, req: &mut Request<s3s::Body>) {
        let headers = req.headers_mut();
        headers.insert("User-Agent", C_USER_AGENT.parse().expect("err"));
    }
@@ -999,25 +1011,217 @@ impl Default for UploadInfo {
    }
 }

+/// Convert HTTP headers to ObjectInfo struct
+/// This function parses various S3 response headers to construct an ObjectInfo struct
+/// containing metadata about an S3 object.
 pub fn to_object_info(bucket_name: &str, object_name: &str, h: &HeaderMap) -> Result<ObjectInfo, std::io::Error> {
-    todo!()
+    // Helper function to get header value as string
+    let get_header = |name: &str| -> String { h.get(name).and_then(|val| val.to_str().ok()).unwrap_or("").to_string() };
+
+    // Get and process the ETag
+    let etag = {
+        let etag_raw = get_header("ETag");
+        // Remove surrounding quotes if present (trimming ETag)
+        let trimmed = etag_raw.trim_start_matches('"').trim_end_matches('"');
+        Some(trimmed.to_string())
+    };
+
+    // Parse content length if it exists
+    let size = {
+        let content_length_str = get_header("Content-Length");
+        if !content_length_str.is_empty() {
+            content_length_str
+                .parse::<i64>()
+                .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "Content-Length is not an integer"))?
+        } else {
+            -1
+        }
+    };
+
+    // Parse Last-Modified time
+    let mod_time = {
+        let last_modified_str = get_header("Last-Modified");
+        if !last_modified_str.is_empty() {
+            // Parse HTTP date format (RFC 7231)
+            // Using time crate to parse HTTP dates
+            let parsed_time = OffsetDateTime::parse(&last_modified_str, &time::format_description::well_known::Rfc2822)
+                .or_else(|_| OffsetDateTime::parse(&last_modified_str, &time::format_description::well_known::Rfc3339))
+                .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "Last-Modified time format is invalid"))?;
+            Some(parsed_time)
+        } else {
+            Some(OffsetDateTime::now_utc())
+        }
+    };
+
+    // Get content type
+    let content_type = {
+        let content_type_raw = get_header("Content-Type");
+        let content_type_trimmed = content_type_raw.trim();
+        if content_type_trimmed.is_empty() {
+            Some("application/octet-stream".to_string())
+        } else {
+            Some(content_type_trimmed.to_string())
+        }
+    };
+
+    // Parse Expires time
+    let expiration = {
+        let expiry_str = get_header("Expires");
+        if !expiry_str.is_empty() {
+            OffsetDateTime::parse(&expiry_str, &time::format_description::well_known::Rfc2822)
+                .or_else(|_| OffsetDateTime::parse(&expiry_str, &time::format_description::well_known::Rfc3339))
+                .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "'Expires' is not in supported format"))?
+        } else {
+            OffsetDateTime::now_utc()
+        }
+    };
+
+    // Extract user metadata (headers prefixed with "X-Amz-Meta-")
+    let user_metadata = {
+        let mut meta = HashMap::new();
+        for (name, value) in h.iter() {
+            let header_name = name.as_str().to_lowercase();
+            if header_name.starts_with("x-amz-meta-") {
+                let key = header_name.strip_prefix("x-amz-meta-").unwrap().to_string();
+                if let Ok(value_str) = value.to_str() {
+                    meta.insert(key, value_str.to_string());
+                }
+            }
+        }
+        meta
+    };
+
+    let user_tag = {
+        let user_tag_str = get_header("X-Amz-Tagging");
+        user_tag_str
+    };
+
+    // Extract user tags count
+    let user_tag_count = {
+        let count_str = get_header("x-amz-tagging-count");
+        if !count_str.is_empty() {
+            count_str
+                .parse::<usize>()
+                .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "x-amz-tagging-count is not an integer"))?
+        } else {
+            0
+        }
+    };
+
+    // Handle restore info
+    let restore = {
+        let restore_hdr = get_header("x-amz-restore");
+        if !restore_hdr.is_empty() {
+            // Simplified restore header parsing - in real implementation, this would parse the specific format
+            // "ongoing-request=\"true\"" or "ongoing-request=\"false\", expiry-date=\"..."
+            let ongoing_restore = restore_hdr.contains("ongoing-request=\"true\"");
+            RestoreInfo {
+                ongoing_restore,
+                expiry_time: if ongoing_restore {
+                    OffsetDateTime::now_utc()
+                } else {
+                    // Try to extract expiry date from the header
+                    // This is simplified - real parsing would be more complex
+                    OffsetDateTime::now_utc()
+                },
+            }
+        } else {
+            RestoreInfo::default()
+        }
+    };
+
+    // Extract version ID
+    let version_id = {
+        let version_id_str = get_header("x-amz-version-id");
+        if !version_id_str.is_empty() {
+            Some(Uuid::parse_str(&version_id_str).unwrap_or_else(|_| Uuid::nil()))
+        } else {
+            None
+        }
+    };
+
+    // Check if it's a delete marker
+    let is_delete_marker = get_header("x-amz-delete-marker") == "true";
+
+    // Get replication status
+    let replication_status = {
+        let status_str = get_header("x-amz-replication-status");
+        ReplicationStatus::from_static(match status_str.as_str() {
+            "COMPLETE" => ReplicationStatus::COMPLETE,
+            "PENDING" => ReplicationStatus::PENDING,
+            "FAILED" => ReplicationStatus::FAILED,
+            "REPLICA" => ReplicationStatus::REPLICA,
+            _ => ReplicationStatus::PENDING,
+        })
+    };
+
+    // Extract expiration rule ID and time (simplified)
+    let (expiration_time, expiration_rule_id) = {
+        // In a real implementation, this would parse the x-amz-expiration header
+        // which typically has format: "expiry-date="Fri, 11 Dec 2020 00:00:00 GMT", rule-id="myrule""
+        let exp_header = get_header("x-amz-expiration");
+        if !exp_header.is_empty() {
+            // Simplified parsing - real implementation would be more thorough
+            (OffsetDateTime::now_utc(), exp_header) // Placeholder
+        } else {
+            (OffsetDateTime::now_utc(), "".to_string())
+        }
+    };
+
+    // Extract checksums
+    let checksum_crc32 = get_header("x-amz-checksum-crc32");
+    let checksum_crc32c = get_header("x-amz-checksum-crc32c");
+    let checksum_sha1 = get_header("x-amz-checksum-sha1");
+    let checksum_sha256 = get_header("x-amz-checksum-sha256");
+    let checksum_crc64nvme = get_header("x-amz-checksum-crc64nvme");
+    let checksum_mode = get_header("x-amz-checksum-mode");
+
+    // Build and return the ObjectInfo struct
+    Ok(ObjectInfo {
+        etag,
+        name: object_name.to_string(),
+        mod_time,
+        size,
+        content_type,
+        metadata: h.clone(),
+        user_metadata,
+        user_tags: "".to_string(), // Tags would need separate parsing
+        user_tag_count,
+        owner: Owner::default(),
+        storage_class: get_header("x-amz-storage-class"),
+        is_latest: true, // Would be determined by versioning settings
+        is_delete_marker,
+        version_id,
+        replication_status,
+        replication_ready: false, // Would be computed based on status
+        expiration: expiration_time,
+        expiration_rule_id,
+        num_versions: 1, // Would be determined by versioning
+        restore,
+        checksum_crc32,
+        checksum_crc32c,
+        checksum_sha1,
+        checksum_sha256,
+        checksum_crc64nvme,
+        checksum_mode,
+    })
 }

 type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;

 //#[derive(Clone)]
 pub struct SendRequest {
-    inner: hyper::client::conn::http1::SendRequest<Body>,
+    inner: hyper::client::conn::http1::SendRequest<s3s::Body>,
 }

-impl From<hyper::client::conn::http1::SendRequest<Body>> for SendRequest {
-    fn from(inner: hyper::client::conn::http1::SendRequest<Body>) -> Self {
+impl From<hyper::client::conn::http1::SendRequest<s3s::Body>> for SendRequest {
+    fn from(inner: hyper::client::conn::http1::SendRequest<s3s::Body>) -> Self {
        Self { inner }
    }
 }

-impl tower::Service<Request<Body>> for SendRequest {
-    type Response = Response<Body>;
+impl tower::Service<Request<s3s::Body>> for SendRequest {
+    type Response = Response<Incoming>;
    type Error = std::io::Error;
    type Future = BoxFuture<'static, Result<Self::Response, Self::Error>>;

@@ -1025,13 +1229,13 @@ impl tower::Service<Request<Body>> for SendRequest {
        self.inner.poll_ready(cx).map_err(std::io::Error::other)
    }

-    fn call(&mut self, req: Request<Body>) -> Self::Future {
+    fn call(&mut self, req: Request<s3s::Body>) -> Self::Future {
        //let req = hyper::Request::builder().uri("/").body(http_body_util::Empty::<Bytes>::new()).unwrap();
        //let req = hyper::Request::builder().uri("/").body(Body::empty()).unwrap();

        let fut = self.inner.send_request(req);

-        Box::pin(async move { fut.await.map_err(std::io::Error::other).map(|res| res.map(Body::from)) })
+        Box::pin(async move { fut.await.map_err(std::io::Error::other) })
    }
 }

@@ -620,7 +620,7 @@ pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str
    use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
    use crate::store_api::{ObjectIO, ObjectOptions};
    use http::HeaderMap;
-    use rand::Rng;
+    use rand::RngExt;
    use std::path::Path;
    use std::time::Duration;
    use tokio::time::sleep;
@@ -90,6 +90,8 @@ pub fn to_unformatted_disk_error(io_err: std::io::Error) -> std::io::Error {
    match io_err.kind() {
        std::io::ErrorKind::NotFound => DiskError::UnformattedDisk.into(),
        std::io::ErrorKind::PermissionDenied => DiskError::DiskAccessDenied.into(),
+        std::io::ErrorKind::UnexpectedEof => DiskError::UnformattedDisk.into(),
+        std::io::ErrorKind::InvalidData => DiskError::UnformattedDisk.into(),
        std::io::ErrorKind::Other => match io_err.downcast::<DiskError>() {
            Ok(err) => match err {
                DiskError::FileNotFound => DiskError::UnformattedDisk.into(),
@@ -97,11 +99,11 @@ pub fn to_unformatted_disk_error(io_err: std::io::Error) -> std::io::Error {
                DiskError::VolumeNotFound => DiskError::UnformattedDisk.into(),
                DiskError::FileAccessDenied => DiskError::DiskAccessDenied.into(),
                DiskError::DiskAccessDenied => DiskError::DiskAccessDenied.into(),
-                _ => DiskError::CorruptedBackend.into(),
+                _ => DiskError::UnformattedDisk.into(),
            },
-            Err(_err) => DiskError::CorruptedBackend.into(),
+            Err(_err) => DiskError::UnformattedDisk.into(),
        },
-        _ => DiskError::CorruptedBackend.into(),
+        _ => DiskError::UnformattedDisk.into(),
    }
 }

@@ -369,18 +371,18 @@ mod tests {
        let result = to_unformatted_disk_error(io_error);
        assert!(contains_disk_error(result, DiskError::DiskAccessDenied));

-        // Test Other error kind with other DiskError -> CorruptedBackend
+        // Test Other error kind with other DiskError -> UnformattedDisk
        let io_error = create_io_error_with_disk_error(DiskError::DiskFull);
        let result = to_unformatted_disk_error(io_error);
-        assert!(contains_disk_error(result, DiskError::CorruptedBackend));
+        assert!(contains_disk_error(result, DiskError::UnformattedDisk));
    }

    #[test]
    fn test_to_unformatted_disk_error_recursive_behavior() {
        // Test with non-Other error kind that should be handled without infinite recursion
        let result = to_unformatted_disk_error(create_io_error(ErrorKind::Interrupted));
-        // This should not cause infinite recursion and should produce CorruptedBackend
-        assert!(contains_disk_error(result, DiskError::CorruptedBackend));
+        // This should not cause infinite recursion and should produce UnformattedDisk
+        assert!(contains_disk_error(result, DiskError::UnformattedDisk));
    }

    #[test]
@@ -339,7 +339,9 @@ impl LocalDisk {

    #[tracing::instrument(level = "debug", skip(self))]
    async fn check_format_json(&self) -> Result<Metadata> {
-        let md = std::fs::metadata(&self.format_path).map_err(to_unformatted_disk_error)?;
+        let md = tokio::fs::metadata(&self.format_path)
+            .await
+            .map_err(to_unformatted_disk_error)?;
        Ok(md)
    }
    async fn make_meta_volumes(&self) -> Result<()> {
@@ -1365,36 +1367,43 @@ impl DiskAPI for LocalDisk {

    #[tracing::instrument(level = "debug", skip(self))]
    async fn get_disk_id(&self) -> Result<Option<Uuid>> {
-        let format_info = {
+        let (id, last_check, file_info) = {
            let format_info = self.format_info.read().await;
-            format_info.clone()
+            (format_info.id, format_info.last_check, format_info.file_info.clone())
        };

-        let id = format_info.id;
-
-        // if format_info.last_check_valid() {
-        //     return Ok(id);
-        // }
-
-        if format_info.file_info.is_some() && id.is_some() {
-            // check last check time
-            if let Some(last_check) = format_info.last_check
-                && last_check.unix_timestamp() + 1 < OffsetDateTime::now_utc().unix_timestamp()
-            {
-                return Ok(id);
-            }
+        // Check if we can use cached value without doing any I/O
+        // If we checked recently (within 1 second) and have valid cache, return immediately
+        if let (Some(id), Some(last_check)) = (id, last_check)
+            && last_check.unix_timestamp() + 1 >= OffsetDateTime::now_utc().unix_timestamp()
+        {
+            return Ok(Some(id));
        }

-        let file_meta = self.check_format_json().await?;
+        // Get current file metadata (async I/O)
+        let file_meta = match self.check_format_json().await {
+            Ok(meta) => meta,
+            Err(e) => {
+                // file does not exist or cannot be accessed, clear cached format info
+                if matches!(e, DiskError::UnformattedDisk | DiskError::DiskNotFound) {
+                    let mut format_info = self.format_info.write().await;
+                    format_info.id = None;
+                    format_info.file_info = None;
+                    format_info.data = Bytes::new();
+                    format_info.last_check = None;
+                }
+                return Err(e);
+            }
+        };

-        if let Some(file_info) = &format_info.file_info
-            && super::fs::same_file(&file_meta, file_info)
+        // Validate cache against current file metadata
+        if let (Some(cached_file_info), Some(id)) = (&file_info, id)
+            && super::fs::same_file(&file_meta, cached_file_info)
        {
+            // Cache is still valid, update last_check and return
            let mut format_info = self.format_info.write().await;
            format_info.last_check = Some(OffsetDateTime::now_utc());
-            drop(format_info);
-
-            return Ok(id);
+            return Ok(Some(id));
        }

        debug!("get_disk_id: read format.json");
@@ -1403,7 +1412,7 @@ impl DiskAPI for LocalDisk {

        let fm = FormatV3::try_from(b.as_slice()).map_err(|e| {
            warn!("decode format.json  err {:?}", e);
-            DiskError::CorruptedBackend
+            DiskError::UnformattedDisk
        })?;

        let (m, n) = fm.find_disk_index_by_disk_id(fm.erasure.this)?;
@@ -2421,11 +2430,12 @@ impl DiskAPI for LocalDisk {
                    return self.write_metadata("", volume, path, fi).await;
                }

-                return if fi.version_id.is_some() {
-                    Err(DiskError::FileVersionNotFound)
+                let ret_err = if fi.version_id.is_some() {
+                    DiskError::FileVersionNotFound
                } else {
-                    Err(DiskError::FileNotFound)
+                    DiskError::FileNotFound
                };
+                return Err(ret_err);
            }
        };

@@ -71,7 +71,7 @@ use rustfs_common::heal_channel::{DriveState, HealChannelPriority, HealItemType,
 use rustfs_config::MI_B;
 use rustfs_filemeta::{
    FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ObjectPartInfo,
-    RawFileInfo, ReplicationStatusType, VersionPurgeStatusType, file_info_from_raw, merge_file_meta_versions,
+    RawFileInfo, ReplicateDecision, ReplicationStatusType, VersionPurgeStatusType, file_info_from_raw, merge_file_meta_versions,
 };
 use rustfs_lock::LockClient;
 use rustfs_lock::fast_lock::types::LockResult;
@@ -122,7 +122,7 @@ const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750);

 /// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds)
 /// Defaults to 30 seconds if not set or invalid
-fn get_lock_acquire_timeout() -> Duration {
+pub fn get_lock_acquire_timeout() -> Duration {
    Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5))
 }

@@ -2251,23 +2251,42 @@ impl SetDisks {

        Ok((fi, parts_metadata, op_online_disks))
    }
-    async fn get_object_info_and_quorum(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<(ObjectInfo, usize)> {
-        let (fi, _, _) = self.get_object_fileinfo(bucket, object, opts, false).await?;
+    async fn get_object_info_and_quorum(
+        &self,
+        bucket: &str,
+        object: &str,
+        opts: &ObjectOptions,
+    ) -> (ObjectInfo, usize, Option<StorageError>) {
+        let fi = match self.get_object_fileinfo(bucket, object, opts, false).await {
+            Ok((fi, _, _)) => fi,
+            Err(e) => return (ObjectInfo::default(), 0, Some(e)),
+        };

        let write_quorum = fi.write_quorum(self.default_write_quorum());

        let oi = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
-        // TODO: replicatio
+
+        if !fi.version_purge_status().is_empty() && opts.version_id.is_some() {
+            return (
+                oi,
+                write_quorum,
+                Some(to_object_err(StorageError::MethodNotAllowed, vec![bucket, object])),
+            );
+        }

        if fi.deleted {
            return if opts.version_id.is_none() || opts.delete_marker {
-                Err(to_object_err(StorageError::FileNotFound, vec![bucket, object]))
+                (oi, write_quorum, Some(to_object_err(StorageError::FileNotFound, vec![bucket, object])))
            } else {
-                Err(to_object_err(StorageError::MethodNotAllowed, vec![bucket, object]))
+                (
+                    oi,
+                    write_quorum,
+                    Some(to_object_err(StorageError::MethodNotAllowed, vec![bucket, object])),
+                )
            };
        }

-        Ok((oi, write_quorum))
+        (oi, write_quorum, None)
    }

    #[allow(clippy::too_many_arguments)]
@@ -3583,6 +3602,10 @@ impl SetDisks {

        match oi {
            Ok(oi) => {
+                // If top level is a delete marker proceed to upload.
+                if oi.delete_marker {
+                    return None;
+                }
                if should_prevent_write(&oi, http_preconditions.if_none_match, http_preconditions.if_match) {
                    return Some(StorageError::PreconditionFailed);
                }
@@ -3672,7 +3695,11 @@ impl ObjectIO for SetDisks {
        }

        if object_info.is_remote() {
-            let gr = get_transitioned_object_reader(bucket, object, &range, &h, &object_info, opts).await?;
+            let mut opts = opts.clone();
+            if object_info.parts.len() == 1 {
+                opts.part_number = Some(1);
+            }
+            let gr = get_transitioned_object_reader(bucket, object, &range, &h, &object_info, &opts).await?;
            return Ok(gr);
        }

@@ -4453,10 +4480,19 @@ impl StorageAPI for SetDisks {
            return Ok(ObjectInfo::default());
        }

-        let (mut goi, write_quorum, gerr) = match self.get_object_info_and_quorum(bucket, object, &opts).await {
-            Ok((oi, wq)) => (oi, wq, None),
-            Err(e) => (ObjectInfo::default(), 0, Some(e)),
-        };
+        // TODO: Lifecycle
+
+        let mut version_found = true;
+        let (mut goi, write_quorum, gerr) = self.get_object_info_and_quorum(bucket, object, &opts).await;
+        if let Some(err) = &gerr
+            && goi.name.is_empty()
+        {
+            if opts.delete_marker {
+                version_found = false;
+            } else {
+                return Err(err.clone());
+            }
+        }

        let otd = ObjectToDelete {
            object_name: object.to_string(),
@@ -4467,9 +4503,16 @@ impl StorageAPI for SetDisks {
            ..Default::default()
        };

-        let version_found = if opts.delete_marker { gerr.is_none() } else { true };
-
-        let dsc = check_replicate_delete(bucket, &otd, &goi, &opts, gerr.map(|e| e.to_string())).await;
+        let dsc = if opts
+            .delete_replication
+            .as_ref()
+            .map(|v| v.replica_status == ReplicationStatusType::Replica)
+            == Some(true)
+        {
+            ReplicateDecision::default()
+        } else {
+            check_replicate_delete(bucket, &otd, &goi, &opts, gerr.map(|e| e.to_string())).await
+        };

        if dsc.replicate_any() {
            opts.set_delete_replication_state(dsc);
@@ -4493,11 +4536,11 @@ impl StorageAPI for SetDisks {
                mark_delete = false;
            }

-            if opts.version_purge_status() != VersionPurgeStatusType::Complete {
+            if opts.version_purge_status() == VersionPurgeStatusType::Complete {
                mark_delete = false;
            }

-            if version_found && (goi.version_purge_status.is_empty() || !goi.delete_marker) {
+            if version_found && (!goi.version_purge_status.is_empty() || !goi.delete_marker) {
                delete_marker = false;
            }
        }
@@ -4542,7 +4585,9 @@ impl StorageAPI for SetDisks {
                .await
                .map_err(|e| to_object_err(e, vec![bucket, object]))?;

-            return Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended));
+            let mut oi = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
+            oi.replication_decision = goi.replication_decision;
+            return Ok(oi);
        }

        let version_id = opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok());
@@ -4741,7 +4786,9 @@ impl StorageAPI for SetDisks {
            }
        }

-        fi.mod_time = opts.mod_time;
+        if opts.mod_time.is_some() {
+            fi.mod_time = opts.mod_time;
+        }
        if let Some(ref version_id) = opts.version_id {
            fi.version_id = Uuid::parse_str(version_id).ok();
        }
@@ -4794,10 +4841,18 @@ impl StorageAPI for SetDisks {
        // Normalize ETags by removing quotes before comparison (PR #592 compatibility)
        let transition_etag = rustfs_utils::path::trim_etag(&opts.transition.etag);
        let stored_etag = rustfs_utils::path::trim_etag(&get_raw_etag(&fi.metadata));
-        if opts.mod_time.expect("err").unix_timestamp() != fi.mod_time.as_ref().expect("err").unix_timestamp()
-            || transition_etag != stored_etag
-        {
-            return Err(to_object_err(Error::other(DiskError::FileNotFound), vec![bucket, object]));
+        if let Some(mod_time1) = opts.mod_time {
+            if let Some(mod_time2) = fi.mod_time.as_ref() {
+                if mod_time1.unix_timestamp() != mod_time2.unix_timestamp()
+                /*|| transition_etag != stored_etag*/
+                {
+                    return Err(to_object_err(Error::other(DiskError::FileNotFound), vec![bucket, object]));
+                }
+            } else {
+                return Err(Error::other("mod_time 2 error.".to_string()));
+            }
+        } else {
+            return Err(Error::other("mod_time 1 error.".to_string()));
        }
        if fi.transition_status == TRANSITION_COMPLETE {
            return Ok(());
@@ -4927,8 +4982,10 @@ impl StorageAPI for SetDisks {
        oi = ObjectInfo::from_file_info(&actual_fi, bucket, object, opts.versioned || opts.version_suspended);
        let ropts = put_restore_opts(bucket, object, &opts.transition.restore_request, &oi).await?;
        if oi.parts.len() == 1 {
+            let mut opts = opts.clone();
+            opts.part_number = Some(1);
            let rs: Option<HTTPRangeSpec> = None;
-            let gr = get_transitioned_object_reader(bucket, object, &rs, &HeaderMap::new(), &oi, opts).await;
+            let gr = get_transitioned_object_reader(bucket, object, &rs, &HeaderMap::new(), &oi, &opts).await;
            if let Err(err) = gr {
                return set_restore_header_fn(&mut oi, Some(to_object_err(err.into(), vec![bucket, object]))).await;
            }
@@ -66,7 +66,7 @@ use crate::{
 use futures::future::join_all;
 use http::HeaderMap;
 use lazy_static::lazy_static;
-use rand::Rng as _;
+use rand::RngExt as _;
 use rustfs_common::heal_channel::{HealItemType, HealOpts};
 use rustfs_common::{GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_HOST, GLOBAL_RUSTFS_PORT};
 use rustfs_filemeta::FileInfo;
@@ -715,7 +715,16 @@ impl ObjectInfo {
            return Ok(actual_size);
        }

-        // TODO: IsEncrypted
+        // Check if object is encrypted
+        // Encrypted objects store original size in x-rustfs-encryption-original-size metadata
+        if let Some(size_str) = self.user_defined.get("x-rustfs-encryption-original-size")
+            && !size_str.is_empty()
+        {
+            let size = size_str
+                .parse::<i64>()
+                .map_err(|e| std::io::Error::other(format!("Failed to parse encryption original size: {e}")))?;
+            return Ok(size);
+        }

        Ok(self.size)
    }
@@ -807,12 +816,11 @@ impl ObjectInfo {

        let mut restore_ongoing = false;
        let mut restore_expires = None;
-        if let Some(restore_status) = fi.metadata.get(AMZ_RESTORE).cloned() {
-            //
-            if let Ok(restore_status) = parse_restore_obj_status(&restore_status) {
-                restore_ongoing = restore_status.on_going();
-                restore_expires = restore_status.expiry();
-            }
+        if let Some(restore_status) = fi.metadata.get(AMZ_RESTORE).cloned()
+            && let Ok(restore_status) = parse_restore_obj_status(&restore_status)
+        {
+            restore_ongoing = restore_status.on_going();
+            restore_expires = restore_status.expiry();
        }

        // Convert parts from rustfs_filemeta::ObjectPartInfo to store_api::ObjectPartInfo
@@ -18,6 +18,13 @@ use regex::Regex;
 use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS};
 use std::collections::HashMap;
 use std::io::{Error, Result};
+use std::sync::LazyLock;
+
+static IP_ADDRESS_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^(\d+\.){3}\d+$").expect("valid ip address regex"));
+static STRICT_BUCKET_NAME_REGEX: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$").expect("valid strict bucket name regex"));
+static NON_STRICT_BUCKET_NAME_REGEX: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"^[A-Za-z0-9][A-Za-z0-9\.\-_:]{1,61}[A-Za-z0-9]$").expect("valid non-strict bucket name regex"));

 pub fn clean_metadata(metadata: &mut HashMap<String, String>) {
    remove_standard_storage_class(metadata);
@@ -70,15 +77,14 @@ fn check_bucket_name(bucket_name: &str, strict: bool) -> Result<()> {
        return Err(Error::other("Bucket name cannot be longer than 63 characters"));
    }

-    let ip_address_regex = Regex::new(r"^(\d+\.){3}\d+$").unwrap();
-    if ip_address_regex.is_match(bucket_name) {
+    if IP_ADDRESS_REGEX.is_match(bucket_name) {
        return Err(Error::other("Bucket name cannot be an IP address"));
    }

    let valid_bucket_name_regex = if strict {
-        Regex::new(r"^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$").unwrap()
+        &*STRICT_BUCKET_NAME_REGEX
    } else {
-        Regex::new(r"^[A-Za-z0-9][A-Za-z0-9\.\-_:]{1,61}[A-Za-z0-9]$").unwrap()
+        &*NON_STRICT_BUCKET_NAME_REGEX
    };

    if !valid_bucket_name_regex.is_match(bucket_name) {
@@ -92,3 +98,153 @@ fn check_bucket_name(bucket_name: &str, strict: bool) -> Result<()> {

    Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // --- Invalid bucket names (should return true) ---
+
+    #[test]
+    fn test_empty_string_is_invalid() {
+        assert!(is_reserved_or_invalid_bucket("", false));
+        assert!(is_reserved_or_invalid_bucket("", true));
+    }
+
+    #[test]
+    fn test_whitespace_only_is_invalid() {
+        assert!(is_reserved_or_invalid_bucket("   ", false));
+        assert!(is_reserved_or_invalid_bucket("   ", true));
+    }
+
+    #[test]
+    fn test_too_short_is_invalid() {
+        assert!(is_reserved_or_invalid_bucket("ab", false));
+        assert!(is_reserved_or_invalid_bucket("ab", true));
+    }
+
+    #[test]
+    fn test_too_long_is_invalid() {
+        let long_name = "a".repeat(64);
+        assert!(is_reserved_or_invalid_bucket(&long_name, false));
+        assert!(is_reserved_or_invalid_bucket(&long_name, true));
+    }
+
+    #[test]
+    fn test_ip_address_is_invalid() {
+        assert!(is_reserved_or_invalid_bucket("192.168.1.1", false));
+        assert!(is_reserved_or_invalid_bucket("10.0.0.1", true));
+    }
+
+    #[test]
+    fn test_double_dots_is_invalid() {
+        assert!(is_reserved_or_invalid_bucket("my..bucket", false));
+        assert!(is_reserved_or_invalid_bucket("my..bucket", true));
+    }
+
+    #[test]
+    fn test_dot_dash_is_invalid() {
+        assert!(is_reserved_or_invalid_bucket("my.-bucket", false));
+        assert!(is_reserved_or_invalid_bucket("my.-bucket", true));
+    }
+
+    #[test]
+    fn test_dash_dot_is_invalid() {
+        assert!(is_reserved_or_invalid_bucket("my-.bucket", false));
+        assert!(is_reserved_or_invalid_bucket("my-.bucket", true));
+    }
+
+    #[test]
+    fn test_reserved_name_rustfs_is_invalid() {
+        assert!(is_reserved_or_invalid_bucket("rustfs", false));
+        assert!(is_reserved_or_invalid_bucket("rustfs", true));
+    }
+
+    #[test]
+    fn test_meta_bucket_is_invalid() {
+        assert!(is_reserved_or_invalid_bucket(RUSTFS_META_BUCKET, false));
+        assert!(is_reserved_or_invalid_bucket(RUSTFS_META_BUCKET, true));
+    }
+
+    #[test]
+    fn test_starts_with_invalid_char_is_invalid() {
+        assert!(is_reserved_or_invalid_bucket("-my-bucket", false));
+        assert!(is_reserved_or_invalid_bucket(".my-bucket", false));
+    }
+
+    #[test]
+    fn test_ends_with_invalid_char_is_invalid() {
+        assert!(is_reserved_or_invalid_bucket("my-bucket-", false));
+        assert!(is_reserved_or_invalid_bucket("my-bucket.", false));
+    }
+
+    // --- Strict vs non-strict differences ---
+
+    #[test]
+    fn test_uppercase_valid_non_strict_invalid_strict() {
+        assert!(!is_reserved_or_invalid_bucket("MyBucket", false));
+        assert!(is_reserved_or_invalid_bucket("MyBucket", true));
+    }
+
+    #[test]
+    fn test_underscore_valid_non_strict_invalid_strict() {
+        assert!(!is_reserved_or_invalid_bucket("my_bucket", false));
+        assert!(is_reserved_or_invalid_bucket("my_bucket", true));
+    }
+
+    #[test]
+    fn test_colon_valid_non_strict_invalid_strict() {
+        assert!(!is_reserved_or_invalid_bucket("my:bucket", false));
+        assert!(is_reserved_or_invalid_bucket("my:bucket", true));
+    }
+
+    // --- Valid bucket names (should return false) ---
+
+    #[test]
+    fn test_simple_valid_bucket() {
+        assert!(!is_reserved_or_invalid_bucket("my-bucket", false));
+        assert!(!is_reserved_or_invalid_bucket("my-bucket", true));
+    }
+
+    #[test]
+    fn test_valid_bucket_with_dots() {
+        assert!(!is_reserved_or_invalid_bucket("my.bucket.name", false));
+        assert!(!is_reserved_or_invalid_bucket("my.bucket.name", true));
+    }
+
+    #[test]
+    fn test_numeric_bucket() {
+        assert!(!is_reserved_or_invalid_bucket("123", false));
+        assert!(!is_reserved_or_invalid_bucket("123", true));
+    }
+
+    #[test]
+    fn test_min_length_bucket() {
+        assert!(!is_reserved_or_invalid_bucket("abc", false));
+        assert!(!is_reserved_or_invalid_bucket("abc", true));
+    }
+
+    #[test]
+    fn test_max_length_bucket() {
+        let max_name = "a".repeat(63);
+        assert!(!is_reserved_or_invalid_bucket(&max_name, false));
+        assert!(!is_reserved_or_invalid_bucket(&max_name, true));
+    }
+
+    #[test]
+    fn test_trailing_slash_stripped() {
+        assert!(!is_reserved_or_invalid_bucket("my-bucket/", false));
+        assert!(!is_reserved_or_invalid_bucket("my-bucket/", true));
+    }
+
+    #[test]
+    fn test_multiple_calls_same_result() {
+        // Ensures static regex caching doesn't break across calls
+        for _ in 0..100 {
+            assert!(!is_reserved_or_invalid_bucket("my-bucket", false));
+            assert!(!is_reserved_or_invalid_bucket("my-bucket", true));
+            assert!(is_reserved_or_invalid_bucket("192.168.1.1", false));
+            assert!(is_reserved_or_invalid_bucket("", false));
+        }
+    }
+}
@@ -21,7 +21,7 @@
 use bytes::Bytes;
 use http::status::StatusCode;
 use lazy_static::lazy_static;
-use rand::Rng;
+use rand::{Rng, RngExt};
 use serde::{Deserialize, Serialize};
 use std::{
    collections::{HashMap, hash_map::Entry},
@@ -30,6 +30,8 @@ use crate::client::{
    transition_api::{Options, TransitionClient, TransitionCore},
    transition_api::{ReadCloser, ReaderImpl},
 };
+use crate::error::ErrorResponse;
+use crate::error::error_resp_to_object_err;
 use crate::tier::{
    tier_config::TierS3,
    warm_backend::{WarmBackend, WarmBackendGetOpts},
@@ -27,7 +27,14 @@ use aws_sdk_s3::Client;
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::primitives::ByteStream;

-use crate::client::transition_api::{ReadCloser, ReaderImpl};
+use crate::client::{
+    api_get_options::GetObjectOptions,
+    api_put_object::PutObjectOptions,
+    api_remove::RemoveObjectOptions,
+    transition_api::{ReadCloser, ReaderImpl},
+};
+use crate::error::ErrorResponse;
+use crate::error::error_resp_to_object_err;
 use crate::tier::{
    tier_config::TierS3,
    warm_backend::{WarmBackend, WarmBackendGetOpts},
@@ -22,6 +22,7 @@ use s3s::header::X_AMZ_RESTORE;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};
+use time::{format_description::FormatItem, macros::format_description};
 use uuid::Uuid;

 pub const ERASURE_ALGORITHM: &str = "rs-vandermonde";
@@ -37,6 +38,9 @@ pub const TIER_SKIP_FV_ID: &str = "tier-skip-fvid";

 const ERR_RESTORE_HDR_MALFORMED: &str = "x-amz-restore header malformed";

+const RFC1123: &[FormatItem<'_>] =
+    format_description!("[weekday repr:short], [day] [month repr:short] [year] [hour]:[minute]:[second] GMT");
+
 #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
 pub struct ObjectPartInfo {
    pub etag: String,
@@ -580,6 +584,7 @@ pub trait RestoreStatusOps {
    fn on_going(&self) -> bool;
    fn on_disk(&self) -> bool;
    fn to_string(&self) -> String;
+    fn to_string2(&self) -> String;
 }

 impl RestoreStatusOps for RestoreStatus {
@@ -618,6 +623,18 @@ impl RestoreStatusOps for RestoreStatus {
                .unwrap()
        )
    }
+
+    fn to_string2(&self) -> String {
+        if self.on_going() {
+            return "ongoing-request=\"true\"".to_string();
+        }
+        format!(
+            "ongoing-request=\"false\", expiry-date=\"{}\"",
+            OffsetDateTime::from(self.restore_expiry_date.clone().unwrap())
+                .format(&RFC1123)
+                .unwrap()
+        )
+    }
 }

 pub fn parse_restore_obj_status(restore_hdr: &str) -> Result<RestoreStatus> {
@@ -650,10 +667,8 @@ pub fn parse_restore_obj_status(restore_hdr: &str) -> Result<RestoreStatus> {
            if expiry_tokens[0].trim() != "expiry-date" {
                return Err(Error::other(ERR_RESTORE_HDR_MALFORMED));
            }
-            let expiry = OffsetDateTime::parse(expiry_tokens[1].trim_matches('"'), &Rfc3339).unwrap();
-            /*if err != nil {
-                return Err(Error::other(ERR_RESTORE_HDR_MALFORMED));
-            }*/
+            let expiry = OffsetDateTime::parse(expiry_tokens[1].trim_matches('"'), &Rfc3339)
+                .map_err(|_| Error::other(ERR_RESTORE_HDR_MALFORMED))?;
            return Ok(RestoreStatus {
                is_restore_in_progress: Some(false),
                restore_expiry_date: Some(Timestamp::from(expiry)),
@@ -14,8 +14,8 @@

 use crate::{
    ErasureAlgo, ErasureInfo, Error, FileInfo, FileInfoVersions, InlineData, ObjectPartInfo, RawFileInfo, ReplicationState,
-    ReplicationStatusType, Result, TIER_FV_ID, TIER_FV_MARKER, VersionPurgeStatusType, replication_statuses_map,
-    version_purge_statuses_map,
+    ReplicationStatusType, Result, TIER_FV_ID, TIER_FV_MARKER, VersionPurgeStatusType, is_restored_object_on_disk,
+    replication_statuses_map, version_purge_statuses_map,
 };
 use byteorder::ByteOrder;
 use bytes::Bytes;
@@ -462,7 +462,7 @@ impl FileMeta {
        self.versions
            .iter()
            .filter(|v| {
-                v.header.version_type == VersionType::Object && v.header.version_id != Some(vid) && v.header.user_data_dir()
+                v.header.version_type == VersionType::Object && v.header.version_id != Some(vid) && v.header.uses_data_dir()
            })
            .map(|v| FileMetaVersion::decode_data_dir_from_meta(&v.meta).unwrap_or_default())
            .filter(|v| v == data_dir)
@@ -615,7 +615,7 @@ impl FileMeta {
    }

    // delete_version deletes version, returns data_dir
-    #[tracing::instrument(skip(self))]
+    #[tracing::instrument(level = "debug", skip(self))]
    pub fn delete_version(&mut self, fi: &FileInfo) -> Result<Option<Uuid>> {
        let vid = Some(fi.version_id.unwrap_or(Uuid::nil()));

@@ -1201,10 +1201,11 @@ impl FileMeta {
            .filter(|v| {
                v.header.version_type == VersionType::Object
                    && v.header.version_id != Some(version_id)
-                    && v.header.user_data_dir()
+                    && v.header.uses_data_dir()
            })
            .filter_map(|v| FileMetaVersion::decode_data_dir_from_meta(&v.meta).ok())
-            .filter(|&dir| dir == data_dir)
+            .filter(|&dir| dir.is_none() || dir != data_dir)
+            //.filter(|&dir| dir != data_dir)
            .count()
    }

@@ -1588,7 +1589,7 @@ impl FileMetaVersionHeader {
        false
    }

-    pub fn user_data_dir(&self) -> bool {
+    pub fn uses_data_dir(&self) -> bool {
        self.flags & Flags::UsesDataDir as u8 != 0
    }

@@ -1961,7 +1962,15 @@ impl MetaObject {
    }

    pub fn uses_data_dir(&self) -> bool {
-        !self.inlinedata()
+        if let Some(status) = self
+            .meta_sys
+            .get(&format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}"))
+            && *status == TRANSITION_COMPLETE.as_bytes().to_vec()
+        {
+            return false;
+        }
+
+        is_restored_object_on_disk(&self.meta_user)
    }

    pub fn inlinedata(&self) -> bool {
@@ -281,7 +281,7 @@ impl ReplicationState {
                    return repl_status;
                }
            }
-        } else if self.replica_status != ReplicationStatusType::default() {
+        } else if !self.replica_status.is_empty() {
            return self.replica_status.clone();
        }

@@ -49,3 +49,4 @@ serial_test = { workspace = true }
 tracing-subscriber = { workspace = true }
 tempfile = { workspace = true }
 walkdir = { workspace = true }
+http = { workspace = true }
@@ -492,11 +492,19 @@ impl HealManager {
                        for (_, disk_opt) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() {
                            if let Some(disk) = disk_opt {
                                // detect unformatted disk via get_disk_id()
-                                if let Err(err) = disk.get_disk_id().await
-                                    && err == DiskError::UnformattedDisk {
+                                match disk.get_disk_id().await {
+                                    Err(DiskError::UnformattedDisk) => {
+                                        info!("start_auto_disk_scanner: Detected unformatted disk: {}", disk.endpoint());
                                        endpoints.push(disk.endpoint());
-                                        continue;
                                    }
+                                    Err(e) => {
+                                        // Log other errors for debugging
+                                        tracing::warn!("start_auto_disk_scanner: Disk {} check failed: {:?}", disk.endpoint(), e);
+                                    }
+                                    Ok(_) => {
+                                        // Disk is formatted, no action needed
+                                    }
+                                }
                            }
                        }

@@ -13,6 +13,7 @@
 // limitations under the License.

 use crate::{Error, Result};
+use rustfs_ecstore::disk::error::DiskError;
 use rustfs_ecstore::disk::{BUCKET_META_PREFIX, DiskAPI, DiskStore, RUSTFS_META_BUCKET};
 use serde::{Deserialize, Serialize};
 use std::path::Path;
@@ -180,7 +181,9 @@ impl ResumeManager {
        };

        // save initial state
-        manager.save_state().await?;
+        if let Err(e) = manager.save_state().await {
+            warn!("Failed to save initial resume state: {}", e);
+        }
        Ok(manager)
    }

@@ -297,12 +300,15 @@ impl ResumeManager {
        let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", state.task_id, RESUME_STATE_FILE));

        let path_str = path_to_str(&file_path)?;
-        self.disk
-            .write_all(RUSTFS_META_BUCKET, path_str, state_data.into())
-            .await
-            .map_err(|e| Error::TaskExecutionFailed {
+        if let Err(e) = self.disk.write_all(RUSTFS_META_BUCKET, path_str, state_data.into()).await {
+            if matches!(e, DiskError::UnformattedDisk) {
+                warn!("Cannot save resume state: unformatted disk");
+                return Ok(());
+            }
+            return Err(Error::TaskExecutionFailed {
                message: format!("Failed to save resume state: {e}"),
-            })?;
+            });
+        }

        debug!("Saved resume state for task: {}", state.task_id);
        Ok(())
@@ -395,7 +401,9 @@ impl CheckpointManager {
        };

        // save initial checkpoint
-        manager.save_checkpoint().await?;
+        if let Err(e) = manager.save_checkpoint().await {
+            warn!("Failed to save initial checkpoint: {}", e);
+        }
        Ok(manager)
    }

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+use http::HeaderMap;
 use rustfs_common::heal_channel::{HealOpts, HealScanMode};
 use rustfs_ecstore::{
    disk::endpoint::Endpoint,
@@ -229,6 +230,19 @@ mod serial_tests {
        // ─── 2️⃣ verify each part file is restored ───────
        assert!(target_part.exists());

+        // ─── 3️⃣ verify object data integrity by actually reading it ───────
+        let mut reader = ecstore
+            .get_object_reader(bucket_name, object_name, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("Failed to get object reader after heal");
+
+        let mut downloaded_data = Vec::new();
+        tokio::io::copy(&mut reader, &mut downloaded_data)
+            .await
+            .expect("Failed to read healed object data");
+
+        assert_eq!(downloaded_data, test_data, "Healed object data does not match original");
+
        info!("Heal object basic test passed");
    }

@@ -312,18 +326,27 @@ mod serial_tests {
        assert!(!format_path.exists(), "format.json still exists after deletion");
        println!("✅ Deleted format.json on disk: {format_path:?}");

-        // Create heal manager with faster interval
-        let cfg = HealConfig {
-            heal_interval: Duration::from_secs(2),
-            ..Default::default()
-        };
-        let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg));
-        heal_manager.start().await.unwrap();
+        let (_result, error) = heal_storage.heal_format(false).await.expect("Failed to heal format");
+        assert!(error.is_none(), "Heal format returned error: {error:?}");

-        // Wait for task completion
-        tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+        // ─── 2️⃣ wait for format.json to be restored with polling + timeout ───────
+        // The minimal scanner interval is clamped to 10s in manager.rs, so we set timeout to 20s
+        let timeout_duration = Duration::from_secs(20);
+        let poll_interval = Duration::from_millis(200);

-        // ─── 2️⃣ verify format.json is restored ───────
+        let result = tokio::time::timeout(timeout_duration, async {
+            loop {
+                if format_path.exists() {
+                    break;
+                }
+                tokio::time::sleep(poll_interval).await;
+            }
+        })
+        .await;
+
+        assert!(result.is_ok(), "format.json was not restored within timeout period");
+
+        // ─── 3️⃣ verify format.json is restored ───────
        assert!(format_path.exists(), "format.json does not exist on disk after heal");

        info!("Heal format basic test passed");
@@ -366,15 +389,49 @@ mod serial_tests {
        let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg));
        heal_manager.start().await.unwrap();

-        // Wait for task completion
-        tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+        // ─── 2️⃣ wait for format.json and part file to be restored with polling + timeout ───────
+        // The minimal scanner interval is clamped to 10s in manager.rs, so we set timeout to 20s
+        let timeout_duration = Duration::from_secs(20);
+        let poll_interval = Duration::from_millis(200);

-        // ─── 2️⃣ verify format.json is restored ───────
+        let result = tokio::time::timeout(timeout_duration, async {
+            loop {
+                if format_path.exists() && target_part.exists() {
+                    break;
+                }
+                tokio::time::sleep(poll_interval).await;
+            }
+        })
+        .await;
+
+        assert!(result.is_ok(), "format.json or part file was not restored within timeout period");
+
+        // ─── 3️⃣ verify format.json is restored ───────
        assert!(format_path.exists(), "format.json does not exist on disk after heal");
-        // ─── 3 verify each part file is restored ───────
+        // ─── 4️⃣ verify each part file is restored ───────
        assert!(target_part.exists());

-        info!("Heal format basic test passed");
+        // Verify object metadata is accessible
+        let obj_info = ecstore
+            .get_object_info(bucket_name, object_name, &ObjectOptions::default())
+            .await
+            .expect("Expected object to be readable after heal");
+        assert_eq!(obj_info.size as usize, test_data.len());
+
+        // Actually read the object data to verify integrity
+        let mut reader = ecstore
+            .get_object_reader(bucket_name, object_name, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("Failed to get object reader");
+
+        let mut downloaded_data = Vec::new();
+        tokio::io::copy(&mut reader, &mut downloaded_data)
+            .await
+            .expect("Failed to read object data");
+
+        assert_eq!(downloaded_data, test_data, "Healed object data does not match original");
+
+        info!("Heal format with data test passed");
    }

    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
@@ -130,14 +130,8 @@ impl ObjectStore {
    }

    fn decrypt_data(data: &[u8]) -> Result<Vec<u8>> {
-        let cred = get_global_action_cred().unwrap_or_default();
-        match rustfs_crypto::decrypt_data(cred.secret_key.as_bytes(), data) {
-            Ok(decrypted) => Ok(decrypted),
-            Err(_) => {
-                warn!("Failed to decrypt IAM config data, treating as unencrypted");
-                Ok(data.to_vec())
-            }
-        }
+        let de = rustfs_crypto::decrypt_data(get_global_action_cred().unwrap_or_default().secret_key.as_bytes(), data)?;
+        Ok(de)
    }

    fn encrypt_data(data: &[u8]) -> Result<Vec<u8>> {
@@ -744,7 +744,7 @@ impl<T: Store> IamSys<T> {
    }

    pub async fn is_allowed_sts(&self, args: &Args<'_>, parent_user: &str) -> bool {
-        let is_owner = parent_user == get_global_action_cred().unwrap().access_key;
+        let is_owner = matches!(get_global_action_cred(), Some(cred) if cred.access_key == parent_user);
        let role_arn = args.get_role_arn();
        let policies = {
            if is_owner {
@@ -794,7 +794,7 @@ impl<T: Store> IamSys<T> {
            return false;
        }

-        let is_owner = parent_user == get_global_action_cred().unwrap().access_key;
+        let is_owner = matches!(get_global_action_cred(), Some(cred) if cred.access_key == parent_user);

        let role_arn = args.get_role_arn();

@@ -13,7 +13,7 @@
 // limitations under the License.

 use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header};
-use rand::{Rng, RngCore};
+use rand::{Rng, RngExt};
 use serde::{Serialize, de::DeserializeOwned};
 use std::io::{Error, Result};

@@ -0,0 +1 @@
+examples/local_data/*
@@ -55,6 +55,7 @@ moka = { workspace = true, features = ["future"] }

 # Additional dependencies
 md5 = { workspace = true }
+arc-swap = { workspace = true }

 # HTTP client for Vault
 reqwest = { workspace = true }
@@ -0,0 +1,251 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! KMS Demo - Comprehensive example demonstrating RustFS KMS capabilities
+//!
+//! This example demonstrates:
+//! - Initializing and configuring KMS service
+//! - Creating master keys
+//! - Generating data encryption keys
+//! - Encrypting and decrypting data using high-level APIs
+//! - Key management operations
+//! - Cache statistics
+//!
+//! Run with: `cargo run --example demo1`
+
+use rustfs_kms::{
+    CreateKeyRequest, DescribeKeyRequest, EncryptionAlgorithm, GenerateDataKeyRequest, KeySpec, KeyUsage, KmsConfig,
+    ListKeysRequest, init_global_kms_service_manager,
+};
+use std::collections::HashMap;
+use std::fs;
+use std::io::Cursor;
+use tokio::io::AsyncReadExt;
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // Note: Tracing is optional - if tracing-subscriber is not available,
+    // the example will still work but with less detailed logging
+
+    println!("=== RustFS KMS Demo ===\n");
+
+    // Step 1: Initialize global KMS service manager
+    println!("1. Initializing KMS service manager...");
+    let service_manager = init_global_kms_service_manager();
+    println!("   ✓ Service manager initialized\n");
+
+    // Step 2: Create a temporary directory for local backend
+    println!("2. Setting up local backend...");
+    if fs::metadata("examples/local_data").is_err() {
+        fs::create_dir_all("examples/local_data")?;
+    }
+    let data_dir = std::path::PathBuf::from("examples/local_data");
+    println!("   ✓ Using data directory: {}\n", data_dir.display());
+
+    // Step 3: Configure KMS with local backend
+    println!("3. Configuring KMS with local backend...");
+    let config = KmsConfig::local(data_dir)
+        .with_default_key("demo-key-default-1".to_string())
+        .with_cache(true);
+    service_manager.configure(config).await?;
+    println!("   ✓ KMS configured\n");
+
+    // Step 4: Start the KMS service
+    println!("4. Starting KMS service...");
+    service_manager.start().await?;
+    println!("   ✓ KMS service started\n");
+
+    // Step 5: Get the encryption service
+    println!("5. Getting encryption service...");
+    let encryption_service = rustfs_kms::get_global_encryption_service()
+        .await
+        .ok_or("Encryption service not available")?;
+    println!("   ✓ Encryption service obtained\n");
+
+    // Step 6: Create a master key
+    println!("6. Creating a master key...");
+    let create_request = CreateKeyRequest {
+        key_name: Some("demo-key-master-1".to_string()),
+        key_usage: KeyUsage::EncryptDecrypt,
+        description: Some("Demo master key for encryption".to_string()),
+        policy: None,
+        tags: {
+            let mut tags = HashMap::new();
+            tags.insert("environment".to_string(), "demo".to_string());
+            tags.insert("purpose".to_string(), "testing".to_string());
+            tags
+        },
+        origin: Some("demo1.rs".to_string()),
+    };
+
+    let create_response = encryption_service.create_key(create_request).await?;
+    println!("   ✓ Master key created:");
+    println!("     - Key ID: {}", create_response.key_id);
+    println!("     - Key State: {:?}", create_response.key_metadata.key_state);
+    println!("     - Key Usage: {:?}", create_response.key_metadata.key_usage);
+    println!("     - Created: {}\n", create_response.key_metadata.creation_date);
+
+    let master_key_id = create_response.key_id.clone();
+
+    // Step 7: Describe the key
+    println!("7. Describing the master key...");
+    let describe_request = DescribeKeyRequest {
+        key_id: master_key_id.clone(),
+    };
+    let describe_response = encryption_service.describe_key(describe_request).await?;
+    let metadata = describe_response.key_metadata;
+    println!("   ✓ Key details:");
+    println!("     - Key ID: {}", metadata.key_id);
+    println!("     - Description: {:?}", metadata.description);
+    println!("     - Key Usage: {:?}", metadata.key_usage);
+    println!("     - Key State: {:?}", metadata.key_state);
+    println!("     - Tags: {:?}\n", metadata.tags);
+
+    // Step 8: Generate a data encryption key (OPTIONAL - for demonstration only)
+    // NOTE: This step is OPTIONAL and only for educational purposes!
+    // In real usage, you can skip this step and go directly to Step 9.
+    // encrypt_object() will automatically generate a data key internally.
+    println!("8. [OPTIONAL] Generating a data encryption key (for demonstration)...");
+    println!("   ⚠️  This step is OPTIONAL - only for understanding the two-layer key architecture:");
+    println!("   - Master Key (CMK): Used to encrypt/decrypt data keys");
+    println!("   - Data Key (DEK): Used to encrypt/decrypt actual data");
+    println!("   In production, you can skip this and use encrypt_object() directly!\n");
+
+    let data_key_request = GenerateDataKeyRequest {
+        key_id: master_key_id.clone(),
+        key_spec: KeySpec::Aes256,
+        encryption_context: {
+            let mut context = HashMap::new();
+            context.insert("bucket".to_string(), "demo-bucket".to_string());
+            context.insert("object_key".to_string(), "demo-object.txt".to_string());
+            context
+        },
+    };
+
+    let data_key_response = encryption_service.generate_data_key(data_key_request).await?;
+    println!("   ✓ Data key generated (for demonstration):");
+    println!("     - Master Key ID: {}", data_key_response.key_id);
+    println!("     - Data Key (plaintext) length: {} bytes", data_key_response.plaintext_key.len());
+    println!(
+        "     - Encrypted Data Key (ciphertext blob) length: {} bytes",
+        data_key_response.ciphertext_blob.len()
+    );
+    println!("     - Note: This data key is NOT used in Step 9 - encrypt_object() generates its own!\n");
+
+    // Step 9: Encrypt some data using high-level API
+    // This is the RECOMMENDED way to encrypt data - everything is handled automatically!
+    println!("9. Encrypting data using object encryption service (RECOMMENDED)...");
+    println!("   ✅ This is all you need! encrypt_object() handles everything:");
+    println!("   1. Validates/creates the master key (if needed)");
+    println!("   2. Generates a NEW data key using the master key (independent of Step 8)");
+    println!("   3. Uses the data key to encrypt the actual data");
+    println!("   4. Stores the encrypted data key (ciphertext blob) in metadata");
+    println!("   You only need to provide the master_key_id - everything else is handled!\n");
+
+    let plaintext = b"Hello, RustFS KMS! This is a test message for encryption.";
+    println!("   Plaintext: {}", String::from_utf8_lossy(plaintext));
+
+    let reader = Cursor::new(plaintext);
+    // Just provide the master_key_id - encrypt_object() handles everything internally!
+    let encryption_result = encryption_service
+        .encrypt_object(
+            "demo-bucket",
+            "demo-object.txt",
+            reader,
+            &EncryptionAlgorithm::Aes256,
+            Some(&master_key_id), // Only need to provide master key ID
+            None,
+        )
+        .await?;
+
+    println!("   ✓ Data encrypted:");
+    println!("     - Encrypted data length: {} bytes", encryption_result.ciphertext.len());
+    println!("     - Algorithm: {}", encryption_result.metadata.algorithm);
+    println!(
+        "     - Master Key ID: {} (used to encrypt the data key)",
+        encryption_result.metadata.key_id
+    );
+    println!(
+        "     - Encrypted Data Key length: {} bytes (stored in metadata)",
+        encryption_result.metadata.encrypted_data_key.len()
+    );
+    println!("     - Original size: {} bytes\n", encryption_result.metadata.original_size);
+
+    // Step 10: Decrypt the data using high-level API
+    println!("10. Decrypting data...");
+    println!("   Note: decrypt_object() has the ENTIRE decryption flow built-in:");
+    println!("   1. Extracts the encrypted data key from metadata");
+    println!("   2. Uses master key to decrypt the data key");
+    println!("   3. Uses the decrypted data key to decrypt the actual data");
+    println!("   You only need to provide the encrypted data and metadata!\n");
+
+    let mut decrypted_reader = encryption_service
+        .decrypt_object(
+            "demo-bucket",
+            "demo-object.txt",
+            encryption_result.ciphertext.clone(),
+            &encryption_result.metadata, // Contains everything needed for decryption
+            None,
+        )
+        .await?;
+
+    let mut decrypted_data = Vec::new();
+    decrypted_reader.read_to_end(&mut decrypted_data).await?;
+
+    println!("   ✓ Data decrypted:");
+    println!("     - Decrypted text: {}\n", String::from_utf8_lossy(&decrypted_data));
+
+    // Verify decryption
+    assert_eq!(plaintext, decrypted_data.as_slice());
+    println!("   ✓ Decryption verified: plaintext matches original\n");
+
+    // Step 11: List all keys
+    println!("11. Listing all keys...");
+    let list_request = ListKeysRequest {
+        limit: Some(10),
+        marker: None,
+        usage_filter: None,
+        status_filter: None,
+    };
+    let list_response = encryption_service.list_keys(list_request).await?;
+    println!("   ✓ Keys found: {}", list_response.keys.len());
+    for (idx, key_info) in list_response.keys.iter().enumerate() {
+        println!("     {}. {} ({:?})", idx + 1, key_info.key_id, key_info.status);
+    }
+    println!();
+
+    // Step 12: Check cache statistics
+    println!("12. Checking cache statistics...");
+    if let Some((hits, misses)) = encryption_service.cache_stats().await {
+        println!("   ✓ Cache statistics:");
+        println!("     - Cache hits: {}", hits);
+        println!("     - Cache misses: {}\n", misses);
+    } else {
+        println!("   - Cache is disabled\n");
+    }
+
+    // Step 13: Health check
+    println!("13. Performing health check...");
+    let is_healthy = encryption_service.health_check().await?;
+    println!("   ✓ KMS backend is healthy: {}\n", is_healthy);
+
+    // Step 14: Stop the service
+    println!("14. Stopping KMS service...");
+    service_manager.stop().await?;
+    println!("   ✓ KMS service stopped\n");
+
+    println!("=== Demo completed successfully! ===");
+
+    Ok(())
+}
@@ -0,0 +1,292 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! KMS Demo 2 - Comprehensive example demonstrating RustFS KMS with Vault backend
+//!
+//! This example demonstrates:
+//! - Initializing and configuring KMS service with Vault backend
+//! - Creating master keys stored in Vault
+//! - Generating data encryption keys
+//! - Encrypting and decrypting data using high-level APIs
+//! - Key management operations with Vault
+//! - Cache statistics
+//!
+//! Prerequisites:
+//! - Vault server running at http://127.0.0.1:8200 (or set RUSTFS_KMS_VAULT_ADDRESS)
+//! - Vault token (set RUSTFS_KMS_VAULT_TOKEN environment variable, or use default "dev-token" for dev mode)
+//!
+//! Run with: `cargo run --example demo2`
+//! Or with custom Vault settings:
+//!   RUSTFS_KMS_VAULT_ADDRESS=http://127.0.0.1:8200 RUSTFS_KMS_VAULT_TOKEN=your-token cargo run --example demo2
+
+use rustfs_kms::{
+    CreateKeyRequest, DescribeKeyRequest, EncryptionAlgorithm, GenerateDataKeyRequest, KeySpec, KeyUsage, KmsConfig, KmsError,
+    ListKeysRequest, init_global_kms_service_manager,
+};
+use std::collections::HashMap;
+use std::io::Cursor;
+use tokio::io::AsyncReadExt;
+use url::Url;
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // Note: Tracing is optional - if tracing-subscriber is not available,
+    // the example will still work but with less detailed logging
+
+    println!("=== RustFS KMS Demo 2 (Vault Backend) ===\n");
+
+    // Step 1: Initialize global KMS service manager
+    println!("1. Initializing KMS service manager...");
+    let service_manager = init_global_kms_service_manager();
+    println!("   ✓ Service manager initialized\n");
+
+    // Step 2: Get Vault configuration from environment or use defaults
+    println!("2. Configuring Vault backend...");
+    let vault_address = std::env::var("RUSTFS_KMS_VAULT_ADDRESS").unwrap_or_else(|_| "http://127.0.0.1:8200".to_string());
+    let vault_token = std::env::var("RUSTFS_KMS_VAULT_TOKEN").unwrap_or_else(|_| {
+        println!("   ⚠️  No RUSTFS_KMS_VAULT_TOKEN found, using default 'dev-token'");
+        println!("      For production, set RUSTFS_KMS_VAULT_TOKEN environment variable");
+        "dev-token".to_string()
+    });
+
+    let vault_url = Url::parse(&vault_address).map_err(|e| format!("Invalid Vault address '{}': {}", vault_address, e))?;
+
+    println!("   ✓ Vault address: {}", vault_address);
+    println!("   ✓ Using token authentication\n");
+
+    // Step 3: Configure KMS with Vault backend
+    println!("3. Configuring KMS with Vault backend...");
+    let config = KmsConfig::vault(vault_url, vault_token)
+        .with_default_key("demo-key-master-1".to_string())
+        .with_cache(true);
+    service_manager.configure(config).await?;
+    println!("   ✓ KMS configured with Vault backend\n");
+
+    // Step 4: Start the KMS service
+    println!("4. Starting KMS service...");
+    service_manager.start().await?;
+    println!("   ✓ KMS service started\n");
+
+    // Step 5: Get the encryption service
+    println!("5. Getting encryption service...");
+    let encryption_service = rustfs_kms::get_global_encryption_service()
+        .await
+        .ok_or("Encryption service not available")?;
+    println!("   ✓ Encryption service obtained\n");
+
+    // Step 6: Create a master key (stored in Vault) or use existing one
+    println!("6. Checking for existing master key in Vault...");
+    let master_key_id = "demo-key-master-1".to_string();
+    let describe_request = DescribeKeyRequest {
+        key_id: master_key_id.clone(),
+    };
+
+    let master_key_id = match encryption_service.describe_key(describe_request).await {
+        Ok(describe_response) => {
+            // Key already exists, use it
+            println!("   ✓ Master key already exists in Vault:");
+            println!("     - Key ID: {}", describe_response.key_metadata.key_id);
+            println!("     - Key State: {:?}", describe_response.key_metadata.key_state);
+            println!("     - Key Usage: {:?}", describe_response.key_metadata.key_usage);
+            println!("     - Created: {}\n", describe_response.key_metadata.creation_date);
+            describe_response.key_metadata.key_id
+        }
+        Err(KmsError::KeyNotFound { .. }) => {
+            // Key doesn't exist, create it
+            println!("   Key not found, creating new master key in Vault...");
+            let create_request = CreateKeyRequest {
+                key_name: Some(master_key_id.clone()),
+                key_usage: KeyUsage::EncryptDecrypt,
+                description: Some("Demo master key for encryption (stored in Vault)".to_string()),
+                policy: None,
+                tags: {
+                    let mut tags = HashMap::new();
+                    tags.insert("environment".to_string(), "demo".to_string());
+                    tags.insert("purpose".to_string(), "testing".to_string());
+                    tags.insert("backend".to_string(), "vault".to_string());
+                    tags
+                },
+                origin: Some("demo2.rs".to_string()),
+            };
+
+            let create_response = encryption_service.create_key(create_request).await?;
+            println!("   ✓ Master key created in Vault:");
+            println!("     - Key ID: {}", create_response.key_id);
+            println!("     - Key State: {:?}", create_response.key_metadata.key_state);
+            println!("     - Key Usage: {:?}", create_response.key_metadata.key_usage);
+            println!("     - Created: {}\n", create_response.key_metadata.creation_date);
+            create_response.key_id
+        }
+        Err(e) => {
+            // Other error, return it
+            return Err(Box::new(e) as Box<dyn std::error::Error>);
+        }
+    };
+
+    // Step 7: Describe the key (retrieved from Vault)
+    println!("7. Describing the master key (from Vault)...");
+    let describe_request = DescribeKeyRequest {
+        key_id: master_key_id.clone(),
+    };
+    let describe_response = encryption_service.describe_key(describe_request).await?;
+    let metadata = describe_response.key_metadata;
+    println!("   ✓ Key details (from Vault):");
+    println!("     - Key ID: {}", metadata.key_id);
+    println!("     - Description: {:?}", metadata.description);
+    println!("     - Key Usage: {:?}", metadata.key_usage);
+    println!("     - Key State: {:?}", metadata.key_state);
+    println!("     - Tags: {:?}\n", metadata.tags);
+
+    // Step 8: Generate a data encryption key (OPTIONAL - for demonstration only)
+    // NOTE: This step is OPTIONAL and only for educational purposes!
+    // In real usage, you can skip this step and go directly to Step 9.
+    // encrypt_object() will automatically generate a data key internally.
+    println!("8. [OPTIONAL] Generating a data encryption key (for demonstration)...");
+    println!("   ⚠️  This step is OPTIONAL - only for understanding the two-layer key architecture:");
+    println!("   - Master Key (CMK): Stored in Vault, used to encrypt/decrypt data keys");
+    println!("   - Data Key (DEK): Generated per object, encrypted by master key");
+    println!("   In production, you can skip this and use encrypt_object() directly!\n");
+
+    let data_key_request = GenerateDataKeyRequest {
+        key_id: master_key_id.clone(),
+        key_spec: KeySpec::Aes256,
+        encryption_context: {
+            let mut context = HashMap::new();
+            context.insert("bucket".to_string(), "demo-bucket".to_string());
+            context.insert("object_key".to_string(), "demo-object.txt".to_string());
+            context
+        },
+    };
+
+    let data_key_response = encryption_service.generate_data_key(data_key_request).await?;
+    println!("   ✓ Data key generated (for demonstration):");
+    println!("     - Master Key ID: {}", data_key_response.key_id);
+    println!("     - Data Key (plaintext) length: {} bytes", data_key_response.plaintext_key.len());
+    println!(
+        "     - Encrypted Data Key (ciphertext blob) length: {} bytes",
+        data_key_response.ciphertext_blob.len()
+    );
+    println!("     - Note: This data key is NOT used in Step 9 - encrypt_object() generates its own!\n");
+
+    // Step 9: Encrypt some data using high-level API
+    // This is the RECOMMENDED way to encrypt data - everything is handled automatically!
+    println!("9. Encrypting data using object encryption service (RECOMMENDED)...");
+    println!("   ✅ This is all you need! encrypt_object() handles everything:");
+    println!("   1. Validates/creates the master key in Vault (if needed)");
+    println!("   2. Generates a NEW data key using the master key from Vault (independent of Step 8)");
+    println!("   3. Uses the data key to encrypt the actual data");
+    println!("   4. Stores the encrypted data key (ciphertext blob) in metadata");
+    println!("   You only need to provide the master_key_id - everything else is handled!\n");
+
+    let plaintext = b"Hello, RustFS KMS with Vault! This is a test message for encryption.";
+    println!("   Plaintext: {}", String::from_utf8_lossy(plaintext));
+
+    let reader = Cursor::new(plaintext);
+    // Just provide the master_key_id - encrypt_object() handles everything internally!
+    let encryption_result = encryption_service
+        .encrypt_object(
+            "demo-bucket",
+            "demo-object.txt",
+            reader,
+            &EncryptionAlgorithm::Aes256,
+            Some(&master_key_id), // Only need to provide master key ID
+            None,
+        )
+        .await?;
+
+    println!("   ✓ Data encrypted:");
+    println!("     - Encrypted data length: {} bytes", encryption_result.ciphertext.len());
+    println!("     - Algorithm: {}", encryption_result.metadata.algorithm);
+    println!(
+        "     - Master Key ID: {} (stored in Vault, used to encrypt the data key)",
+        encryption_result.metadata.key_id
+    );
+    println!(
+        "     - Encrypted Data Key length: {} bytes (stored in metadata)",
+        encryption_result.metadata.encrypted_data_key.len()
+    );
+    println!("     - Original size: {} bytes\n", encryption_result.metadata.original_size);
+
+    // Step 10: Decrypt the data using high-level API
+    println!("10. Decrypting data...");
+    println!("   Note: decrypt_object() has the ENTIRE decryption flow built-in:");
+    println!("   1. Extracts the encrypted data key from metadata");
+    println!("   2. Uses master key from Vault to decrypt the data key");
+    println!("   3. Uses the decrypted data key to decrypt the actual data");
+    println!("   You only need to provide the encrypted data and metadata!\n");
+
+    let mut decrypted_reader = encryption_service
+        .decrypt_object(
+            "demo-bucket",
+            "demo-object.txt",
+            encryption_result.ciphertext.clone(),
+            &encryption_result.metadata, // Contains everything needed for decryption
+            None,
+        )
+        .await?;
+
+    let mut decrypted_data = Vec::new();
+    decrypted_reader.read_to_end(&mut decrypted_data).await?;
+
+    println!("   ✓ Data decrypted:");
+    println!("     - Decrypted text: {}\n", String::from_utf8_lossy(&decrypted_data));
+
+    // Verify decryption
+    assert_eq!(plaintext, decrypted_data.as_slice());
+    println!("   ✓ Decryption verified: plaintext matches original\n");
+
+    // Step 11: List all keys (from Vault)
+    println!("11. Listing all keys (from Vault)...");
+    let list_request = ListKeysRequest {
+        limit: Some(10),
+        marker: None,
+        usage_filter: None,
+        status_filter: None,
+    };
+    let list_response = encryption_service.list_keys(list_request).await?;
+    println!("   ✓ Keys found in Vault: {}", list_response.keys.len());
+    for (idx, key_info) in list_response.keys.iter().enumerate() {
+        println!("     {}. {} ({:?})", idx + 1, key_info.key_id, key_info.status);
+    }
+    println!();
+
+    // Step 12: Check cache statistics
+    println!("12. Checking cache statistics...");
+    if let Some((hits, misses)) = encryption_service.cache_stats().await {
+        println!("   ✓ Cache statistics:");
+        println!("     - Cache hits: {}", hits);
+        println!("     - Cache misses: {}\n", misses);
+    } else {
+        println!("   - Cache is disabled\n");
+    }
+
+    // Step 13: Health check (verifies Vault connectivity)
+    println!("13. Performing health check (Vault connectivity)...");
+    let is_healthy = encryption_service.health_check().await?;
+    println!("   ✓ KMS backend (Vault) is healthy: {}\n", is_healthy);
+
+    // Step 14: Stop the service
+    println!("14. Stopping KMS service...");
+    service_manager.stop().await?;
+    println!("   ✓ KMS service stopped\n");
+
+    println!("=== Demo 2 (Vault Backend) completed successfully! ===");
+    println!("\n💡 Tips:");
+    println!("   - Keys are now stored in Vault at: {}/v1/secret/data/rustfs/kms/keys/", vault_address);
+    println!("   - You can verify keys in Vault using: vault kv list secret/rustfs/kms/keys/");
+    println!("   - For production, use proper Vault authentication (AppRole, etc.)");
+    println!("   - See examples/VAULT_SETUP.md for detailed Vault configuration guide");
+
+    Ok(())
+}
@@ -271,7 +271,7 @@ impl ConfigureVaultKmsRequest {
        KmsConfig {
            backend: KmsBackend::Vault,
            default_key_id: self.default_key_id.clone(),
-            backend_config: BackendConfig::Vault(VaultConfig {
+            backend_config: BackendConfig::Vault(Box::new(VaultConfig {
                address: self.address.clone(),
                auth_method: self.auth_method.clone(),
                namespace: self.namespace.clone(),
@@ -288,7 +288,7 @@ impl ConfigureVaultKmsRequest {
                } else {
                    None
                },
-            }),
+            })),
            timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
            retry_attempts: self.retry_attempts.unwrap_or(3),
            enable_cache: self.enable_cache.unwrap_or(true),
@@ -17,6 +17,7 @@
 use crate::backends::{BackendInfo, KmsBackend, KmsClient};
 use crate::config::KmsConfig;
 use crate::config::LocalConfig;
+use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
 use crate::error::{KmsError, Result};
 use crate::types::*;
 use aes_gcm::{
@@ -24,11 +25,13 @@ use aes_gcm::{
    aead::{Aead, KeyInit},
 };
 use async_trait::async_trait;
+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
 use jiff::Zoned;
-use rand::Rng;
+use rand::RngExt;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::path::PathBuf;
+use std::time::Duration;
 use tokio::fs;
 use tokio::sync::RwLock;
 use tracing::{debug, info, warn};
@@ -37,9 +40,11 @@ use tracing::{debug, info, warn};
 pub struct LocalKmsClient {
    config: LocalConfig,
    /// In-memory cache of loaded keys for performance
-    key_cache: RwLock<HashMap<String, MasterKey>>,
+    key_cache: RwLock<HashMap<String, MasterKeyInfo>>,
    /// Master encryption key for encrypting stored keys
    master_cipher: Option<Aes256Gcm>,
+    /// DEK encryption implementation
+    dek_crypto: AesDekCrypto,
 }

 /// Serializable representation of a master key stored on disk
@@ -55,24 +60,12 @@ struct StoredMasterKey {
    created_at: Zoned,
    rotated_at: Option<Zoned>,
    created_by: Option<String>,
-    /// Encrypted key material (32 bytes for AES-256)
-    encrypted_key_material: Vec<u8>,
+    /// Encrypted key material (32 bytes encoded in base64 for AES-256)
+    encrypted_key_material: String,
    /// Nonce used for encryption
    nonce: Vec<u8>,
 }

-/// Data key envelope stored with each data key generation
-#[derive(Debug, Clone, Serialize, Deserialize)]
-struct DataKeyEnvelope {
-    key_id: String,
-    master_key_id: String,
-    key_spec: String,
-    encrypted_key: Vec<u8>,
-    nonce: Vec<u8>,
-    encryption_context: HashMap<String, String>,
-    created_at: Zoned,
-}
-
 impl LocalKmsClient {
    /// Create a new local KMS client
    pub async fn new(config: LocalConfig) -> Result<Self> {
@@ -95,6 +88,7 @@ impl LocalKmsClient {
            config,
            key_cache: RwLock::new(HashMap::new()),
            master_cipher,
+            dek_crypto: AesDekCrypto::new(),
        })
    }

@@ -116,8 +110,8 @@ impl LocalKmsClient {
        self.config.key_dir.join(format!("{key_id}.key"))
    }

-    /// Load a master key from disk
-    async fn load_master_key(&self, key_id: &str) -> Result<MasterKey> {
+    /// Decode and decrypt a stored key file, returning both the metadata and decrypted key material
+    async fn decode_stored_key(&self, key_id: &str) -> Result<(StoredMasterKey, Vec<u8>)> {
        let key_path = self.master_key_path(key_id);
        if !key_path.exists() {
            return Err(KmsError::key_not_found(key_id));
@@ -127,7 +121,7 @@ impl LocalKmsClient {
        let stored_key: StoredMasterKey = serde_json::from_slice(&content)?;

        // Decrypt key material if master cipher is available
-        let _key_material = if let Some(ref cipher) = self.master_cipher {
+        let key_material = if let Some(ref cipher) = self.master_cipher {
            if stored_key.nonce.len() != 12 {
                return Err(KmsError::cryptographic_error("nonce", "Invalid nonce length"));
            }
@@ -136,14 +130,29 @@ impl LocalKmsClient {
            nonce_array.copy_from_slice(&stored_key.nonce);
            let nonce = Nonce::from(nonce_array);

+            // Decode base64 string to bytes
+            let encrypted_bytes = BASE64
+                .decode(&stored_key.encrypted_key_material)
+                .map_err(|e| KmsError::cryptographic_error("base64_decode", e.to_string()))?;
+
            cipher
-                .decrypt(&nonce, stored_key.encrypted_key_material.as_ref())
+                .decrypt(&nonce, encrypted_bytes.as_ref())
                .map_err(|e| KmsError::cryptographic_error("decrypt", e.to_string()))?
        } else {
-            stored_key.encrypted_key_material
+            // Decode base64 string to bytes when no encryption
+            BASE64
+                .decode(&stored_key.encrypted_key_material)
+                .map_err(|e| KmsError::cryptographic_error("base64_decode", e.to_string()))?
        };

-        Ok(MasterKey {
+        Ok((stored_key, key_material))
+    }
+
+    /// Load a master key from disk
+    async fn load_master_key(&self, key_id: &str) -> Result<MasterKeyInfo> {
+        let (stored_key, _key_material) = self.decode_stored_key(key_id).await?;
+
+        Ok(MasterKeyInfo {
            key_id: stored_key.key_id,
            version: stored_key.version,
            algorithm: stored_key.algorithm,
@@ -158,7 +167,7 @@ impl LocalKmsClient {
    }

    /// Save a master key to disk
-    async fn save_master_key(&self, master_key: &MasterKey, key_material: &[u8]) -> Result<()> {
+    async fn save_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<()> {
        let key_path = self.master_key_path(&master_key.key_id);

        // Encrypt key material if master cipher is available
@@ -170,9 +179,11 @@ impl LocalKmsClient {
            let encrypted = cipher
                .encrypt(&nonce, key_material)
                .map_err(|e| KmsError::cryptographic_error("encrypt", e.to_string()))?;
-            (encrypted, nonce.to_vec())
+            // Encode encrypted bytes to base64 string
+            (BASE64.encode(&encrypted), nonce.to_vec())
        } else {
-            (key_material.to_vec(), Vec::new())
+            // Encode key material to base64 string when no encryption
+            (BASE64.encode(key_material), Vec::new())
        };

        let stored_key = StoredMasterKey {
@@ -210,39 +221,9 @@ impl LocalKmsClient {
        Ok(())
    }

-    /// Generate a random 256-bit key
-    fn generate_key_material() -> Vec<u8> {
-        let mut key_material = vec![0u8; 32]; // 256 bits
-        rand::rng().fill(&mut key_material[..]);
-        key_material
-    }
-
    /// Get the actual key material for a master key
    async fn get_key_material(&self, key_id: &str) -> Result<Vec<u8>> {
-        let key_path = self.master_key_path(key_id);
-
-        if !key_path.exists() {
-            return Err(KmsError::key_not_found(key_id));
-        }
-
-        let content = fs::read(&key_path).await?;
-        let stored_key: StoredMasterKey = serde_json::from_slice(&content)?;
-
-        // Decrypt key material if master cipher is available
-        let key_material = if let Some(ref cipher) = self.master_cipher {
-            if stored_key.nonce.len() != 12 {
-                return Err(KmsError::cryptographic_error("nonce", "Invalid nonce length"));
-            }
-            let mut nonce_array = [0u8; 12];
-            nonce_array.copy_from_slice(&stored_key.nonce);
-            let nonce = Nonce::from(nonce_array);
-            cipher
-                .decrypt(&nonce, stored_key.encrypted_key_material.as_ref())
-                .map_err(|e| KmsError::cryptographic_error("decrypt", e.to_string()))?
-        } else {
-            stored_key.encrypted_key_material
-        };
-
+        let (_stored_key, key_material) = self.decode_stored_key(key_id).await?;
        Ok(key_material)
    }

@@ -250,53 +231,22 @@ impl LocalKmsClient {
    async fn encrypt_with_master_key(&self, key_id: &str, plaintext: &[u8]) -> Result<(Vec<u8>, Vec<u8>)> {
        // Load the actual master key material
        let key_material = self.get_key_material(key_id).await?;
-        let key = Key::<Aes256Gcm>::try_from(key_material.as_slice())
-            .map_err(|_| KmsError::cryptographic_error("key", "Invalid key length"))?;
-        let cipher = Aes256Gcm::new(&key);
-
-        let mut nonce_bytes = [0u8; 12];
-        rand::rng().fill(&mut nonce_bytes[..]);
-
-        let nonce = Nonce::from(nonce_bytes);
-
-        let ciphertext = cipher
-            .encrypt(&nonce, plaintext)
-            .map_err(|e| KmsError::cryptographic_error("encrypt", e.to_string()))?;
-
-        Ok((ciphertext, nonce_bytes.to_vec()))
+        self.dek_crypto.encrypt(&key_material, plaintext).await
    }

    /// Decrypt data using a master key
    async fn decrypt_with_master_key(&self, key_id: &str, ciphertext: &[u8], nonce: &[u8]) -> Result<Vec<u8>> {
-        if nonce.len() != 12 {
-            return Err(KmsError::cryptographic_error("nonce", "Invalid nonce length"));
-        }
        // Load the actual master key material
        let key_material = self.get_key_material(key_id).await?;
-        let key = Key::<Aes256Gcm>::try_from(key_material.as_slice())
-            .map_err(|_| KmsError::cryptographic_error("key", "Invalid key length"))?;
-        let cipher = Aes256Gcm::new(&key);
-
-        let mut nonce_array = [0u8; 12];
-        nonce_array.copy_from_slice(nonce);
-        let nonce_ref = Nonce::from(nonce_array);
-
-        let plaintext = cipher
-            .decrypt(&nonce_ref, ciphertext)
-            .map_err(|e| KmsError::cryptographic_error("decrypt", e.to_string()))?;
-
-        Ok(plaintext)
+        self.dek_crypto.decrypt(&key_material, ciphertext, nonce).await
    }
 }

 #[async_trait]
 impl KmsClient for LocalKmsClient {
-    async fn generate_data_key(&self, request: &GenerateKeyRequest, context: Option<&OperationContext>) -> Result<DataKey> {
+    async fn generate_data_key(&self, request: &GenerateKeyRequest, _context: Option<&OperationContext>) -> Result<DataKeyInfo> {
        debug!("Generating data key for master key: {}", request.master_key_id);

-        // Verify master key exists
-        let _master_key = self.describe_key(&request.master_key_id, context).await?;
-
        // Generate random data key material
        let key_length = match request.key_spec.as_str() {
            "AES_256" => 32,
@@ -310,7 +260,7 @@ impl KmsClient for LocalKmsClient {
        // Encrypt the data key with the master key
        let (encrypted_key, nonce) = self.encrypt_with_master_key(&request.master_key_id, &plaintext_key).await?;

-        // Create data key envelope
+        // Create data key envelope with master key version for rotation support
        let envelope = DataKeyEnvelope {
            key_id: uuid::Uuid::new_v4().to_string(),
            master_key_id: request.master_key_id.clone(),
@@ -324,7 +274,7 @@ impl KmsClient for LocalKmsClient {
        // Serialize the envelope as the ciphertext
        let ciphertext = serde_json::to_vec(&envelope)?;

-        let data_key = DataKey::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone());
+        let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone());

        info!("Generated data key for master key: {}", request.master_key_id);
        Ok(data_key)
@@ -359,15 +309,19 @@ impl KmsClient for LocalKmsClient {
        let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext)?;

        // Verify encryption context matches
-        if !request.encryption_context.is_empty() {
-            for (key, expected_value) in &request.encryption_context {
-                if let Some(actual_value) = envelope.encryption_context.get(key) {
-                    if actual_value != expected_value {
-                        return Err(KmsError::context_mismatch(format!(
-                            "Context mismatch for key '{key}': expected '{expected_value}', got '{actual_value}'"
-                        )));
-                    }
-                } else {
+        // Check that all keys in envelope.encryption_context are present in request.encryption_context
+        // and their values match. This ensures the context used for decryption matches what was used for encryption.
+        for (key, expected_value) in &envelope.encryption_context {
+            if let Some(actual_value) = request.encryption_context.get(key) {
+                if actual_value != expected_value {
+                    return Err(KmsError::context_mismatch(format!(
+                        "Context mismatch for key '{key}': expected '{expected_value}', got '{actual_value}'"
+                    )));
+                }
+            } else {
+                // If request.encryption_context is empty, allow decryption (backward compatibility)
+                // Otherwise, require all envelope context keys to be present
+                if !request.encryption_context.is_empty() {
                    return Err(KmsError::context_mismatch(format!("Missing context key '{key}'")));
                }
            }
@@ -382,7 +336,7 @@ impl KmsClient for LocalKmsClient {
        Ok(plaintext)
    }

-    async fn create_key(&self, key_id: &str, algorithm: &str, context: Option<&OperationContext>) -> Result<MasterKey> {
+    async fn create_key(&self, key_id: &str, algorithm: &str, context: Option<&OperationContext>) -> Result<MasterKeyInfo> {
        debug!("Creating master key: {}", key_id);

        // Check if key already exists
@@ -396,13 +350,13 @@ impl KmsClient for LocalKmsClient {
        }

        // Generate key material
-        let key_material = Self::generate_key_material();
+        let key_material = generate_key_material(algorithm)?;

        let created_by = context
            .map(|ctx| ctx.principal.clone())
            .unwrap_or_else(|| "local-kms".to_string());

-        let master_key = MasterKey::new_with_description(key_id.to_string(), algorithm.to_string(), Some(created_by), None);
+        let master_key = MasterKeyInfo::new_with_description(key_id.to_string(), algorithm.to_string(), Some(created_by), None);

        // Save to disk
        self.save_master_key(&master_key, &key_material).await?;
@@ -490,7 +444,7 @@ impl KmsClient for LocalKmsClient {

        // For simplicity, we'll regenerate key material
        // In a real implementation, we'd preserve the original key material
-        let key_material = Self::generate_key_material();
+        let key_material = generate_key_material(&master_key.algorithm)?;
        self.save_master_key(&master_key, &key_material).await?;

        // Update cache
@@ -507,7 +461,7 @@ impl KmsClient for LocalKmsClient {
        let mut master_key = self.load_master_key(key_id).await?;
        master_key.status = KeyStatus::Disabled;

-        let key_material = Self::generate_key_material();
+        let key_material = generate_key_material(&master_key.algorithm)?;
        self.save_master_key(&master_key, &key_material).await?;

        // Update cache
@@ -529,7 +483,7 @@ impl KmsClient for LocalKmsClient {
        let mut master_key = self.load_master_key(key_id).await?;
        master_key.status = KeyStatus::PendingDeletion;

-        let key_material = Self::generate_key_material();
+        let key_material = generate_key_material(&master_key.algorithm)?;
        self.save_master_key(&master_key, &key_material).await?;

        // Update cache
@@ -546,7 +500,7 @@ impl KmsClient for LocalKmsClient {
        let mut master_key = self.load_master_key(key_id).await?;
        master_key.status = KeyStatus::Active;

-        let key_material = Self::generate_key_material();
+        let key_material = generate_key_material(&master_key.algorithm)?;
        self.save_master_key(&master_key, &key_material).await?;

        // Update cache
@@ -557,7 +511,7 @@ impl KmsClient for LocalKmsClient {
        Ok(())
    }

-    async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKey> {
+    async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> {
        debug!("Rotating key: {}", key_id);

        let mut master_key = self.load_master_key(key_id).await?;
@@ -565,7 +519,7 @@ impl KmsClient for LocalKmsClient {
        master_key.rotated_at = Some(Zoned::now());

        // Generate new key material
-        let key_material = Self::generate_key_material();
+        let key_material = generate_key_material(&master_key.algorithm)?;
        self.save_master_key(&master_key, &key_material).await?;

        // Update cache
@@ -625,12 +579,13 @@ impl KmsBackend for LocalKmsBackend {

        // Create master key with description directly
        let _master_key = {
+            let algorithm = "AES_256";
            // Generate key material
-            let key_material = LocalKmsClient::generate_key_material();
+            let key_material = generate_key_material(algorithm)?;

-            let master_key = MasterKey::new_with_description(
+            let master_key = MasterKeyInfo::new_with_description(
                key_id.clone(),
-                "AES_256".to_string(),
+                algorithm.to_string(),
                Some("local-kms".to_string()),
                request.description.clone(),
            );
@@ -787,35 +742,19 @@ impl KmsBackend for LocalKmsBackend {
                return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30".to_string()));
            }

-            let deletion_date = Zoned::now() + jiff::Span::new().days(days as i64);
+            let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
            master_key.status = KeyStatus::PendingDeletion;

            (Some(deletion_date.to_string()), Some(deletion_date))
        };

        // Save the updated key to disk - preserve existing key material!
-        // Load the stored key from disk to get the existing key material
-        let key_path = self.client.master_key_path(key_id);
-        let content = tokio::fs::read(&key_path)
+        // Load and decode the stored key to get the existing key material
+        let (_stored_key, existing_key_material) = self
+            .client
+            .decode_stored_key(key_id)
            .await
-            .map_err(|e| KmsError::internal_error(format!("Failed to read key file: {e}")))?;
-        let stored_key: StoredMasterKey =
-            serde_json::from_slice(&content).map_err(|e| KmsError::internal_error(format!("Failed to parse stored key: {e}")))?;
-
-        // Decrypt the existing key material to preserve it
-        let existing_key_material = if let Some(ref cipher) = self.client.master_cipher {
-            if stored_key.nonce.len() != 12 {
-                return Err(KmsError::cryptographic_error("nonce", "Invalid nonce length"));
-            }
-            let mut nonce_array = [0u8; 12];
-            nonce_array.copy_from_slice(&stored_key.nonce);
-            let nonce = Nonce::from(nonce_array);
-            cipher
-                .decrypt(&nonce, stored_key.encrypted_key_material.as_ref())
-                .map_err(|e| KmsError::cryptographic_error("decrypt", e.to_string()))?
-        } else {
-            stored_key.encrypted_key_material
-        };
+            .map_err(|e| KmsError::internal_error(format!("Failed to decode key: {e}")))?;

        self.client.save_master_key(&master_key, &existing_key_material).await?;

@@ -861,8 +800,14 @@ impl KmsBackend for LocalKmsBackend {
        master_key.status = KeyStatus::Active;

        // Save the updated key to disk - this is the missing critical step!
-        let key_material = LocalKmsClient::generate_key_material();
-        self.client.save_master_key(&master_key, &key_material).await?;
+        // Preserve existing key material instead of generating new one
+        let (_stored_key, existing_key_material) = self
+            .client
+            .decode_stored_key(key_id)
+            .await
+            .map_err(|e| KmsError::internal_error(format!("Failed to decode key: {e}")))?;
+
+        self.client.save_master_key(&master_key, &existing_key_material).await?;

        // Update cache
        let mut cache = self.client.key_cache.write().await;
@@ -36,7 +36,7 @@ pub trait KmsClient: Send + Sync {
    ///
    /// # Returns
    /// Returns a DataKey containing both plaintext and encrypted key material
-    async fn generate_data_key(&self, request: &GenerateKeyRequest, context: Option<&OperationContext>) -> Result<DataKey>;
+    async fn generate_data_key(&self, request: &GenerateKeyRequest, context: Option<&OperationContext>) -> Result<DataKeyInfo>;

    /// Encrypt data directly using a master key
    ///
@@ -67,7 +67,7 @@ pub trait KmsClient: Send + Sync {
    /// * `key_id` - Unique identifier for the new key
    /// * `algorithm` - Key algorithm (e.g., "AES_256")
    /// * `context` - Optional operation context for auditing
-    async fn create_key(&self, key_id: &str, algorithm: &str, context: Option<&OperationContext>) -> Result<MasterKey>;
+    async fn create_key(&self, key_id: &str, algorithm: &str, context: Option<&OperationContext>) -> Result<MasterKeyInfo>;

    /// Get information about a specific key
    ///
@@ -139,7 +139,7 @@ pub trait KmsClient: Send + Sync {
    /// # Arguments
    /// * `key_id` - The key identifier
    /// * `context` - Optional operation context for auditing
-    async fn rotate_key(&self, key_id: &str, context: Option<&OperationContext>) -> Result<MasterKey>;
+    async fn rotate_key(&self, key_id: &str, context: Option<&OperationContext>) -> Result<MasterKeyInfo>;

    /// Health check
    ///
@@ -16,14 +16,15 @@

 use crate::backends::{BackendInfo, KmsBackend, KmsClient};
 use crate::config::{KmsConfig, VaultConfig};
+use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
 use crate::error::{KmsError, Result};
 use crate::types::*;
 use async_trait::async_trait;
 use base64::{Engine as _, engine::general_purpose};
 use jiff::Zoned;
-use rand::RngCore;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
+use std::time::Duration;
 use tracing::{debug, info, warn};
 use vaultrs::{
    client::{VaultClient, VaultClientSettingsBuilder},
@@ -38,6 +39,8 @@ pub struct VaultKmsClient {
    kv_mount: String,
    /// Path prefix for storing keys
    key_path_prefix: String,
+    /// DEK encryption implementation
+    dek_crypto: AesDekCrypto,
 }

 /// Key data stored in Vault
@@ -102,6 +105,7 @@ impl VaultKmsClient {
            kv_mount: config.kv_mount.clone(),
            key_path_prefix: config.key_path_prefix.clone(),
            config,
+            dek_crypto: AesDekCrypto::new(),
        })
    }

@@ -110,19 +114,6 @@ impl VaultKmsClient {
        format!("{}/{}", self.key_path_prefix, key_id)
    }

-    /// Generate key material for the given algorithm
-    fn generate_key_material(algorithm: &str) -> Result<Vec<u8>> {
-        let key_size = match algorithm {
-            "AES_256" => 32,
-            "AES_128" => 16,
-            _ => return Err(KmsError::unsupported_algorithm(algorithm)),
-        };
-
-        let mut key_material = vec![0u8; key_size];
-        rand::rng().fill_bytes(&mut key_material);
-        Ok(key_material)
-    }
-
    /// Encrypt key material using Vault's transit engine
    async fn encrypt_key_material(&self, key_material: &[u8]) -> Result<String> {
        // For simplicity, we'll base64 encode the key material
@@ -139,6 +130,64 @@ impl VaultKmsClient {
            .map_err(|e| KmsError::cryptographic_error("decrypt", e.to_string()))
    }

+    /// Get the actual key material for a master key
+    async fn get_key_material(&self, key_id: &str) -> Result<Vec<u8>> {
+        let mut key_data = self.get_key_data(key_id).await?;
+
+        // If encrypted_key_material is empty, generate and store it (fix for old keys)
+        if key_data.encrypted_key_material.is_empty() {
+            warn!("Key {} has empty encrypted_key_material, generating and storing new key material", key_id);
+            let key_material = generate_key_material(&key_data.algorithm)?;
+            key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?;
+            // Store the updated key data back to Vault
+            self.store_key_data(key_id, &key_data).await?;
+            return Ok(key_material);
+        }
+
+        let key_material = match self.decrypt_key_material(&key_data.encrypted_key_material).await {
+            Ok(km) => km,
+            Err(e) => {
+                warn!("Failed to decrypt key material for key {}: {}, generating new key material", key_id, e);
+                let new_key_material = generate_key_material(&key_data.algorithm)?;
+                key_data.encrypted_key_material = self.encrypt_key_material(&new_key_material).await?;
+                // Store the updated key data back to Vault
+                self.store_key_data(key_id, &key_data).await?;
+                return Ok(new_key_material);
+            }
+        };
+
+        // Validate key material length (should be 32 bytes for AES-256)
+        if key_material.len() != 32 {
+            // Try to fix: generate new key material if length is wrong
+            warn!(
+                "Key {} has invalid key material length ({} bytes), generating new key material",
+                key_id,
+                key_material.len()
+            );
+            let new_key_material = generate_key_material(&key_data.algorithm)?;
+            key_data.encrypted_key_material = self.encrypt_key_material(&new_key_material).await?;
+            // Store the updated key data back to Vault
+            self.store_key_data(key_id, &key_data).await?;
+            return Ok(new_key_material);
+        }
+
+        Ok(key_material)
+    }
+
+    /// Encrypt data using a master key
+    async fn encrypt_with_master_key(&self, key_id: &str, plaintext: &[u8]) -> Result<(Vec<u8>, Vec<u8>)> {
+        // Load the actual master key material
+        let key_material = self.get_key_material(key_id).await?;
+        self.dek_crypto.encrypt(&key_material, plaintext).await
+    }
+
+    /// Decrypt data using a master key
+    async fn decrypt_with_master_key(&self, key_id: &str, ciphertext: &[u8], nonce: &[u8]) -> Result<Vec<u8>> {
+        // Load the actual master key material
+        let key_material = self.get_key_material(key_id).await?;
+        self.dek_crypto.decrypt(&key_material, ciphertext, nonce).await
+    }
+
    /// Store key data in Vault
    async fn store_key_data(&self, key_id: &str, key_data: &VaultKeyData) -> Result<()> {
        let path = self.key_path(key_id);
@@ -154,19 +203,36 @@ impl VaultKmsClient {
    async fn store_key_metadata(&self, key_id: &str, request: &CreateKeyRequest) -> Result<()> {
        debug!("Storing key metadata for {}, input tags: {:?}", key_id, request.tags);

+        // Get existing key data to preserve encrypted_key_material and other fields
+        // This is called after create_key, so the key should already exist
+        let mut existing_key_data = self.get_key_data(key_id).await?;
+
+        // If encrypted_key_material is empty, generate it (this handles the case where
+        // an old key was created without proper key material)
+        if existing_key_data.encrypted_key_material.is_empty() {
+            warn!("Key {} has empty encrypted_key_material, generating new key material", key_id);
+            let key_material = generate_key_material(&existing_key_data.algorithm)?;
+            existing_key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?;
+        }
+
+        // Update only the metadata fields, preserving the encrypted_key_material
        let key_data = VaultKeyData {
-            algorithm: "AES_256".to_string(),
+            algorithm: existing_key_data.algorithm.clone(),
            usage: request.key_usage.clone(),
-            created_at: Zoned::now(),
-            status: KeyStatus::Active,
-            version: 1,
+            created_at: existing_key_data.created_at,
+            status: existing_key_data.status,
+            version: existing_key_data.version,
            description: request.description.clone(),
-            metadata: HashMap::new(),
+            metadata: existing_key_data.metadata.clone(),
            tags: request.tags.clone(),
-            encrypted_key_material: String::new(), // Not used for transit keys
+            encrypted_key_material: existing_key_data.encrypted_key_material.clone(), // Preserve the key material
        };

-        debug!("VaultKeyData tags before storage: {:?}", key_data.tags);
+        debug!(
+            "VaultKeyData tags before storage: {:?}, encrypted_key_material length: {}",
+            key_data.tags,
+            key_data.encrypted_key_material.len()
+        );
        self.store_key_data(key_id, &key_data).await
    }

@@ -225,36 +291,33 @@ impl VaultKmsClient {

 #[async_trait]
 impl KmsClient for VaultKmsClient {
-    async fn generate_data_key(&self, request: &GenerateKeyRequest, context: Option<&OperationContext>) -> Result<DataKey> {
+    async fn generate_data_key(&self, request: &GenerateKeyRequest, _context: Option<&OperationContext>) -> Result<DataKeyInfo> {
        debug!("Generating data key for master key: {}", request.master_key_id);

-        // Verify master key exists
-        let _master_key = self.describe_key(&request.master_key_id, context).await?;
-
-        // Generate data key material
-        let key_length = match request.key_spec.as_str() {
-            "AES_256" => 32,
-            "AES_128" => 16,
-            _ => return Err(KmsError::unsupported_algorithm(&request.key_spec)),
-        };
-
-        let mut plaintext_key = vec![0u8; key_length];
-        rand::rng().fill_bytes(&mut plaintext_key);
+        // Generate random data key material using the existing method
+        let plaintext_key = generate_key_material(&request.key_spec)?;

        // Encrypt the data key with the master key
-        let encrypted_key = self.encrypt_key_material(&plaintext_key).await?;
+        let (encrypted_key, nonce) = self.encrypt_with_master_key(&request.master_key_id, &plaintext_key).await?;

-        Ok(DataKey {
-            key_id: request.master_key_id.clone(),
-            version: 1,
-            plaintext: Some(plaintext_key),
-            ciphertext: general_purpose::STANDARD
-                .decode(&encrypted_key)
-                .map_err(|e| KmsError::cryptographic_error("decode", e.to_string()))?,
+        // Create data key envelope with master key version for rotation support
+        let envelope = DataKeyEnvelope {
+            key_id: uuid::Uuid::new_v4().to_string(),
+            master_key_id: request.master_key_id.clone(),
            key_spec: request.key_spec.clone(),
-            metadata: request.encryption_context.clone(),
+            encrypted_key: encrypted_key.clone(),
+            nonce,
+            encryption_context: request.encryption_context.clone(),
            created_at: Zoned::now(),
-        })
+        };
+
+        // Serialize the envelope as the ciphertext
+        let ciphertext = serde_json::to_vec(&envelope)?;
+
+        let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone());
+
+        info!("Generated data key for master key: {}", request.master_key_id);
+        Ok(data_key)
    }

    async fn encrypt(&self, request: &EncryptRequest, _context: Option<&OperationContext>) -> Result<EncryptResponse> {
@@ -279,15 +342,42 @@ impl KmsClient for VaultKmsClient {
        })
    }

-    async fn decrypt(&self, _request: &DecryptRequest, _context: Option<&OperationContext>) -> Result<Vec<u8>> {
+    async fn decrypt(&self, request: &DecryptRequest, _context: Option<&OperationContext>) -> Result<Vec<u8>> {
        debug!("Decrypting data");

-        // For this simple implementation, we assume the key ID is embedded in the ciphertext metadata
-        // In practice, you'd extract this from the ciphertext envelope
-        Err(KmsError::invalid_operation("Decrypt not fully implemented for Vault backend"))
+        // Parse the data key envelope from ciphertext
+        let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext)
+            .map_err(|e| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {e}")))?;
+
+        // Verify encryption context matches
+        // Check that all keys in envelope.encryption_context are present in request.encryption_context
+        // and their values match. This ensures the context used for decryption matches what was used for encryption.
+        for (key, expected_value) in &envelope.encryption_context {
+            if let Some(actual_value) = request.encryption_context.get(key) {
+                if actual_value != expected_value {
+                    return Err(KmsError::context_mismatch(format!(
+                        "Context mismatch for key '{key}': expected '{expected_value}', got '{actual_value}'"
+                    )));
+                }
+            } else {
+                // If request.encryption_context is empty, allow decryption (backward compatibility)
+                // Otherwise, require all envelope context keys to be present
+                if !request.encryption_context.is_empty() {
+                    return Err(KmsError::context_mismatch(format!("Missing context key '{key}'")));
+                }
+            }
+        }
+
+        // Decrypt the data key
+        let plaintext = self
+            .decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce)
+            .await?;
+
+        info!("Successfully decrypted data");
+        Ok(plaintext)
    }

-    async fn create_key(&self, key_id: &str, algorithm: &str, _context: Option<&OperationContext>) -> Result<MasterKey> {
+    async fn create_key(&self, key_id: &str, algorithm: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> {
        debug!("Creating master key: {} with algorithm: {}", key_id, algorithm);

        // Check if key already exists
@@ -296,7 +386,7 @@ impl KmsClient for VaultKmsClient {
        }

        // Generate key material
-        let key_material = Self::generate_key_material(algorithm)?;
+        let key_material = generate_key_material(algorithm)?;
        let encrypted_material = self.encrypt_key_material(&key_material).await?;

        // Create key data
@@ -315,7 +405,7 @@ impl KmsClient for VaultKmsClient {
        // Store in Vault
        self.store_key_data(key_id, &key_data).await?;

-        let master_key = MasterKey {
+        let master_key = MasterKeyInfo {
            key_id: key_id.to_string(),
            version: key_data.version,
            algorithm: key_data.algorithm.clone(),
@@ -438,19 +528,19 @@ impl KmsClient for VaultKmsClient {
        Ok(())
    }

-    async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKey> {
+    async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> {
        debug!("Rotating key: {}", key_id);

        let mut key_data = self.get_key_data(key_id).await?;
        key_data.version += 1;

        // Generate new key material
-        let key_material = Self::generate_key_material(&key_data.algorithm)?;
+        let key_material = generate_key_material(&key_data.algorithm)?;
        key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?;

        self.store_key_data(key_id, &key_data).await?;

-        let master_key = MasterKey {
+        let master_key = MasterKeyInfo {
            key_id: key_id.to_string(),
            version: key_data.version,
            algorithm: key_data.algorithm,
@@ -506,7 +596,7 @@ impl VaultKmsBackend {
    /// Create a new VaultKmsBackend
    pub async fn new(config: KmsConfig) -> Result<Self> {
        let vault_config = match &config.backend_config {
-            crate::config::BackendConfig::Vault(vault_config) => vault_config.clone(),
+            crate::config::BackendConfig::Vault(vault_config) => (**vault_config).clone(),
            _ => return Err(KmsError::configuration_error("Expected Vault backend configuration")),
        };

@@ -681,7 +771,7 @@ impl KmsBackend for VaultKmsBackend {
                ));
            }

-            let deletion_date = Zoned::now() + jiff::Span::new().days(days as i64);
+            let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
            key_metadata.key_state = KeyState::PendingDeletion;
            key_metadata.deletion_date = Some(deletion_date.clone());

@@ -69,7 +69,7 @@ pub enum BackendConfig {
    /// Local backend configuration
    Local(LocalConfig),
    /// Vault backend configuration
-    Vault(VaultConfig),
+    Vault(Box<VaultConfig>),
 }

 impl Default for BackendConfig {
@@ -194,11 +194,11 @@ impl KmsConfig {
    pub fn vault(address: Url, token: String) -> Self {
        Self {
            backend: KmsBackend::Vault,
-            backend_config: BackendConfig::Vault(VaultConfig {
+            backend_config: BackendConfig::Vault(Box::new(VaultConfig {
                address: address.to_string(),
                auth_method: VaultAuthMethod::Token { token },
                ..Default::default()
-            }),
+            })),
            ..Default::default()
        }
    }
@@ -207,11 +207,11 @@ impl KmsConfig {
    pub fn vault_approle(address: Url, role_id: String, secret_id: String) -> Self {
        Self {
            backend: KmsBackend::Vault,
-            backend_config: BackendConfig::Vault(VaultConfig {
+            backend_config: BackendConfig::Vault(Box::new(VaultConfig {
                address: address.to_string(),
                auth_method: VaultAuthMethod::AppRole { role_id, secret_id },
                ..Default::default()
-            }),
+            })),
            ..Default::default()
        }
    }
@@ -353,7 +353,7 @@ impl KmsConfig {
                let address = std::env::var("RUSTFS_KMS_VAULT_ADDRESS").unwrap_or_else(|_| "http://localhost:8200".to_string());
                let token = std::env::var("RUSTFS_KMS_VAULT_TOKEN").unwrap_or_else(|_| "dev-token".to_string());

-                config.backend_config = BackendConfig::Vault(VaultConfig {
+                config.backend_config = BackendConfig::Vault(Box::new(VaultConfig {
                    address,
                    auth_method: VaultAuthMethod::Token { token },
                    namespace: std::env::var("RUSTFS_KMS_VAULT_NAMESPACE").ok(),
@@ -362,7 +362,7 @@ impl KmsConfig {
                    key_path_prefix: std::env::var("RUSTFS_KMS_VAULT_KEY_PREFIX")
                        .unwrap_or_else(|_| "rustfs/kms/keys".to_string()),
                    tls: None,
-                });
+                }));
            }
        }

@@ -21,7 +21,7 @@ use aes_gcm::{
    aead::{Aead, KeyInit},
 };
 use chacha20poly1305::ChaCha20Poly1305;
-use rand::Rng;
+use rand::RngExt;

 /// Trait for object encryption ciphers
 #[cfg_attr(not(test), allow(dead_code))]
@@ -0,0 +1,314 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Data Encryption Key (DEK) encryption interface and implementations
+//!
+//! This module provides a unified interface for encrypting and decrypting
+//! data encryption keys using master keys. It abstracts the encryption
+//! operations so that different backends can share the same encryption logic.
+
+#![allow(dead_code)] // Trait methods may be used by implementations
+
+use crate::error::{KmsError, Result};
+use async_trait::async_trait;
+use jiff::Zoned;
+use rand::Rng;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+/// Data key envelope for encrypting/decrypting data keys
+///
+/// This structure stores the encrypted DEK along with metadata needed for decryption.
+/// The `master_key_version` field records which version of the KEK (Key Encryption Key)
+/// was used to encrypt this DEK, enabling proper key rotation support.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DataKeyEnvelope {
+    pub key_id: String,
+    pub master_key_id: String,
+    pub key_spec: String,
+    pub encrypted_key: Vec<u8>,
+    pub nonce: Vec<u8>,
+    pub encryption_context: HashMap<String, String>,
+    pub created_at: Zoned,
+}
+
+/// Trait for encrypting and decrypting data encryption keys (DEK)
+///
+/// This trait abstracts the encryption operations used to protect
+/// data encryption keys with master keys. Different implementations
+/// can use different encryption algorithms (e.g., AES-256-GCM).
+#[async_trait]
+pub trait DekCrypto: Send + Sync {
+    /// Encrypt plaintext data using a master key material
+    ///
+    /// # Arguments
+    /// * `key_material` - The master key material (raw bytes)
+    /// * `plaintext` - The data to encrypt
+    ///
+    /// # Returns
+    /// A tuple of (ciphertext, nonce) where:
+    /// - `ciphertext` - The encrypted data
+    /// - `nonce` - The nonce used for encryption (should be stored with ciphertext)
+    async fn encrypt(&self, key_material: &[u8], plaintext: &[u8]) -> Result<(Vec<u8>, Vec<u8>)>;
+
+    /// Decrypt ciphertext data using a master key material
+    ///
+    /// # Arguments
+    /// * `key_material` - The master key material (raw bytes)
+    /// * `ciphertext` - The encrypted data
+    /// * `nonce` - The nonce used for encryption
+    ///
+    /// # Returns
+    /// The decrypted plaintext data
+    async fn decrypt(&self, key_material: &[u8], ciphertext: &[u8], nonce: &[u8]) -> Result<Vec<u8>>;
+
+    /// Get the algorithm name used by this implementation
+    #[allow(dead_code)] // May be used by implementations or for debugging
+    fn algorithm(&self) -> &'static str;
+
+    /// Get the required key material size in bytes
+    #[allow(dead_code)] // May be used by implementations or for debugging
+    fn key_size(&self) -> usize;
+}
+
+/// AES-256-GCM implementation of DEK encryption
+pub struct AesDekCrypto;
+
+impl AesDekCrypto {
+    /// Create a new AES-256-GCM DEK crypto instance
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+#[async_trait]
+impl DekCrypto for AesDekCrypto {
+    async fn encrypt(&self, key_material: &[u8], plaintext: &[u8]) -> Result<(Vec<u8>, Vec<u8>)> {
+        use aes_gcm::{
+            Aes256Gcm, Key, Nonce,
+            aead::{Aead, KeyInit},
+        };
+
+        // Validate key material length
+        if key_material.len() != 32 {
+            return Err(KmsError::cryptographic_error(
+                "key",
+                format!("Invalid key length: expected 32 bytes, got {}", key_material.len()),
+            ));
+        }
+
+        // Create cipher from key material
+        let key =
+            Key::<Aes256Gcm>::try_from(key_material).map_err(|_| KmsError::cryptographic_error("key", "Invalid key length"))?;
+        let cipher = Aes256Gcm::new(&key);
+
+        // Generate random nonce (12 bytes for GCM)
+        let mut nonce_bytes = [0u8; 12];
+        rand::rng().fill_bytes(&mut nonce_bytes);
+        let nonce = Nonce::from(nonce_bytes);
+
+        // Encrypt plaintext
+        let ciphertext = cipher
+            .encrypt(&nonce, plaintext)
+            .map_err(|e| KmsError::cryptographic_error("encrypt", e.to_string()))?;
+
+        Ok((ciphertext, nonce_bytes.to_vec()))
+    }
+
+    async fn decrypt(&self, key_material: &[u8], ciphertext: &[u8], nonce: &[u8]) -> Result<Vec<u8>> {
+        use aes_gcm::{
+            Aes256Gcm, Key, Nonce,
+            aead::{Aead, KeyInit},
+        };
+
+        // Validate nonce length
+        if nonce.len() != 12 {
+            return Err(KmsError::cryptographic_error("nonce", "Invalid nonce length: expected 12 bytes"));
+        }
+
+        // Validate key material length
+        if key_material.len() != 32 {
+            return Err(KmsError::cryptographic_error(
+                "key",
+                format!("Invalid key length: expected 32 bytes, got {}", key_material.len()),
+            ));
+        }
+
+        // Create cipher from key material
+        let key =
+            Key::<Aes256Gcm>::try_from(key_material).map_err(|_| KmsError::cryptographic_error("key", "Invalid key length"))?;
+        let cipher = Aes256Gcm::new(&key);
+
+        // Convert nonce
+        let mut nonce_array = [0u8; 12];
+        nonce_array.copy_from_slice(nonce);
+        let nonce_ref = Nonce::from(nonce_array);
+
+        // Decrypt ciphertext
+        let plaintext = cipher
+            .decrypt(&nonce_ref, ciphertext)
+            .map_err(|e| KmsError::cryptographic_error("decrypt", e.to_string()))?;
+
+        Ok(plaintext)
+    }
+
+    #[allow(dead_code)] // Trait method, may be used by implementations
+    fn algorithm(&self) -> &'static str {
+        "AES-256-GCM"
+    }
+
+    #[allow(dead_code)] // Trait method, may be used by implementations
+    fn key_size(&self) -> usize {
+        32 // 256 bits
+    }
+}
+
+impl Default for AesDekCrypto {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Generate random key material for the given algorithm
+///
+/// # Arguments
+/// * `algorithm` - The key algorithm (e.g., "AES_256", "AES_128")
+///
+/// # Returns
+/// A vector containing the generated key material
+pub fn generate_key_material(algorithm: &str) -> Result<Vec<u8>> {
+    let key_size = match algorithm {
+        "AES_256" => 32,
+        "AES_128" => 16,
+        _ => return Err(KmsError::unsupported_algorithm(algorithm)),
+    };
+
+    let mut key_material = vec![0u8; key_size];
+    rand::rng().fill_bytes(&mut key_material);
+    Ok(key_material)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_aes_dek_crypto_encrypt_decrypt() {
+        let crypto = AesDekCrypto::new();
+
+        // Generate test key material
+        let key_material = generate_key_material("AES_256").expect("Failed to generate key material");
+        let plaintext = b"Hello, World! This is a test message.";
+
+        // Test encryption
+        let (ciphertext, nonce) = crypto
+            .encrypt(&key_material, plaintext)
+            .await
+            .expect("Encryption should succeed");
+
+        assert!(!ciphertext.is_empty());
+        assert_eq!(nonce.len(), 12);
+        assert_ne!(ciphertext, plaintext);
+
+        // Test decryption
+        let decrypted = crypto
+            .decrypt(&key_material, &ciphertext, &nonce)
+            .await
+            .expect("Decryption should succeed");
+
+        assert_eq!(decrypted, plaintext);
+    }
+
+    #[tokio::test]
+    async fn test_aes_dek_crypto_invalid_key_size() {
+        let crypto = AesDekCrypto::new();
+        let invalid_key = vec![0u8; 16]; // Too short
+        let plaintext = b"test";
+
+        let result = crypto.encrypt(&invalid_key, plaintext).await;
+        assert!(result.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_aes_dek_crypto_invalid_nonce() {
+        let crypto = AesDekCrypto::new();
+        let key_material = generate_key_material("AES_256").expect("Failed to generate key material");
+        let ciphertext = vec![0u8; 16];
+        let invalid_nonce = vec![0u8; 8]; // Too short
+
+        let result = crypto.decrypt(&key_material, &ciphertext, &invalid_nonce).await;
+        assert!(result.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_generate_key_material() {
+        let key_256 = generate_key_material("AES_256").expect("Should generate AES_256 key");
+        assert_eq!(key_256.len(), 32);
+
+        let key_128 = generate_key_material("AES_128").expect("Should generate AES_128 key");
+        assert_eq!(key_128.len(), 16);
+
+        // Keys should be different
+        let key_256_2 = generate_key_material("AES_256").expect("Should generate AES_256 key");
+        assert_ne!(key_256, key_256_2);
+
+        // Invalid algorithm
+        assert!(generate_key_material("INVALID").is_err());
+    }
+
+    #[tokio::test]
+    async fn test_data_key_envelope_serialization() {
+        let envelope = DataKeyEnvelope {
+            key_id: "test-key-id".to_string(),
+            master_key_id: "master-key-id".to_string(),
+            key_spec: "AES_256".to_string(),
+            encrypted_key: vec![1, 2, 3, 4],
+            nonce: vec![5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
+            encryption_context: {
+                let mut map = HashMap::new();
+                map.insert("bucket".to_string(), "test-bucket".to_string());
+                map
+            },
+            created_at: Zoned::now(),
+        };
+
+        // Test serialization
+        let serialized = serde_json::to_vec(&envelope).expect("Serialization should succeed");
+        assert!(!serialized.is_empty());
+
+        // Test deserialization
+        let deserialized: DataKeyEnvelope = serde_json::from_slice(&serialized).expect("Deserialization should succeed");
+        assert_eq!(deserialized.key_id, envelope.key_id);
+        assert_eq!(deserialized.master_key_id, envelope.master_key_id);
+        assert_eq!(deserialized.encrypted_key, envelope.encrypted_key);
+    }
+
+    #[tokio::test]
+    async fn test_data_key_envelope_backward_compatibility() {
+        // Test deserialization with current Zoned format (with timezone annotation)
+        let envelope_json = r#"{
+            "key_id": "test-key-id",
+            "master_key_id": "master-key-id",
+            "key_spec": "AES_256",
+            "encrypted_key": [1, 2, 3, 4],
+            "nonce": [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
+            "encryption_context": {"bucket": "test-bucket"},
+            "created_at": "2024-01-01T00:00:00+00:00[UTC]"
+        }"#;
+
+        let deserialized: DataKeyEnvelope = serde_json::from_str(envelope_json).expect("Should deserialize current format");
+        assert_eq!(deserialized.key_id, "test-key-id");
+        assert_eq!(deserialized.master_key_id, "master-key-id");
+    }
+}
@@ -14,7 +14,7 @@

 //! Object encryption service implementation

-mod ciphers;
-pub mod service;
+pub mod ciphers;
+pub mod dek;

-pub use service::ObjectEncryptionService;
+pub use dek::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
@@ -63,6 +63,7 @@ pub mod config;
 mod encryption;
 mod error;
 pub mod manager;
+pub mod service;
 pub mod service_manager;
 pub mod types;

@@ -73,10 +74,9 @@ pub use api_types::{
    UntagKeyRequest, UntagKeyResponse, UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
 };
 pub use config::*;
-pub use encryption::ObjectEncryptionService;
-pub use encryption::service::DataKey;
 pub use error::{KmsError, Result};
 pub use manager::KmsManager;
+pub use service::{DataKey, ObjectEncryptionService};
 pub use service_manager::{
    KmsServiceManager, KmsServiceStatus, get_global_encryption_service, get_global_kms_service_manager,
    init_global_kms_service_manager,
@@ -112,6 +112,7 @@ pub fn shutdown_global_services() {
 #[cfg(test)]
 mod tests {
    use super::*;
+    use std::sync::Arc;
    use tempfile::TempDir;

    #[tokio::test]
@@ -139,4 +140,91 @@ mod tests {
        // Test stop
        manager.stop().await.expect("Stop should succeed");
    }
+
+    #[tokio::test]
+    async fn test_versioned_service_reconfiguration() {
+        // Test versioned service reconfiguration for zero-downtime
+        let manager = KmsServiceManager::new();
+
+        // Initial state: no version
+        assert!(manager.get_service_version().await.is_none());
+
+        // Start first service
+        let temp_dir1 = TempDir::new().expect("Failed to create temp dir");
+        let config1 = KmsConfig::local(temp_dir1.path().to_path_buf());
+        manager
+            .configure(config1.clone())
+            .await
+            .expect("Configuration should succeed");
+        manager.start().await.expect("Start should succeed");
+
+        // Verify version 1
+        let version1 = manager.get_service_version().await.expect("Service should have version");
+        assert_eq!(version1, 1);
+
+        // Get service reference (simulating ongoing operation)
+        let service1 = manager.get_encryption_service().await.expect("Service should be available");
+
+        // Reconfigure to new service (zero-downtime)
+        let temp_dir2 = TempDir::new().expect("Failed to create temp dir");
+        let config2 = KmsConfig::local(temp_dir2.path().to_path_buf());
+        manager.reconfigure(config2).await.expect("Reconfiguration should succeed");
+
+        // Verify version 2
+        let version2 = manager.get_service_version().await.expect("Service should have version");
+        assert_eq!(version2, 2);
+
+        // Old service reference should still be valid (Arc keeps it alive)
+        // New requests should get version 2
+        let service2 = manager.get_encryption_service().await.expect("Service should be available");
+
+        // Verify they are different instances
+        assert!(!Arc::ptr_eq(&service1, &service2));
+
+        // Old service should still work (simulating long-running operation)
+        // This demonstrates zero-downtime: old operations continue, new operations use new service
+        assert!(service1.health_check().await.is_ok());
+        assert!(service2.health_check().await.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_concurrent_reconfiguration() {
+        // Test that concurrent reconfiguration requests are serialized
+        let manager = Arc::new(KmsServiceManager::new());
+
+        let temp_dir = TempDir::new().expect("Failed to create temp dir");
+        let base_path = temp_dir.path().to_path_buf();
+
+        // Initial configuration
+        let config1 = KmsConfig::local(base_path.clone());
+        manager.configure(config1).await.expect("Configuration should succeed");
+        manager.start().await.expect("Start should succeed");
+
+        // Spawn multiple concurrent reconfiguration requests
+        let mut handles = Vec::new();
+        for _i in 0..5 {
+            let manager_clone = manager.clone();
+            let path = base_path.clone();
+            let handle = tokio::spawn(async move {
+                let config = KmsConfig::local(path);
+                manager_clone.reconfigure(config).await
+            });
+            handles.push(handle);
+        }
+
+        // Wait for all reconfigurations to complete
+        let mut results = Vec::new();
+        for handle in handles {
+            results.push(handle.await);
+        }
+
+        // All should succeed (serialized by mutex)
+        for result in results {
+            assert!(result.expect("Task should complete").is_ok());
+        }
+
+        // Final version should be 6 (1 initial + 5 reconfigurations)
+        let final_version = manager.get_service_version().await.expect("Service should have version");
+        assert_eq!(final_version, 6);
+    }
 }
@@ -274,6 +274,8 @@ impl ObjectEncryptionService {
        // Build encryption context
        let mut context = encryption_context.cloned().unwrap_or_default();
        context.insert("bucket".to_string(), bucket.to_string());
+        context.insert("object_key".to_string(), object_key.to_string());
+        // Backward compatibility: also include legacy "object" context key
        context.insert("object".to_string(), object_key.to_string());
        context.insert("algorithm".to_string(), algorithm.as_str().to_string());

@@ -16,11 +16,15 @@

 use crate::backends::{KmsBackend, local::LocalKmsBackend};
 use crate::config::{BackendConfig, KmsConfig};
-use crate::encryption::service::ObjectEncryptionService;
 use crate::error::{KmsError, Result};
 use crate::manager::KmsManager;
-use std::sync::{Arc, OnceLock};
-use tokio::sync::RwLock;
+use crate::service::ObjectEncryptionService;
+use arc_swap::ArcSwap;
+use std::sync::{
+    Arc, OnceLock,
+    atomic::{AtomicU64, Ordering},
+};
+use tokio::sync::{Mutex, RwLock};
 use tracing::{error, info, warn};

 /// KMS service status
@@ -36,26 +40,43 @@ pub enum KmsServiceStatus {
    Error(String),
 }

-/// Dynamic KMS service manager
+/// Service version information for zero-downtime reconfiguration
+#[derive(Clone)]
+struct ServiceVersion {
+    /// Service version number (monotonically increasing)
+    version: u64,
+    /// The encryption service instance
+    service: Arc<ObjectEncryptionService>,
+    /// The KMS manager instance
+    manager: Arc<KmsManager>,
+}
+
+/// Dynamic KMS service manager with versioned services for zero-downtime reconfiguration
 pub struct KmsServiceManager {
-    /// Current KMS manager (if running)
-    manager: Arc<RwLock<Option<Arc<KmsManager>>>>,
-    /// Current encryption service (if running)
-    encryption_service: Arc<RwLock<Option<Arc<ObjectEncryptionService>>>>,
+    /// Current service version (if running)
+    /// Uses ArcSwap for atomic, lock-free service switching
+    /// This allows instant atomic updates without blocking readers
+    current_service: ArcSwap<Option<ServiceVersion>>,
    /// Current configuration
    config: Arc<RwLock<Option<KmsConfig>>>,
    /// Current status
    status: Arc<RwLock<KmsServiceStatus>>,
+    /// Version counter (monotonically increasing)
+    version_counter: Arc<AtomicU64>,
+    /// Mutex to protect lifecycle operations (start, stop, reconfigure)
+    /// This ensures only one lifecycle operation happens at a time
+    lifecycle_mutex: Arc<Mutex<()>>,
 }

 impl KmsServiceManager {
    /// Create a new KMS service manager (not configured)
    pub fn new() -> Self {
        Self {
-            manager: Arc::new(RwLock::new(None)),
-            encryption_service: Arc::new(RwLock::new(None)),
+            current_service: ArcSwap::from_pointee(None),
            config: Arc::new(RwLock::new(None)),
            status: Arc::new(RwLock::new(KmsServiceStatus::NotConfigured)),
+            version_counter: Arc::new(AtomicU64::new(0)),
+            lifecycle_mutex: Arc::new(Mutex::new(())),
        }
    }

@@ -89,6 +110,12 @@ impl KmsServiceManager {

    /// Start KMS service with current configuration
    pub async fn start(&self) -> Result<()> {
+        let _guard = self.lifecycle_mutex.lock().await;
+        self.start_internal().await
+    }
+
+    /// Internal start implementation (called within lifecycle mutex)
+    async fn start_internal(&self) -> Result<()> {
        let config = {
            let config_guard = self.config.read().await;
            match config_guard.as_ref() {
@@ -105,23 +132,11 @@ impl KmsServiceManager {

        info!("Starting KMS service with backend: {:?}", config.backend);

-        match self.create_backend(&config).await {
-            Ok(backend) => {
-                // Create KMS manager
-                let kms_manager = Arc::new(KmsManager::new(backend, config));
-
-                // Create encryption service
-                let encryption_service = Arc::new(ObjectEncryptionService::new((*kms_manager).clone()));
-
-                // Update manager and service
-                {
-                    let mut manager = self.manager.write().await;
-                    *manager = Some(kms_manager);
-                }
-                {
-                    let mut service = self.encryption_service.write().await;
-                    *service = Some(encryption_service);
-                }
+        match self.create_service_version(&config).await {
+            Ok(service_version) => {
+                // Atomically update to new service version (lock-free, instant)
+                // ArcSwap::store() is a true atomic operation using CAS
+                self.current_service.store(Arc::new(Some(service_version)));

                // Update status
                {
@@ -143,18 +158,21 @@ impl KmsServiceManager {
    }

    /// Stop KMS service
+    ///
+    /// Note: This stops accepting new operations, but existing operations using
+    /// the service will continue until they complete (due to Arc reference counting).
    pub async fn stop(&self) -> Result<()> {
+        let _guard = self.lifecycle_mutex.lock().await;
+        self.stop_internal().await
+    }
+
+    /// Internal stop implementation (called within lifecycle mutex)
+    async fn stop_internal(&self) -> Result<()> {
        info!("Stopping KMS service");

-        // Clear manager and service
-        {
-            let mut manager = self.manager.write().await;
-            *manager = None;
-        }
-        {
-            let mut service = self.encryption_service.write().await;
-            *service = None;
-        }
+        // Atomically clear current service version (lock-free, instant)
+        // Note: Existing Arc references will keep the service alive until operations complete
+        self.current_service.store(Arc::new(None));

        // Update status (keep configuration)
        {
@@ -164,37 +182,96 @@ impl KmsServiceManager {
            }
        }

-        info!("KMS service stopped successfully");
+        info!("KMS service stopped successfully (existing operations may continue)");
        Ok(())
    }

-    /// Reconfigure and restart KMS service
+    /// Reconfigure and restart KMS service with zero-downtime
+    ///
+    /// This method implements versioned service switching:
+    /// 1. Creates a new service version without stopping the old one
+    /// 2. Atomically switches to the new version
+    /// 3. Old operations continue using the old service (via Arc reference counting)
+    /// 4. New operations automatically use the new service
+    ///
+    /// This ensures zero downtime during reconfiguration, even for long-running
+    /// operations like encrypting large files.
    pub async fn reconfigure(&self, new_config: KmsConfig) -> Result<()> {
-        info!("Reconfiguring KMS service");
+        let _guard = self.lifecycle_mutex.lock().await;

-        // Stop current service if running
-        if matches!(self.get_status().await, KmsServiceStatus::Running) {
-            self.stop().await?;
-        }
+        info!("Reconfiguring KMS service (zero-downtime)");

        // Configure with new config
-        self.configure(new_config).await?;
+        {
+            let mut config = self.config.write().await;
+            *config = Some(new_config.clone());
+        }

-        // Start with new configuration
-        self.start().await?;
+        // Create new service version without stopping old one
+        // This allows existing operations to continue while new operations use new service
+        match self.create_service_version(&new_config).await {
+            Ok(new_service_version) => {
+                // Get old version for logging (lock-free read)
+                let old_version = self.current_service.load().as_ref().as_ref().map(|sv| sv.version);

-        info!("KMS service reconfigured successfully");
-        Ok(())
+                // Atomically switch to new service version (lock-free, instant CAS operation)
+                // This is a true atomic operation - no waiting for locks, instant switch
+                // Old service will be dropped when no more Arc references exist
+                self.current_service.store(Arc::new(Some(new_service_version.clone())));
+
+                // Update status
+                {
+                    let mut status = self.status.write().await;
+                    *status = KmsServiceStatus::Running;
+                }
+
+                if let Some(old_ver) = old_version {
+                    info!(
+                        "KMS service reconfigured successfully: version {} -> {} (old service will be cleaned up when operations complete)",
+                        old_ver, new_service_version.version
+                    );
+                } else {
+                    info!(
+                        "KMS service reconfigured successfully: version {} (service started)",
+                        new_service_version.version
+                    );
+                }
+                Ok(())
+            }
+            Err(e) => {
+                let err_msg = format!("Failed to reconfigure KMS: {e}");
+                error!("{}", err_msg);
+                let mut status = self.status.write().await;
+                *status = KmsServiceStatus::Error(err_msg.clone());
+                Err(KmsError::backend_error(&err_msg))
+            }
+        }
    }

    /// Get KMS manager (if running)
+    ///
+    /// Returns the manager from the current service version.
+    /// Uses lock-free atomic load for optimal performance.
    pub async fn get_manager(&self) -> Option<Arc<KmsManager>> {
-        self.manager.read().await.clone()
+        self.current_service.load().as_ref().as_ref().map(|sv| sv.manager.clone())
    }

-    /// Get encryption service (if running)  
+    /// Get encryption service (if running)
+    ///
+    /// Returns the service from the current service version.
+    /// Uses lock-free atomic load - no blocking, instant access.
+    /// This ensures new operations always use the latest service version,
+    /// while existing operations continue using their Arc references.
    pub async fn get_encryption_service(&self) -> Option<Arc<ObjectEncryptionService>> {
-        self.encryption_service.read().await.clone()
+        self.current_service.load().as_ref().as_ref().map(|sv| sv.service.clone())
+    }
+
+    /// Get current service version number
+    ///
+    /// Useful for monitoring and debugging.
+    /// Uses lock-free atomic load.
+    pub async fn get_service_version(&self) -> Option<u64> {
+        self.current_service.load().as_ref().as_ref().map(|sv| sv.version)
    }

    /// Health check for the KMS service
@@ -226,20 +303,40 @@ impl KmsServiceManager {
        }
    }

-    /// Create backend from configuration
-    async fn create_backend(&self, config: &KmsConfig) -> Result<Arc<dyn KmsBackend>> {
-        match &config.backend_config {
+    /// Create a new service version from configuration
+    ///
+    /// This creates a new backend, manager, and service, and assigns it a new version number.
+    async fn create_service_version(&self, config: &KmsConfig) -> Result<ServiceVersion> {
+        // Increment version counter
+        let version = self.version_counter.fetch_add(1, Ordering::Relaxed) + 1;
+
+        info!("Creating KMS service version {} with backend: {:?}", version, config.backend);
+
+        // Create backend
+        let backend = match &config.backend_config {
            BackendConfig::Local(_) => {
-                info!("Creating Local KMS backend");
+                info!("Creating Local KMS backend for version {}", version);
                let backend = LocalKmsBackend::new(config.clone()).await?;
-                Ok(Arc::new(backend))
+                Arc::new(backend) as Arc<dyn KmsBackend>
            }
            BackendConfig::Vault(_) => {
-                info!("Creating Vault KMS backend");
+                info!("Creating Vault KMS backend for version {}", version);
                let backend = crate::backends::vault::VaultKmsBackend::new(config.clone()).await?;
-                Ok(Arc::new(backend))
+                Arc::new(backend) as Arc<dyn KmsBackend>
            }
-        }
+        };
+
+        // Create KMS manager
+        let kms_manager = Arc::new(KmsManager::new(backend, config.clone()));
+
+        // Create encryption service
+        let encryption_service = Arc::new(ObjectEncryptionService::new((*kms_manager).clone()));
+
+        Ok(ServiceVersion {
+            version,
+            service: encryption_service,
+            manager: kms_manager,
+        })
    }
 }

@@ -22,7 +22,7 @@ use zeroize::Zeroize;

 /// Data encryption key (DEK) used for encrypting object data
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct DataKey {
+pub struct DataKeyInfo {
    /// Key identifier
    pub key_id: String,
    /// Key version
@@ -40,7 +40,7 @@ pub struct DataKey {
    pub created_at: Zoned,
 }

-impl DataKey {
+impl DataKeyInfo {
    /// Create a new data key
    ///
    /// # Arguments
@@ -96,7 +96,7 @@ impl DataKey {

 /// Master key stored in KMS backend
 #[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct MasterKey {
+pub struct MasterKeyInfo {
    /// Unique key identifier
    pub key_id: String,
    /// Key version
@@ -119,7 +119,7 @@ pub struct MasterKey {
    pub created_by: Option<String>,
 }

-impl MasterKey {
+impl MasterKeyInfo {
    /// Create a new master key
    ///
    /// # Arguments
@@ -226,8 +226,8 @@ pub struct KeyInfo {
    pub created_by: Option<String>,
 }

-impl From<MasterKey> for KeyInfo {
-    fn from(master_key: MasterKey) -> Self {
+impl From<MasterKeyInfo> for KeyInfo {
+    fn from(master_key: MasterKeyInfo) -> Self {
        Self {
            key_id: master_key.key_id,
            description: master_key.description,
@@ -913,7 +913,7 @@ pub struct CancelKeyDeletionResponse {
 }

 // SECURITY: Implement Drop to automatically zero sensitive data when DataKey is dropped
-impl Drop for DataKey {
+impl Drop for DataKeyInfo {
    fn drop(&mut self) {
        self.clear_plaintext();
    }
@@ -0,0 +1,38 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[package]
+name = "rustfs-metrics"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+homepage.workspace = true
+description.workspace = true
+keywords.workspace = true
+categories.workspace = true
+authors.workspace = true
+
+[dependencies]
+rustfs-ecstore = { workspace = true }
+rustfs-utils = { workspace = true }
+metrics = { workspace = true }
+sysinfo = { workspace = true }
+tokio = { workspace = true, features = ["rt-multi-thread", "rt", "time", "macros"] }
+tokio-util = { workspace = true }
+tracing = { workspace = true }
+
+[lints]
+workspace = true
@@ -0,0 +1,201 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Per-bucket metrics collector.
+//!
+//! Collects usage metrics for each bucket in the cluster, including
+//! size, object counts, and quota information.
+
+use crate::MetricType;
+use crate::format::PrometheusMetric;
+use std::borrow::Cow;
+
+/// Usage statistics for a single bucket.
+#[derive(Debug, Clone, Default)]
+pub struct BucketStats {
+    /// Bucket name
+    pub name: String,
+    /// Total bytes used by the bucket
+    pub size_bytes: u64,
+    /// Total number of objects in the bucket
+    pub objects_count: u64,
+    /// Quota limit in bytes (0 means no quota)
+    pub quota_bytes: u64,
+}
+
+// Static metric definitions
+const METRIC_SIZE: &str = "rustfs_bucket_usage_bytes";
+const METRIC_OBJECTS: &str = "rustfs_bucket_objects_total";
+const METRIC_QUOTA: &str = "rustfs_bucket_quota_bytes";
+
+const HELP_SIZE: &str = "Total bytes used by the bucket";
+const HELP_OBJECTS: &str = "Total number of objects in the bucket";
+const HELP_QUOTA: &str = "Quota limit in bytes for the bucket";
+
+/// Collects per-bucket usage metrics from the provided bucket statistics.
+///
+/// # Metrics Produced
+///
+/// For each bucket, the following metrics are produced with a `bucket` label:
+///
+/// - `rustfs_bucket_usage_bytes`: Total bytes used by the bucket
+/// - `rustfs_bucket_objects_total`: Total number of objects in the bucket
+/// - `rustfs_bucket_quota_bytes`: Quota limit in bytes (0 if no quota configured)
+///
+/// # Arguments
+///
+/// * `buckets` - Slice of bucket statistics
+///
+/// # Example
+///
+/// ```
+/// use rustfs_metrics::collectors::{collect_bucket_metrics, BucketStats};
+///
+/// let buckets = vec![
+///     BucketStats {
+///         name: "my-bucket".to_string(),
+///         size_bytes: 1_000_000,
+///         objects_count: 100,
+///         quota_bytes: 10_000_000,
+///     },
+/// ];
+/// let metrics = collect_bucket_metrics(&buckets);
+/// assert_eq!(metrics.len(), 3); // size, objects, quota
+/// ```
+#[must_use]
+#[inline]
+pub fn collect_bucket_metrics(buckets: &[BucketStats]) -> Vec<PrometheusMetric> {
+    if buckets.is_empty() {
+        return Vec::new();
+    }
+
+    let mut metrics = Vec::with_capacity(buckets.len() * 3);
+
+    for bucket in buckets {
+        let bucket_label: Cow<'static, str> = Cow::Owned(bucket.name.clone());
+
+        // Bucket size in bytes
+        metrics.push(
+            PrometheusMetric::new(METRIC_SIZE, MetricType::Gauge, HELP_SIZE, bucket.size_bytes as f64)
+                .with_label("bucket", bucket_label.clone()),
+        );
+
+        // Object count
+        metrics.push(
+            PrometheusMetric::new(METRIC_OBJECTS, MetricType::Gauge, HELP_OBJECTS, bucket.objects_count as f64)
+                .with_label("bucket", bucket_label.clone()),
+        );
+
+        // Quota (always emit, 0 when no quota configured for consistent PromQL queries)
+        metrics.push(
+            PrometheusMetric::new(METRIC_QUOTA, MetricType::Gauge, HELP_QUOTA, bucket.quota_bytes as f64)
+                .with_label("bucket", bucket_label),
+        );
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::format::report_metrics;
+
+    #[test]
+    fn test_collect_bucket_metrics() {
+        let buckets = vec![
+            BucketStats {
+                name: "test-bucket".to_string(),
+                size_bytes: 1000,
+                objects_count: 50,
+                quota_bytes: 0,
+            },
+            BucketStats {
+                name: "other-bucket".to_string(),
+                size_bytes: 2000,
+                objects_count: 100,
+                quota_bytes: 0,
+            },
+        ];
+
+        let metrics = collect_bucket_metrics(&buckets);
+        report_metrics(&metrics); // This will compile and run, but we can't easily assert on the global recorder state here.
+
+        // 2 buckets * 3 metrics each (size, objects, quota) = 6 metrics
+        assert_eq!(metrics.len(), 6);
+
+        // Verify test-bucket metrics
+        let test_bucket_size = metrics
+            .iter()
+            .find(|m| m.name == METRIC_SIZE && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "test-bucket"));
+        assert!(test_bucket_size.is_some());
+        assert_eq!(test_bucket_size.map(|m| m.value), Some(1000.0));
+    }
+
+    #[test]
+    fn test_collect_bucket_metrics_with_quotas() {
+        let buckets = vec![BucketStats {
+            name: "quota-bucket".to_string(),
+            size_bytes: 500,
+            objects_count: 10,
+            quota_bytes: 10000,
+        }];
+
+        let metrics = collect_bucket_metrics(&buckets);
+        report_metrics(&metrics);
+
+        // 1 bucket * 3 metrics (size, objects, quota) = 3 metrics
+        assert_eq!(metrics.len(), 3);
+
+        // Verify quota metric exists
+        let quota_metric = metrics.iter().find(|m| m.name == METRIC_QUOTA);
+        assert!(quota_metric.is_some());
+        assert_eq!(quota_metric.map(|m| m.value), Some(10000.0));
+    }
+
+    #[test]
+    fn test_collect_bucket_metrics_empty() {
+        let buckets: Vec<BucketStats> = vec![];
+        let metrics = collect_bucket_metrics(&buckets);
+        assert!(metrics.is_empty());
+    }
+
+    #[test]
+    fn test_collect_bucket_metrics_zero_quota_always_reported() {
+        let buckets = vec![BucketStats {
+            name: "no-quota-bucket".to_string(),
+            size_bytes: 100,
+            objects_count: 5,
+            quota_bytes: 0,
+        }];
+
+        let metrics = collect_bucket_metrics(&buckets);
+        report_metrics(&metrics);
+
+        // Zero quota should still produce a quota metric with value 0 for consistent PromQL queries
+        assert_eq!(metrics.len(), 3);
+        let quota_metric = metrics.iter().find(|m| m.name == METRIC_QUOTA);
+        assert!(quota_metric.is_some());
+        assert_eq!(quota_metric.map(|m| m.value), Some(0.0));
+    }
+
+    #[test]
+    fn test_bucket_stats_default() {
+        let stats = BucketStats::default();
+        assert!(stats.name.is_empty());
+        assert_eq!(stats.size_bytes, 0);
+        assert_eq!(stats.objects_count, 0);
+        assert_eq!(stats.quota_bytes, 0);
+    }
+}
@@ -0,0 +1,196 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Cluster-wide metrics collector.
+//!
+//! Collects aggregate metrics across the entire RustFS cluster including
+//! total capacity, usage, and object counts.
+
+use crate::MetricType;
+use crate::format::PrometheusMetric;
+
+/// Cluster capacity and usage statistics for metrics collection.
+///
+/// This struct provides a decoupled interface for collecting cluster metrics
+/// without depending on specific internal types. HTTP handlers should populate
+/// this struct from their available data sources.
+#[derive(Debug, Clone, Default)]
+pub struct ClusterStats {
+    /// Total raw storage capacity across all disks in bytes
+    pub raw_capacity_bytes: u64,
+    /// Usable capacity after erasure coding overhead in bytes
+    pub usable_capacity_bytes: u64,
+    /// Currently used storage in bytes
+    pub used_bytes: u64,
+    /// Available free storage in bytes
+    pub free_bytes: u64,
+    /// Total number of objects in the cluster
+    pub objects_count: u64,
+    /// Total number of buckets in the cluster
+    pub buckets_count: u64,
+}
+
+// Static metric definitions to avoid allocations
+const METRIC_RAW_CAPACITY: &str = "rustfs_cluster_capacity_raw_total_bytes";
+const METRIC_USABLE_CAPACITY: &str = "rustfs_cluster_capacity_usable_total_bytes";
+const METRIC_USED: &str = "rustfs_cluster_capacity_used_bytes";
+const METRIC_FREE: &str = "rustfs_cluster_capacity_free_bytes";
+const METRIC_OBJECTS: &str = "rustfs_cluster_objects_total";
+const METRIC_BUCKETS: &str = "rustfs_cluster_buckets_total";
+
+const HELP_RAW_CAPACITY: &str = "Total raw storage capacity in bytes across all disks";
+const HELP_USABLE_CAPACITY: &str = "Total usable storage capacity in bytes (accounting for erasure coding)";
+const HELP_USED: &str = "Total used storage capacity in bytes";
+const HELP_FREE: &str = "Total free storage capacity in bytes";
+const HELP_OBJECTS: &str = "Total number of objects in the cluster";
+const HELP_BUCKETS: &str = "Total number of buckets in the cluster";
+
+/// Number of metrics produced by this collector.
+const METRIC_COUNT: usize = 6;
+
+/// Collects cluster-wide metrics from the provided statistics.
+///
+/// # Metrics Produced
+///
+/// - `rustfs_cluster_capacity_raw_total_bytes`: Total raw storage capacity across all disks
+/// - `rustfs_cluster_capacity_usable_total_bytes`: Usable capacity after erasure coding overhead
+/// - `rustfs_cluster_capacity_used_bytes`: Currently used storage capacity
+/// - `rustfs_cluster_capacity_free_bytes`: Available free storage capacity
+/// - `rustfs_cluster_objects_total`: Total number of objects in the cluster
+/// - `rustfs_cluster_buckets_total`: Total number of buckets in the cluster
+///
+/// # Arguments
+///
+/// * `stats` - Cluster statistics containing capacity and usage data
+///
+/// # Example
+///
+/// ```
+/// use rustfs_metrics::collectors::{collect_cluster_metrics, ClusterStats};
+///
+/// let stats = ClusterStats {
+///     raw_capacity_bytes: 10_000_000_000,
+///     usable_capacity_bytes: 8_000_000_000,
+///     used_bytes: 2_000_000_000,
+///     free_bytes: 6_000_000_000,
+///     objects_count: 1000,
+///     buckets_count: 10,
+/// };
+/// let metrics = collect_cluster_metrics(&stats);
+/// assert_eq!(metrics.len(), 6);
+/// ```
+#[must_use]
+#[inline]
+pub fn collect_cluster_metrics(stats: &ClusterStats) -> Vec<PrometheusMetric> {
+    let mut metrics = Vec::with_capacity(METRIC_COUNT);
+
+    metrics.push(PrometheusMetric::new(
+        METRIC_RAW_CAPACITY,
+        MetricType::Gauge,
+        HELP_RAW_CAPACITY,
+        stats.raw_capacity_bytes as f64,
+    ));
+    metrics.push(PrometheusMetric::new(
+        METRIC_USABLE_CAPACITY,
+        MetricType::Gauge,
+        HELP_USABLE_CAPACITY,
+        stats.usable_capacity_bytes as f64,
+    ));
+    metrics.push(PrometheusMetric::new(METRIC_USED, MetricType::Gauge, HELP_USED, stats.used_bytes as f64));
+    metrics.push(PrometheusMetric::new(METRIC_FREE, MetricType::Gauge, HELP_FREE, stats.free_bytes as f64));
+    metrics.push(PrometheusMetric::new(
+        METRIC_OBJECTS,
+        MetricType::Gauge,
+        HELP_OBJECTS,
+        stats.objects_count as f64,
+    ));
+    metrics.push(PrometheusMetric::new(
+        METRIC_BUCKETS,
+        MetricType::Gauge,
+        HELP_BUCKETS,
+        stats.buckets_count as f64,
+    ));
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::format::report_metrics;
+
+    #[test]
+    fn test_collect_cluster_metrics() {
+        let stats = ClusterStats {
+            raw_capacity_bytes: 3000,
+            usable_capacity_bytes: 2500,
+            used_bytes: 1200,
+            free_bytes: 1300,
+            objects_count: 100,
+            buckets_count: 5,
+        };
+
+        let metrics = collect_cluster_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 6);
+
+        // Verify raw capacity
+        let raw_capacity = metrics.iter().find(|m| m.name == METRIC_RAW_CAPACITY);
+        assert!(raw_capacity.is_some());
+        assert_eq!(raw_capacity.map(|m| m.value), Some(3000.0));
+
+        // Verify used capacity
+        let used = metrics.iter().find(|m| m.name == METRIC_USED);
+        assert!(used.is_some());
+        assert_eq!(used.map(|m| m.value), Some(1200.0));
+
+        // Verify object count
+        let objects = metrics.iter().find(|m| m.name == METRIC_OBJECTS);
+        assert!(objects.is_some());
+        assert_eq!(objects.map(|m| m.value), Some(100.0));
+
+        // Verify bucket count
+        let buckets = metrics.iter().find(|m| m.name == METRIC_BUCKETS);
+        assert!(buckets.is_some());
+        assert_eq!(buckets.map(|m| m.value), Some(5.0));
+    }
+
+    #[test]
+    fn test_collect_cluster_metrics_empty() {
+        let stats = ClusterStats::default();
+
+        let metrics = collect_cluster_metrics(&stats);
+        report_metrics(&metrics);
+
+        assert_eq!(metrics.len(), 6);
+
+        // All values should be zero
+        for metric in &metrics {
+            assert_eq!(metric.value, 0.0);
+            assert!(metric.labels.is_empty());
+        }
+    }
+
+    #[test]
+    fn test_cluster_stats_default() {
+        let stats = ClusterStats::default();
+        assert_eq!(stats.raw_capacity_bytes, 0);
+        assert_eq!(stats.usable_capacity_bytes, 0);
+        assert_eq!(stats.used_bytes, 0);
+        assert_eq!(stats.free_bytes, 0);
+        assert_eq!(stats.objects_count, 0);
+        assert_eq!(stats.buckets_count, 0);
+    }
+}
@@ -0,0 +1,316 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::collectors::{
+    BucketStats, ClusterStats, DiskStats, ResourceStats, collect_bucket_metrics, collect_cluster_metrics, collect_node_metrics,
+    collect_resource_metrics,
+};
+use crate::constants::{
+    DEFAULT_BUCKET_METRICS_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL,
+    DEFAULT_RESOURCE_METRICS_INTERVAL, ENV_BUCKET_METRICS_INTERVAL, ENV_CLUSTER_METRICS_INTERVAL, ENV_DEFAULT_METRICS_INTERVAL,
+    ENV_NODE_METRICS_INTERVAL, ENV_RESOURCE_METRICS_INTERVAL,
+};
+use crate::format::report_metrics;
+use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
+use rustfs_ecstore::data_usage::load_data_usage_from_backend;
+use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
+use rustfs_ecstore::store_api::BucketOptions;
+use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
+use rustfs_utils::get_env_opt_u64;
+use std::sync::OnceLock;
+use std::time::{Duration, Instant};
+use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System};
+use tokio_util::sync::CancellationToken;
+use tracing::warn;
+
+/// Process start time for calculating uptime.
+static PROCESS_START: OnceLock<Instant> = OnceLock::new();
+
+/// Get the process start time, initializing it on first call.
+#[inline]
+fn get_process_start() -> &'static Instant {
+    PROCESS_START.get_or_init(Instant::now)
+}
+
+/// Collect cluster statistics from the storage layer.
+async fn collect_cluster_stats() -> ClusterStats {
+    let Some(store) = new_object_layer_fn() else {
+        return ClusterStats::default();
+    };
+
+    let storage_info = store.storage_info().await;
+
+    let raw_capacity: u64 = storage_info.disks.iter().map(|d| d.total_space).sum();
+    let used: u64 = storage_info.disks.iter().map(|d| d.used_space).sum();
+    let usable_capacity = get_total_usable_capacity(&storage_info.disks, &storage_info) as u64;
+    let free = get_total_usable_capacity_free(&storage_info.disks, &storage_info) as u64;
+
+    // Get bucket and object counts from data usage info
+    let (buckets_count, objects_count) = match load_data_usage_from_backend(store.clone()).await {
+        Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count),
+        Err(e) => {
+            warn!("Failed to load data usage from backend: {}", e);
+            // Fall back to bucket list for buckets_count, objects_count stays 0
+            let buckets = store
+                .list_bucket(&BucketOptions {
+                    cached: true,
+                    ..Default::default()
+                })
+                .await
+                .unwrap_or_else(|e| {
+                    warn!("Failed to list buckets for cluster metrics: {}", e);
+                    Vec::new()
+                });
+            (buckets.len() as u64, 0)
+        }
+    };
+
+    ClusterStats {
+        raw_capacity_bytes: raw_capacity,
+        usable_capacity_bytes: usable_capacity,
+        used_bytes: used,
+        free_bytes: free,
+        objects_count,
+        buckets_count,
+    }
+}
+
+/// Collect bucket statistics from the storage layer.
+async fn collect_bucket_stats() -> Vec<BucketStats> {
+    let Some(store) = new_object_layer_fn() else {
+        return Vec::new();
+    };
+
+    // Load data usage info from backend to get bucket sizes and object counts
+    let data_usage = match load_data_usage_from_backend(store.clone()).await {
+        Ok(info) => Some(info),
+        Err(e) => {
+            warn!("Failed to load data usage from backend for bucket metrics: {}", e);
+            None
+        }
+    };
+
+    let buckets = match store
+        .list_bucket(&BucketOptions {
+            cached: true,
+            ..Default::default()
+        })
+        .await
+    {
+        Ok(b) => b,
+        Err(e) => {
+            warn!("Failed to list buckets for metrics: {}", e);
+            return Vec::new();
+        }
+    };
+
+    // Build bucket stats with real data from DataUsageInfo
+    let mut stats = Vec::with_capacity(buckets.len());
+    for bucket in buckets {
+        if bucket.name.starts_with('.') {
+            continue;
+        }
+
+        // Get size and objects_count from data usage info
+        let (size_bytes, objects_count) = data_usage
+            .as_ref()
+            .and_then(|du| du.buckets_usage.get(&bucket.name))
+            .map(|bui| (bui.size, bui.objects_count))
+            .unwrap_or((0, 0));
+
+        // Get quota from bucket metadata
+        let quota_bytes = match get_quota_config(&bucket.name).await {
+            Ok((quota, _)) => quota.get_quota_limit().unwrap_or(0),
+            Err(_) => 0, // No quota configured or error
+        };
+
+        stats.push(BucketStats {
+            name: bucket.name,
+            size_bytes,
+            objects_count,
+            quota_bytes,
+        });
+    }
+
+    stats
+}
+
+/// Collect disk statistics from the storage layer.
+async fn collect_disk_stats() -> Vec<DiskStats> {
+    let Some(store) = new_object_layer_fn() else {
+        return Vec::new();
+    };
+
+    let storage_info = store.storage_info().await;
+
+    storage_info
+        .disks
+        .iter()
+        .map(|disk| DiskStats {
+            server: disk.endpoint.clone(),
+            drive: disk.drive_path.clone(),
+            total_bytes: disk.total_space,
+            used_bytes: disk.used_space,
+            free_bytes: disk.available_space,
+        })
+        .collect()
+}
+
+/// Collect resource statistics for the current process.
+///
+/// Collects:
+/// - Uptime: Calculated from process start time
+/// - Memory: Process resident set size from sysinfo
+/// - CPU: Process CPU usage percentage from sysinfo
+#[inline]
+fn collect_process_stats() -> ResourceStats {
+    let uptime_seconds = get_process_start().elapsed().as_secs();
+
+    // Use sysinfo for process metrics
+    let mut sys = System::new();
+    let pid = Pid::from_u32(std::process::id());
+    sys.refresh_processes_specifics(
+        ProcessesToUpdate::Some(&[pid]),
+        true,
+        ProcessRefreshKind::nothing().with_cpu().with_memory(),
+    );
+
+    if let Some(process) = sys.process(pid) {
+        ResourceStats {
+            cpu_percent: process.cpu_usage() as f64,
+            memory_bytes: process.memory(),
+            uptime_seconds,
+        }
+    } else {
+        // Fallback if process not found
+        ResourceStats {
+            cpu_percent: 0.0,
+            memory_bytes: 0,
+            uptime_seconds,
+        }
+    }
+}
+
+/// Initialize the metrics collection system with periodic background tasks for cluster, bucket, node, and resource metrics.
+///
+/// This function spawns background tasks that periodically collect metrics
+/// and report them using the `metrics` crate.
+///
+/// # Arguments
+///
+/// * `token` - A cancellation token to gracefully stop the metrics collection tasks.
+pub fn init_metrics_collectors(token: CancellationToken) {
+    // Initialize process start time
+    get_process_start();
+
+    // Helper closure to determine interval for a specific metric type
+    let get_interval = |env_key: &str, type_default: Duration| -> Duration {
+        // 1. Try specific env var
+        // 2. Fallback to global default env var (if set differently from hardcoded default)
+        // 3. Fallback to type specific default
+
+        // Helper to check if value is valid (non-zero)
+        let is_valid = |v: u64| v > 0;
+
+        if let Some(val) = get_env_opt_u64(env_key).filter(|&v| is_valid(v)) {
+            Duration::from_secs(val)
+        } else if let Some(val) = get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL).filter(|&v| is_valid(v)) {
+            Duration::from_secs(val)
+        } else {
+            type_default
+        }
+    };
+
+    let cluster_interval = get_interval(ENV_CLUSTER_METRICS_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL);
+    let bucket_interval = get_interval(ENV_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL);
+    let node_interval = get_interval(ENV_NODE_METRICS_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL);
+    let resource_interval = get_interval(ENV_RESOURCE_METRICS_INTERVAL, DEFAULT_RESOURCE_METRICS_INTERVAL);
+
+    // Spawn task for cluster metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(cluster_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let stats = collect_cluster_stats().await;
+                    let metrics = collect_cluster_metrics(&stats);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for cluster stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
+    // Spawn task for bucket metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(bucket_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let stats = collect_bucket_stats().await;
+                    let metrics = collect_bucket_metrics(&stats);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for bucket stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
+    // Spawn task for node/disk metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(node_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let stats = collect_disk_stats().await;
+                    let metrics = collect_node_metrics(&stats);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for node/disk stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
+    // Spawn task for resource metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(resource_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    // Resource stats collection is synchronous but fast
+                    let stats = collect_process_stats();
+                    let metrics = collect_resource_metrics(&stats);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for resource stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+}
@@ -0,0 +1,73 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Prometheus metric collectors for RustFS.
+//!
+//! This module provides collectors that convert RustFS data into Prometheus
+//! metrics format. Each collector is responsible for a specific domain:
+//!
+//! - [`cluster`]: Cluster-wide capacity and object statistics
+//! - [`bucket`]: Per-bucket usage and quota metrics
+//! - [`node`]: Per-node disk capacity and health metrics
+//! - [`resource`]: System resource metrics (CPU, memory, uptime)
+//!
+//! # Design Philosophy
+//!
+//! Collectors accept simple data structs rather than internal RustFS types.
+//! This design allows HTTP handlers to populate the structs from their
+//! available data sources without creating circular dependencies.
+//!
+//! # Example
+//!
+//! ```
+//! use rustfs_metrics::collectors::{
+//!     collect_cluster_metrics, ClusterStats,
+//!     collect_bucket_metrics, BucketStats,
+//!     collect_node_metrics, DiskStats,
+//!     collect_resource_metrics, ResourceStats,
+//! };
+//! use rustfs_metrics::report_metrics;
+//!
+//! // Collect cluster metrics
+//! let cluster_stats = ClusterStats {
+//!     raw_capacity_bytes: 1_000_000_000,
+//!     used_bytes: 500_000_000,
+//!     ..Default::default()
+//! };
+//! let mut metrics = collect_cluster_metrics(&cluster_stats);
+//!
+//! // Add bucket metrics
+//! let bucket_stats = vec![BucketStats {
+//!     name: "my-bucket".to_string(),
+//!     size_bytes: 100_000,
+//!     objects_count: 50,
+//!     ..Default::default()
+//! }];
+//! metrics.extend(collect_bucket_metrics(&bucket_stats));
+//!
+//! // Report to metrics system
+//! report_metrics(&metrics);
+//! ```
+
+mod bucket;
+mod cluster;
+pub(crate) mod global;
+mod node;
+mod resource;
+
+pub use bucket::{BucketStats, collect_bucket_metrics};
+pub use cluster::{ClusterStats, collect_cluster_metrics};
+pub use global::init_metrics_collectors;
+pub use node::{DiskStats, collect_node_metrics};
+pub use resource::{ResourceStats, collect_resource_metrics};
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
evan slack GitHub loverustfs	9824171995	feat(observability): Add grafana dashboard, observability changes (#1770 ) Co-authored-by: loverustfs <[email protected]>	2026-02-11 15:55:08 +08:00
heihutu GitHub houseme	ecceb8fd1c	build(deps): bump the dependencies group with 2+ updates (#1769 ) Co-authored-by: houseme <[email protected]>	2026-02-11 09:39:47 +08:00
TyooughtulandGitHub	1184806c3f	Fix/resolve pr 1710 (#1743 )	2026-02-11 08:24:55 +08:00
housemeandGitHub	4411c625e2	feat(metrics): async collection with configurable intervals & graceful shutdown (#1768 )	2026-02-10 21:37:24 +08:00
Md. Amdadul Bari Imad GitHub houseme	c07ed61989	fix(entrypoint): remove dead HTTP URL check in volume filtering (#1761 ) Co-authored-by: houseme <[email protected]>	2026-02-10 12:07:08 +08:00
yxrxy GitHub houseme heihutu	da58f8e291	fix: Allow non-admin users to read bucket quota configuration. (#1759 ) Co-authored-by: houseme <[email protected]> Co-authored-by: heihutu <[email protected]>	2026-02-10 11:23:21 +08:00
mengyu-sxyz GitHub loverustfs houseme heihutu	aa011ade19	fix: improve part size calculation in optimal_part_info function (#1532 ) Signed-off-by: mengyu-sxyz <[email protected]> Co-authored-by: loverustfs <[email protected]> Co-authored-by: houseme <[email protected]> Co-authored-by: heihutu <[email protected]>	2026-02-10 10:56:47 +08:00
LoganZ2 GitHub houseme Copilot	ccf3b29df5	fix: stabilize head metadata responses and heal tests (#1732 ) Signed-off-by: LoganZ2 <[email protected]> Co-authored-by: houseme <[email protected]> Co-authored-by: Copilot <[email protected]>	2026-02-10 09:44:14 +08:00
evan slack GitHub loverustfs Copilot houseme	682b5bbb2f	perf(scanner): Change DataUseageEntry from clone to borrow (#1757 ) Signed-off-by: evan slack <[email protected]> Signed-off-by: houseme <[email protected]> Co-authored-by: loverustfs <[email protected]> Co-authored-by: Copilot <[email protected]> Co-authored-by: houseme <[email protected]>	2026-02-10 08:53:26 +08:00
LeonWang0735 GitHub loverustfs	f4e9ef2edc	fix(replication): avoid re-replication loop in Active-Active replication (#1751 ) Co-authored-by: loverustfs <[email protected]>	2026-02-09 14:11:30 +08:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>houseme	ff8c1c782a	build(deps): bump libunftp from 0.21.0 to 0.22.0 in the dependencies group across 1 directory (#1756 ) Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: houseme <[email protected]>	2026-02-09 13:43:47 +08:00
housemeandGitHub	ca6076fe18	build(deps): bump the dependencies group with 5 updates (#1755 )	2026-02-09 11:31:43 +08:00
evan slackandGitHub	58ee140324	perf(regex): Compile bucket validation regex once (#1753 )	2026-02-09 10:50:17 +08:00
thorntonmc GitHub houseme	927f3a57d7	perf(quota): Skip expensive usage checks when no quota configured (#1749 ) Co-authored-by: houseme <[email protected]>	2026-02-08 22:37:53 +08:00
majingheandGitHub	a574285ab2	feat: add support for mtls with kubernetes installation (#1741 )	2026-02-08 09:31:58 +08:00
LeonWang0735 GitHub houseme Copilot	60793c17d7	fix: persist replication status and timestamp after replicate_object (#1747 ) Signed-off-by: houseme <[email protected]> Co-authored-by: houseme <[email protected]> Co-authored-by: Copilot <[email protected]>	2026-02-07 12:50:37 +08:00
housemeandGitHub	0b870d6301	build(deps): bump the dependencies group with 19 updates (#1745 )	2026-02-07 12:22:14 +08:00
Isaac Mills GitHub loverustfs Copilot houseme	d635ee8d2e	Propogate tracing context from HTTP requests into spans (#1739 ) Signed-off-by: Isaac Mills <[email protected]> Signed-off-by: houseme <[email protected]> Co-authored-by: loverustfs <[email protected]> Co-authored-by: Copilot <[email protected]> Co-authored-by: houseme <[email protected]>	2026-02-07 11:52:31 +08:00
yxrxy GitHub houseme heihutu loverustfs	5c2eda356e	feat: migrate FTP/SFTP to protocols crate and update dependencies (#1580 ) Signed-off-by: yxrxy <[email protected]> Signed-off-by: houseme <[email protected]> Signed-off-by: heihutu <[email protected]> Co-authored-by: houseme <[email protected]> Co-authored-by: heihutu <[email protected]> Co-authored-by: loverustfs <[email protected]>	2026-02-06 20:58:42 +08:00
LeonWang0735 GitHub houseme loverustfs	a2b88a79ec	test(e2e_test): add automated cluster environment for conditional PUT race test (#1673 ) Co-authored-by: houseme <[email protected]> Co-authored-by: loverustfs <[email protected]>	2026-02-06 15:01:59 +08:00
LeonWang0735 GitHub houseme	6eb90e7df9	fix: fetch_owner set to true when calling list_objects_v2 in the list_objects function (#1730 ) Co-authored-by: houseme <[email protected]>	2026-02-05 22:48:32 +08:00
majinghe GitHub houseme	466429b958	feat: add contour as ingress controller with http proxy (#1729 ) Co-authored-by: houseme <[email protected]>	2026-02-05 21:04:18 +08:00
GatewayJandGitHub	c8411fd62c	fix: bucket policy id field serde (#1726 )	2026-02-05 18:38:57 +08:00
houseme GitHub Copilot	6bba41f11f	Fix/fix issues #1564 (#1708 ) Signed-off-by: houseme <[email protected]> Co-authored-by: Copilot <[email protected]>	2026-02-05 13:45:14 +08:00
majingheandGitHub	e30781654d	enhancement: add support for http to https redirect for traefik gatew… (#1712 )	2026-02-04 20:21:42 +08:00
唐小鸭 GitHub houseme	7a42af922d	Refactor: refactor SSE layer and KMS subsystem (#1703 ) Co-authored-by: houseme <[email protected]>	2026-02-04 16:10:33 +08:00
weisdandGitHub	4d19b069c3	fix: replication delete (#1714 )	2026-02-04 13:39:35 +08:00
loverustfsandGitHub	a4563f7b41	fix: return null versionId when suspended (#1066 ) (#1709 )	2026-02-04 09:22:52 +08:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	174e12bf66	build(deps): bump bytes from 1.11.0 to 1.11.1 (#1711 ) Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2026-02-04 08:41:30 +08:00
majingheandGitHub	f03034b99e	feat: add glibc based docker image support (#1705 ) Signed-off-by: majinghe <[email protected]>	2026-02-03 21:27:20 +08:00
LeonWang0735 GitHub loverustfs	36f14acbe9	fix: object lock compliance mode allows deletion (#1687 ) Co-authored-by: loverustfs <[email protected]>	2026-02-03 17:06:24 +08:00
majingheandGitHub	2f66f15524	feat: add obs log rotations environment variables (#1702 )	2026-02-03 10:35:30 +08:00
houseme GitHub majinghe GatewayJ Copilot houseme heihutu	cb468fb32f	Refactor trusted-proxies: modernize utils, improve safety, and fix clippy lints (#1693 ) Co-authored-by: majinghe <[email protected]> Co-authored-by: GatewayJ <[email protected]> Co-authored-by: Copilot <[email protected]> Co-authored-by: houseme <[email protected]> Co-authored-by: heihutu <[email protected]>	2026-02-03 01:06:22 +08:00
housemeandGitHub	d1a70176a2	fix: Fixed that `account_id` returns normal value (#1695 )	2026-02-02 20:15:54 +08:00
LeonWang0735 GitHub loverustfs	ec4458f846	Fix/correctly handle terraform s3 backend with versioned bucket (#1686 ) Co-authored-by: loverustfs <[email protected]>	2026-02-02 12:43:07 +08:00
yxrxyandGitHub	00ccc19e27	Revert "fix: resolve Issue #1465 - IAM credential change crash (#1535 )" (#1685 )	2026-02-02 07:14:53 +08:00
houseme GitHub Copilot	07cf2feaad	fix(pprof): Fixed the problem that `pprof` crate does not support the window platform (#1681 ) Signed-off-by: houseme <[email protected]> Co-authored-by: Copilot <[email protected]>	2026-02-01 00:12:55 +08:00
likewu GitHub loverustfs	087f58b7c8	fix(lifecycle): lifecycle fixes (#1625 ) Signed-off-by: likewu <[email protected]> Co-authored-by: loverustfs <[email protected]>	2026-01-31 19:14:47 +08:00
moechs GitHub houseme	6fc35e442c	fix: add gatewayApi.enabled check to TraefikService template (#1679 ) Signed-off-by: moechs <[email protected]> Co-authored-by: houseme <[email protected]>	2026-01-31 17:42:27 +08:00
majingheandGitHub	a798b20308	fix: init container security hardened for operation permission error (#1680 )	2026-01-31 17:00:42 +08:00
housemeandGitHub	38b779b924	feat(profiling): support cross-platform memory profiling with mimalloc and pprof (#1674 )	2026-01-30 22:23:49 +08:00