Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0f5efb47f7 | ||
|
|
c81267c600 | ||
|
|
1b34bf76eb | ||
|
|
c8fe9ff345 |
@@ -50,11 +50,10 @@ consider adding it to the script's `checked_files` list.
|
||||
|
||||
## `check_doc_paths.sh`
|
||||
|
||||
Instruction docs (`AGENTS.md`, `CLAUDE.md`, `ARCHITECTURE.md`) and every
|
||||
Markdown file under `docs/` (architecture, operations, testing, index) must not
|
||||
reference repo file paths that no longer exist. If your refactor moved code,
|
||||
update the docs that point at it — the error message lists `doc -> stale-path`
|
||||
pairs. Cite paths plus symbol names, never line numbers (see `docs/README.md`).
|
||||
Instruction/architecture docs (`AGENTS.md`, `CLAUDE.md`, `ARCHITECTURE.md`,
|
||||
`docs/architecture/*.md`) must not reference repo file paths that no longer
|
||||
exist. If your refactor moved code, update the docs that point at it — the
|
||||
error message lists `doc -> stale-path` pairs.
|
||||
|
||||
## `check_no_planning_docs.sh`
|
||||
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=9dccb0cd537cf79ae70c1c20e8281d36d03f2f09f81142a5341e26e3dc18709d
|
||||
sha256-darwin=ef914ec0b8daa9c2c5e52f501d339914662f42d6f6ed9d33877d56b97adf16f9
|
||||
sha256-linux=a8a816d7bb0e7cb5632b1863b33794bcb9fc7e765f150aa5e1bf16518e28dfb4
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=d06524b44de97ed8f62b0fd8cf9fa504e3cd520ffcaacc32691d6f890ebe7f20
|
||||
sha256=51da41c54167602f2bd6c45921b39a44562bf3cfcdf468d992bb992c62cad7fd
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=db9bd8cdcb0abe43461aa6b36499b17cabd4098e5b34e300b1a0f0d0f34d9884
|
||||
sha256=dbebfbab9b9efd4eff31211e69dd32235dc00e207f2ab0dd919a1b2ac9e724c2
|
||||
|
||||
@@ -355,8 +355,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
# allowlist", so any new replication test lands in nightly by default (never
|
||||
# silently unrun) until it is explicitly blessed as fast here. Keep the two
|
||||
# regexes byte-identical. The committed profile selection digests make changes
|
||||
# visible in CI; list current membership with `cargo nextest list -p e2e_test
|
||||
# --profile <profile>` (platform-dependent; see docs/testing/README.md).
|
||||
# visible in CI; current counts live in docs/testing/e2e-suite-inventory.md.
|
||||
# HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
|
||||
# (#4724) because they set a loopback (127.0.0.1) replication target that the
|
||||
# SSRF egress guard rejected on every PR after repl-1 (#4712). That is fixed —
|
||||
@@ -509,7 +508,7 @@ path = "junit.xml"
|
||||
# quota, checksum, encryption,
|
||||
# security-boundary, ... — that the fast PR `e2e-smoke` subset deliberately
|
||||
# skips. Budget <= 45 min; authority for the suite count is `cargo nextest list
|
||||
# --profile e2e-full -p e2e_test` (platform-dependent; see docs/testing/README.md).
|
||||
# --profile e2e-full` (see docs/testing/e2e-suite-inventory.md).
|
||||
#
|
||||
# The filter is "the whole e2e_test crate MINUS the sets owned by other lanes":
|
||||
# * protocols:: — FTPS/SFTP/WebDAV, run from the dedicated protocol profile
|
||||
|
||||
@@ -1065,7 +1065,7 @@ jobs:
|
||||
while IFS= read -r preview_tag; do
|
||||
[[ -n "$preview_tag" ]] || continue
|
||||
echo "🧹 Deleting preview release $preview_tag (tag kept)"
|
||||
gh release delete "$preview_tag" --repo "${GITHUB_REPOSITORY}" --yes
|
||||
gh release delete "$preview_tag" --yes
|
||||
DELETED=$((DELETED + 1))
|
||||
done < <(
|
||||
jq -r --arg tag "$TAG" '
|
||||
|
||||
@@ -12,10 +12,10 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Functional chain driver: runs the ten functional suites in a fixed order
|
||||
# (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
|
||||
# replication, with performance on its own runner in parallel) and guarantees
|
||||
# the chain keeps moving even when individual suites fail.
|
||||
# Functional chain driver: runs the nine functional suites in a fixed order
|
||||
# (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security, with
|
||||
# performance on its own runner in parallel) and guarantees the chain keeps
|
||||
# moving even when individual suites fail.
|
||||
#
|
||||
# Each suite workflow can still be dispatched standalone (workflow_dispatch);
|
||||
# only chain-triggered runs forward to the next suite via repository_dispatch,
|
||||
|
||||
@@ -284,52 +284,21 @@ jobs:
|
||||
|
||||
- name: "Continue functional chain (next: Pool expansion)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-pool' \
|
||||
-F 'client_payload[from_suite]=heal'; then
|
||||
echo "dispatched next suite Pool expansion (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Pool expansion after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after heal (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **heal** to **Pool expansion** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-pool'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-pool'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: Pool expansion"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-pool' \
|
||||
-F 'client_payload[from_suite]=heal'
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
@@ -340,52 +340,21 @@ jobs:
|
||||
|
||||
- name: "Continue functional chain (next: Tier)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-tier' \
|
||||
-F 'client_payload[from_suite]=kms'; then
|
||||
echo "dispatched next suite Tier (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Tier after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after kms (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **kms** to **Tier** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-tier'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-tier'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: Tier"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-tier' \
|
||||
-F 'client_payload[from_suite]=kms'
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
@@ -596,52 +596,21 @@ jobs:
|
||||
|
||||
- name: "Continue functional chain (next: Security)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-security' \
|
||||
-F 'client_payload[from_suite]=pool'; then
|
||||
echo "dispatched next suite Security (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Security after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after pool (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **pool** to **Security** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-security'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-security'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: Security"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-security' \
|
||||
-F 'client_payload[from_suite]=pool'
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
@@ -1,365 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: RustFS Replication Test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rustfs_version:
|
||||
description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
package_url:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
type: string
|
||||
suite:
|
||||
description: 'Suite to run (all = bucket REP-* then site SITE-*)'
|
||||
type: choice
|
||||
options:
|
||||
- all
|
||||
- bucket
|
||||
- site
|
||||
default: all
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the security suite finishes. This is the
|
||||
# last link of the functional chain.
|
||||
types: [rustfs-chain-replication]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# The replication suite uses the same shared VMs as the other functional
|
||||
# tests, so it must serialize with them instead of running in parallel.
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
replication-test:
|
||||
runs-on: smoke-testing
|
||||
# A failed replication run must not break the chain or the workflow: the
|
||||
# failure is reported to rustfs/backlog instead (see the issue step).
|
||||
continue-on-error: true
|
||||
timeout-minutes: 360
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
# auto-testing is private: clone it with the dedicated PF token (not
|
||||
# GITHUB_TOKEN) and retry transient GitHub/network failures.
|
||||
- name: Checkout auto-testing scripts (with retry)
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
rm -rf auto-testing
|
||||
for attempt in 1 2 3 4 5; do
|
||||
if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
|
||||
echo "auto-testing cloned (attempt ${attempt})"
|
||||
exit 0
|
||||
fi
|
||||
rm -rf auto-testing
|
||||
echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
|
||||
sleep $((attempt * 15))
|
||||
done
|
||||
echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
|
||||
exit 1
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
openssl version
|
||||
aws --version
|
||||
df -h /data | tail -1 || true
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs rustfs-rep2 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /data/rustfs-rep2 /var/log/rustfs /var/log/rustfs-rep2 /var/lib/rustfs/kms
|
||||
'
|
||||
done
|
||||
|
||||
- name: Run replication suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-replication.log
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-replication-test.sh
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
SUITE='${{ inputs.suite }}'
|
||||
ARGS=(-y --log-file "${LOG_FILE}")
|
||||
if [ "${SUITE}" = "all" ] || [ -z "${SUITE}" ] || [ "${SUITE}" = "null" ]; then
|
||||
ARGS+=(--suite all)
|
||||
else
|
||||
ARGS+=(--suite "${SUITE}")
|
||||
fi
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
ARGS+=(--package-url "${PACKAGE_URL}")
|
||||
elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
|
||||
ARGS+=(--version "${RUSTFS_VERSION}")
|
||||
else
|
||||
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
||||
fi
|
||||
./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-replication.log
|
||||
REPORT_FILE: /tmp/rustfs-replication-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
|
||||
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
RUSTFS_VERSION_INFO="N/A"
|
||||
if [ "${#NODES[@]}" -gt 0 ]; then
|
||||
DETECTED_VERSION="$(ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
|
||||
"${SSH_USER}@${NODES[0]}" 'rustfs --version' 2>/dev/null | tr -d '\r' | head -n 1 || true)"
|
||||
if [ -n "${DETECTED_VERSION}" ]; then
|
||||
RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
|
||||
fi
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-replication-cases.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file = sys.argv[1], sys.argv[2]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS replication test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-replication-report.md
|
||||
SUITE: replication
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
SUITE: 'replication'
|
||||
SUITE_LABEL: 'Replication (bucket + site)'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
REPORT_FILE: '/tmp/rustfs-replication-report.md'
|
||||
LOG_FILE: '/tmp/rustfs-replication.log'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
|
||||
exit 0
|
||||
fi
|
||||
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
|
||||
EXISTING="$(gh issue list -R rustfs/backlog --state all \
|
||||
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
|
||||
--json number --jq '.[].number' || true)"
|
||||
if [ -n "${EXISTING}" ]; then
|
||||
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
|
||||
exit 0
|
||||
fi
|
||||
redact() {
|
||||
sed -E \
|
||||
-e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
|
||||
-e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
|
||||
}
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The **${SUITE_LABEL}** functional suite failed."
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
else
|
||||
echo "(no report or log file was produced)"
|
||||
fi
|
||||
} | head -c 55000 > "${BODY_FILE}"
|
||||
gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
|
||||
if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test; then
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
|
||||
fi
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload report and logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-replication-${{ github.run_id }}
|
||||
path: |
|
||||
/tmp/rustfs-replication.log
|
||||
/tmp/rustfs-replication-report.md
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs rustfs-rep2 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /data/rustfs-rep2 /var/log/rustfs /var/log/rustfs-rep2
|
||||
'
|
||||
done
|
||||
|
||||
- name: Chain complete
|
||||
# Replication is the last link of the functional chain: nothing to
|
||||
# dispatch after it. This step just records that the chain finished.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
run: |
|
||||
echo "Functional chain complete: replication (final suite) finished."
|
||||
echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}"
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "RustFS replication suite failed"
|
||||
echo "Package source: ${{ inputs.package_url || inputs.rustfs_version || 'nightly (R2 latest)' }}"
|
||||
echo "See the uploaded report and log artifacts for details."
|
||||
@@ -320,52 +320,21 @@ jobs:
|
||||
|
||||
- name: "Continue functional chain (next: KMS)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-kms' \
|
||||
-F 'client_payload[from_suite]=s3'; then
|
||||
echo "dispatched next suite KMS (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch KMS after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after s3 (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **s3** to **KMS** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-kms'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-kms'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: KMS"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-kms' \
|
||||
-F 'client_payload[from_suite]=s3'
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
@@ -47,7 +47,7 @@ on:
|
||||
type: boolean
|
||||
default: true
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the pool expansion suite finishes.
|
||||
# Chain handoff: dispatched when the pool expansion suite finishes (last link).
|
||||
types: [rustfs-chain-security]
|
||||
|
||||
permissions:
|
||||
@@ -292,24 +292,6 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: "Continue functional chain (next: Replication)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: Replication"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-replication' \
|
||||
-F 'client_payload[from_suite]=security'
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
|
||||
@@ -335,52 +335,21 @@ jobs:
|
||||
|
||||
- name: "Continue functional chain (next: Heal)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-heal' \
|
||||
-F 'client_payload[from_suite]=storage'; then
|
||||
echo "dispatched next suite Heal (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Heal after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after storage (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **storage** to **Heal** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-heal'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-heal'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: Heal"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-heal' \
|
||||
-F 'client_payload[from_suite]=storage'
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
@@ -11,6 +11,10 @@ on:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
type: string
|
||||
package_sha256:
|
||||
description: 'Optional SHA-256 for package_url; mismatch is an infrastructure failure.'
|
||||
required: false
|
||||
type: string
|
||||
rc_sha256:
|
||||
description: 'Optional SHA-256 for the preinstalled rc binary; mismatch is an infrastructure failure.'
|
||||
required: false
|
||||
@@ -71,13 +75,20 @@ jobs:
|
||||
- name: Checkout auto-testing scripts (with retry)
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
AUTO_TESTING_REF: cxymds/fix-2132-tier-log-isolation
|
||||
AUTO_TESTING_COMMIT: 02da54dd62110649dc2860fc5fcd9e08d2e9a1ca
|
||||
run: |
|
||||
set -euo pipefail
|
||||
rm -rf auto-testing
|
||||
for attempt in 1 2 3 4 5; do
|
||||
if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
|
||||
echo "auto-testing cloned (attempt ${attempt})"
|
||||
exit 0
|
||||
if gh repo clone rustfs/auto-testing auto-testing -- \
|
||||
--branch "${AUTO_TESTING_REF}" --single-branch --depth 1 --quiet; then
|
||||
actual_commit="$(git -C auto-testing rev-parse HEAD)"
|
||||
if [[ "${actual_commit}" == "${AUTO_TESTING_COMMIT}" ]]; then
|
||||
echo "auto-testing ${actual_commit} cloned (attempt ${attempt})"
|
||||
exit 0
|
||||
fi
|
||||
echo "auto-testing commit mismatch: expected ${AUTO_TESTING_COMMIT}, got ${actual_commit}" >&2
|
||||
fi
|
||||
rm -rf auto-testing
|
||||
echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
|
||||
@@ -86,6 +97,39 @@ jobs:
|
||||
echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
|
||||
exit 1
|
||||
|
||||
- name: Download exact rc candidate
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
||||
with:
|
||||
repository: rustfs/rustfs-release-validation
|
||||
run-id: '33465191972'
|
||||
name: rc-under-test-33465191972-1
|
||||
path: ${{ runner.temp }}/issue-2128-rc
|
||||
github-token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Verify exact rc candidate
|
||||
env:
|
||||
RC_BIN: ${{ runner.temp }}/issue-2128-rc/rc
|
||||
RC_PROVENANCE: ${{ runner.temp }}/issue-2128-rc/rc-build.json
|
||||
RC_EXPECTED_COMMIT: f6b9b509a60ef172a2b037d638c2cac46e762129
|
||||
RC_EXPECTED_SHA256: 3d128d99f05403f4028c7c9ae24b03d66a3e98f2090e66cb7f9f45a9e11fdce1
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test -s "${RC_BIN}"
|
||||
test -s "${RC_PROVENANCE}"
|
||||
jq -e \
|
||||
--arg commit "${RC_EXPECTED_COMMIT}" \
|
||||
--arg digest "${RC_EXPECTED_SHA256}" \
|
||||
'.repository == "rustfs/cli"
|
||||
and .requestedCommit == $commit
|
||||
and .resolvedCommit == $commit
|
||||
and .binarySha256 == $digest
|
||||
and .target == "x86_64-unknown-linux-gnu"' \
|
||||
"${RC_PROVENANCE}" >/dev/null
|
||||
actual_sha256="$(sha256sum -- "${RC_BIN}" | awk '{print $1}')"
|
||||
test "${actual_sha256}" = "${RC_EXPECTED_SHA256}"
|
||||
chmod 0555 "${RC_BIN}"
|
||||
"${RC_BIN}" --version
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
@@ -146,13 +190,15 @@ jobs:
|
||||
continue-on-error: true
|
||||
env:
|
||||
PACKAGE_URL_INPUT: ${{ inputs.package_url }}
|
||||
PACKAGE_SHA256_INPUT: ${{ inputs.package_sha256 }}
|
||||
RUSTFS_VERSION_INPUT: ${{ inputs.rustfs_version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
LOG_FILE="${TIER_ARTIFACTS_DIR}/rustfs-tier.log"
|
||||
chmod +x auto-testing/rustfs-tier-test.sh
|
||||
RC_BIN="$(command -v rc)"
|
||||
RC_BIN="${RUNNER_TEMP}/issue-2128-rc/rc"
|
||||
PACKAGE_URL="${PACKAGE_URL_INPUT}"
|
||||
PACKAGE_SHA256="${PACKAGE_SHA256_INPUT}"
|
||||
RUSTFS_VERSION="${RUSTFS_VERSION_INPUT}"
|
||||
ARGS=(
|
||||
--all-topologies
|
||||
@@ -164,6 +210,9 @@ jobs:
|
||||
if [ -n "${RUSTFS_EXPECTED_RC_SHA256}" ]; then
|
||||
ARGS+=(--expected-rc-sha256 "${RUSTFS_EXPECTED_RC_SHA256}")
|
||||
fi
|
||||
if [ -n "${PACKAGE_SHA256}" ]; then
|
||||
ARGS+=(--sha256 "${PACKAGE_SHA256}")
|
||||
fi
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
ARGS+=(--package-url "${PACKAGE_URL}")
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
@@ -430,52 +479,21 @@ jobs:
|
||||
|
||||
- name: "Continue functional chain (next: Storage engine)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-storage' \
|
||||
-F 'client_payload[from_suite]=tier'; then
|
||||
echo "dispatched next suite Storage engine (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Storage engine after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after tier (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **tier** to **Storage engine** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-storage'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-storage'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: Storage engine"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-storage' \
|
||||
-F 'client_payload[from_suite]=tier'
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
@@ -388,52 +388,21 @@ jobs:
|
||||
|
||||
- name: "Continue functional chain (next: S3 compatibility)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-s3' \
|
||||
-F 'client_payload[from_suite]=upgrade'; then
|
||||
echo "dispatched next suite S3 compatibility (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch S3 compatibility after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after upgrade (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **upgrade** to **S3 compatibility** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-s3'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-s3'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: S3 compatibility"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-s3' \
|
||||
-F 'client_payload[from_suite]=upgrade'
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: overtrue/repo-visuals-action@fd79cba437ecfac933d00a69add17eb95d3939c3 # v1.3.1
|
||||
- uses: overtrue/repo-visuals-action@72f34d24769ff5d341956da2f23952594ef2f1e2 # v1.3.0
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
output-branch: star-history
|
||||
|
||||
+3
-2
@@ -57,6 +57,9 @@ docs/*
|
||||
!docs/operations/**
|
||||
!docs/testing/
|
||||
!docs/testing/**
|
||||
docs/heal-scanner-logging-governance.md
|
||||
docs/benchmark/rustfs-target-bench/
|
||||
docs/benchmark/*.md
|
||||
.codegraph/*
|
||||
.docker/test/compat/data/*
|
||||
.docker/test/compat/kms/*
|
||||
@@ -80,8 +83,6 @@ worktrees/*
|
||||
|
||||
# Local AI-agent review artifacts (omo evidence dumps)
|
||||
.omo/
|
||||
# Legacy per-tool skill dir; skills live in .agents/skills (shared by all agents)
|
||||
.mimocode/
|
||||
|
||||
# insta scratch files; the accepted .snap files ARE the assertions and are committed
|
||||
*.snap.new
|
||||
|
||||
@@ -86,7 +86,6 @@ This file contains repository-wide rules. Use the nearest subdirectory
|
||||
- CI gates: `.github/workflows/ci.yml`.
|
||||
- PR format: `.github/pull_request_template.md`.
|
||||
- Architecture routing: `ARCHITECTURE.md` and `docs/architecture/README.md`.
|
||||
- Knowledge-base index and documentation rules: `docs/architecture/README.md`.
|
||||
- Agent skills: `.agents/skills/*/SKILL.md`.
|
||||
|
||||
Do not commit one-shot plans, trackers, migration ledgers, benchmark snapshots,
|
||||
|
||||
+1
-1
@@ -62,7 +62,7 @@ rustfs/ # Workspace root (virtual manifest)
|
||||
│ ├── utils/ # Pure utility functions
|
||||
│ ├── ... # (see "Crate Reference" below)
|
||||
│ └── e2e_test/ # End-to-end integration tests
|
||||
└── docs/ # Agent knowledge base: contracts, runbooks, testing rules (index: docs/architecture/README.md)
|
||||
└── docs/ # Design documents and analysis
|
||||
```
|
||||
|
||||
### Main Crate Layers (`rustfs/src/`)
|
||||
|
||||
@@ -27,7 +27,6 @@ make build-docker BUILD_OS=ubuntu22.04
|
||||
|
||||
## Where to look (do not duplicate here)
|
||||
|
||||
- Agent knowledge base index and doc-writing rules: [docs/architecture/README.md](docs/architecture/README.md)
|
||||
- Crate membership: `Cargo.toml` `[workspace].members`
|
||||
- Architecture, layering, crate map: [ARCHITECTURE.md](ARCHITECTURE.md)
|
||||
- Migration guardrails & readiness contracts: [docs/architecture/](docs/architecture/README.md)
|
||||
|
||||
Generated
+46
-47
@@ -347,9 +347,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
|
||||
|
||||
[[package]]
|
||||
name = "arrow"
|
||||
version = "59.3.0"
|
||||
version = "59.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c14b3d39f306bc28fd639d59f06e17a0f377d0021e1b7e9054e4d6fedc98774"
|
||||
checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -368,9 +368,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-arith"
|
||||
version = "59.3.0"
|
||||
version = "59.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce2961626677665b2195eb59242af4c7befe7b8737ca2050295389362380104e"
|
||||
checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -382,9 +382,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-array"
|
||||
version = "59.3.0"
|
||||
version = "59.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584"
|
||||
checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow-buffer",
|
||||
@@ -401,9 +401,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-buffer"
|
||||
version = "59.3.0"
|
||||
version = "59.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182"
|
||||
checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"half",
|
||||
@@ -413,9 +413,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-cast"
|
||||
version = "59.3.0"
|
||||
version = "59.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "635c9c635668ad26adf76cce8fb276c4be7cf06e63bd516de7da514f9680ee53"
|
||||
checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -435,9 +435,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-csv"
|
||||
version = "59.3.0"
|
||||
version = "59.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c2ebf8d631e79b02c16cf5ae860561272c26024ec88fce389a56aaddd558e86"
|
||||
checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-cast",
|
||||
@@ -450,9 +450,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-data"
|
||||
version = "59.3.0"
|
||||
version = "59.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358"
|
||||
checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d"
|
||||
dependencies = [
|
||||
"arrow-buffer",
|
||||
"arrow-schema",
|
||||
@@ -463,9 +463,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-ipc"
|
||||
version = "59.3.0"
|
||||
version = "59.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599"
|
||||
checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -479,9 +479,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-json"
|
||||
version = "59.3.0"
|
||||
version = "59.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2f57d7a81969f24ccf80809587b76c09897e6f829d2d65a5976bfb3218851f1"
|
||||
checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -504,9 +504,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-ord"
|
||||
version = "59.3.0"
|
||||
version = "59.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c900759f3bd8354fd4196bc4403eee846894dc2adf66b4225472006a0bf18c5"
|
||||
checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -517,9 +517,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-row"
|
||||
version = "59.3.0"
|
||||
version = "59.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f4c6425032e28266e3fc4ff680805e57e670d6ea92473043f3e65b7ed6ac79f2"
|
||||
checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -530,9 +530,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-schema"
|
||||
version = "59.3.0"
|
||||
version = "59.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd"
|
||||
checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_json",
|
||||
@@ -540,9 +540,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-select"
|
||||
version = "59.3.0"
|
||||
version = "59.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8"
|
||||
checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow-array",
|
||||
@@ -554,9 +554,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-string"
|
||||
version = "59.3.0"
|
||||
version = "59.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e0813f3c35c1cfea65e14c20a953440f7783c088b7ad2d0db162ccdeefcec14"
|
||||
checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -919,9 +919,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-rs"
|
||||
version = "1.18.1"
|
||||
version = "1.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e"
|
||||
checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e"
|
||||
dependencies = [
|
||||
"aws-lc-sys",
|
||||
"untrusted 0.7.1",
|
||||
@@ -930,9 +930,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-sys"
|
||||
version = "0.45.0"
|
||||
version = "0.44.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27"
|
||||
checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cmake",
|
||||
@@ -6106,9 +6106,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
|
||||
|
||||
[[package]]
|
||||
name = "libredox"
|
||||
version = "0.1.23"
|
||||
version = "0.1.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed"
|
||||
checksum = "d7955dfc218a8afb29dfeffd540e3a6e96baeb94fe7138228dd7cc6937fbbf96"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
@@ -7014,7 +7014,7 @@ version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"base64 0.21.7",
|
||||
"chrono",
|
||||
"getrandom 0.2.17",
|
||||
"http 1.5.0",
|
||||
@@ -7528,9 +7528,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "parquet"
|
||||
version = "59.3.0"
|
||||
version = "59.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff322f54b1a0f9288e614ed1f2d329b380af5476420db19f46ffb865e1163d73"
|
||||
checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow-array",
|
||||
@@ -8241,7 +8241,7 @@ version = "0.13.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"heck 0.4.1",
|
||||
"itertools 0.14.0",
|
||||
"log",
|
||||
"multimap",
|
||||
@@ -8261,7 +8261,7 @@ version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"heck 0.4.1",
|
||||
"itertools 0.14.0",
|
||||
"log",
|
||||
"multimap",
|
||||
@@ -10091,6 +10091,7 @@ dependencies = [
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -10449,7 +10450,6 @@ dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
"criterion",
|
||||
"faster-hex",
|
||||
"futures",
|
||||
"hex-simd",
|
||||
@@ -10461,7 +10461,6 @@ dependencies = [
|
||||
"md-5 0.11.0",
|
||||
"minlz",
|
||||
"pin-project-lite",
|
||||
"proptest",
|
||||
"rand 0.10.2",
|
||||
"reqwest",
|
||||
"rustfs-config",
|
||||
@@ -11085,7 +11084,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
[[package]]
|
||||
name = "s3s"
|
||||
version = "0.15.0"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=28e9ebb23dd2fb7d667084f34121b4aa4807a5c6#28e9ebb23dd2fb7d667084f34121b4aa4807a5c6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrayvec",
|
||||
@@ -11143,7 +11142,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "s3s-rfc2047"
|
||||
version = "0.16.0-alpha.1"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=28e9ebb23dd2fb7d667084f34121b4aa4807a5c6#28e9ebb23dd2fb7d667084f34121b4aa4807a5c6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"thiserror 2.0.20",
|
||||
@@ -11152,7 +11151,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "s3s-sigv2"
|
||||
version = "0.16.0-alpha.1"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=28e9ebb23dd2fb7d667084f34121b4aa4807a5c6#28e9ebb23dd2fb7d667084f34121b4aa4807a5c6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -11165,7 +11164,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "s3s-sigv4"
|
||||
version = "0.16.0-alpha.1"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=bdcb6259339c41369f9f1c60e3a42b5ab8da607b#bdcb6259339c41369f9f1c60e3a42b5ab8da607b"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=28e9ebb23dd2fb7d667084f34121b4aa4807a5c6#28e9ebb23dd2fb7d667084f34121b4aa4807a5c6"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"base64-simd",
|
||||
@@ -11788,9 +11787,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.16.0"
|
||||
version = "1.15.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f"
|
||||
checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
+2
-2
@@ -307,11 +307,11 @@ rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "bdcb6259339c41369f9f1c60e3a42b5ab8da607b", version = "0.15.0", features = ["minio"] }
|
||||
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "28e9ebb23dd2fb7d667084f34121b4aa4807a5c6", version = "0.15.0", features = ["minio"] }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
smallvec = { version = "1.16.0" }
|
||||
smallvec = { version = "1.15.2" }
|
||||
compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.3.0" }
|
||||
|
||||
@@ -48,33 +48,16 @@ Unlike other storage systems, RustFS is released under the permissible Apache 2.
|
||||
- **Open Source**: Licensed under Apache 2.0, encouraging unrestricted community contributions and commercial usage.
|
||||
- **User-Friendly**: Designed with simplicity in mind for easy deployment and management.
|
||||
|
||||
Status legend: ✅ Available — shipped and covered by CI gates; 🧪 Preview — shipped behind an opt-in flag or with a bounded compatibility claim.
|
||||
|
||||
| Feature | Status | Feature | Status |
|
||||
| :------------------------------- | :----------- | :--------------------------------- | :----------- |
|
||||
| **S3 Core Features** | ✅ Available | **Distributed Mode** | ✅ Available |
|
||||
| **Upload / Download** | ✅ Available | **Single Node Mode** | ✅ Available |
|
||||
| **Versioning** | ✅ Available | **Bitrot Protection** | ✅ Available |
|
||||
| **Object Lock (WORM)** | ✅ Available | **Healing & Scanner** | ✅ Available |
|
||||
| **Server-Side Encryption** | ✅ Available | **Pool Expansion / Decommission** | ✅ Available |
|
||||
| **RustFS KMS** | ✅ Available | **Bucket Replication** | ✅ Available |
|
||||
| **Lifecycle Management (ILM)** | ✅ Available | **Site Replication** | ✅ Available |
|
||||
| **ILM Tiering (Remote S3)** | ✅ Available | **Bucket Quota** | ✅ Available |
|
||||
| **S3 Select** | ✅ Available | **Event Notifications** | ✅ Available |
|
||||
| **S3 Tables (Iceberg REST)** | 🧪 Preview | **Audit Logging** | ✅ Available |
|
||||
| **IAM / Policies** | ✅ Available | **Logging & Observability** | ✅ Available |
|
||||
| **OIDC / SSO** | ✅ Available | **Web Console** | ✅ Available |
|
||||
| **Keystone Auth** | ✅ Available | **K8s Helm Charts** | ✅ Available |
|
||||
| **Swift API** | ✅ Available | **FTPS / WebDAV** | ✅ Available |
|
||||
| **Multi-Tenancy** | ✅ Available | **SFTP** | ✅ Available |
|
||||
| **MinIO On-Disk Compatibility** | 🧪 Preview | | |
|
||||
|
||||
Notes:
|
||||
|
||||
- **RustFS KMS**: Vault (KV2 / Transit) and AWS KMS backends are supported for production. The `Local` and `Static` backends are for development and testing only. See [KMS backend security properties](docs/operations/kms-backend-security.md).
|
||||
- **Swift API / SFTP**: opt-in cargo features (`--features swift`, `--features sftp`, or `full`). FTPS and WebDAV are enabled in the default build.
|
||||
- **S3 Tables**: ships as an Iceberg REST Catalog with automated PyIceberg and DuckDB coverage; other engines and vendor profiles carry bounded claims listed in the [S3 Tables support matrix](docs/architecture/s3-tables-support-matrix.md).
|
||||
- **MinIO On-Disk Compatibility**: gated behind the `rio-v2` feature and not part of the default build. Objects MinIO encrypted are not readable by RustFS. See [MinIO file-format interoperability](docs/architecture/minio-file-format-compat.md).
|
||||
| Feature | Status | Feature | Status |
|
||||
| :---------------------- | :----------- | :----------------------- | :--------------- |
|
||||
| **S3 Core Features** | ✅ Available | **Bitrot Protection** | ✅ Available |
|
||||
| **Upload / Download** | ✅ Available | **Single Node Mode** | ✅ Available |
|
||||
| **Versioning** | ✅ Available | **Bucket Replication** | ✅ Available |
|
||||
| **Logging** | ✅ Available | **Lifecycle Management** | 🚧 Under Testing |
|
||||
| **Event Notifications** | ✅ Available | **Distributed Mode** | 🚧 Under Testing |
|
||||
| **K8s Helm Charts** | ✅ Available | **RustFS KMS** | 🚧 Under Testing |
|
||||
| **Keystone Auth** | ✅ Available | **Multi-Tenancy** | ✅ Available |
|
||||
| **Swift API** | ✅ Available | **Swift Metadata Ops** | 🚧 Partial |
|
||||
|
||||
## RustFS vs MinIO Performance
|
||||
|
||||
|
||||
@@ -233,8 +233,8 @@ spawn error. Install the pinned CI version before running their profiles.
|
||||
[`src/policy/README.md`](src/policy/README.md),
|
||||
[`src/protocols/README.md`](src/protocols/README.md),
|
||||
[`src/reliant/README.md`](src/reliant/README.md)
|
||||
- Per-module counts: `cargo nextest list -p e2e_test --profile <profile>`
|
||||
(one-liner in [`docs/testing/README.md`](../../docs/testing/README.md))
|
||||
- Authoritative per-module counts:
|
||||
[`docs/testing/e2e-suite-inventory.md`](../../docs/testing/e2e-suite-inventory.md)
|
||||
- Test pyramid & flake policy: [`docs/testing/README.md`](../../docs/testing/README.md)
|
||||
|
||||
## CI smoke subset (`--profile e2e-smoke`)
|
||||
@@ -271,12 +271,12 @@ Note on `#[serial]`: nextest runs each test in its own process, so
|
||||
parallel-safe by construction (random port + isolated temp dir), which the
|
||||
current subset is.
|
||||
|
||||
### Test inventory
|
||||
### Authoritative test inventory
|
||||
|
||||
Per-module counts are not committed; list them with
|
||||
`cargo nextest list -p e2e_test --profile <profile>` (the result is
|
||||
platform-dependent because some modules are linux-only; the `jq` one-liner is
|
||||
in `docs/testing/README.md`). When a profile membership change is
|
||||
`docs/testing/e2e-suite-inventory.md` records the per-module test counts as
|
||||
listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or
|
||||
moving e2e tests so acceptance numbers in the test-strategy issues
|
||||
(backlog#1147–#1155) stay auditable. When a profile membership change is
|
||||
intentional, review its JSON listing before updating the matching
|
||||
`.config/e2e-*-selection.txt` test-ID digest. Update only the platform that
|
||||
produced the listing:
|
||||
|
||||
@@ -22,7 +22,7 @@ mod tests {
|
||||
use aws_sdk_s3::config::{Credentials, Region, RequestChecksumCalculation};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart, ServerSideEncryption};
|
||||
use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
|
||||
@@ -260,117 +260,6 @@ mod tests {
|
||||
info!("PASSED: HeadObject returns stored SHA256 digest");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_head_object_returns_sse_s3_checksum() {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_SSE_S3_MASTER_KEY", "MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI="),
|
||||
("RUSTFS_CONSOLE_ENABLE", "false"),
|
||||
],
|
||||
)
|
||||
.await
|
||||
.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-sse-s3-checksum-head";
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("encrypted.txt")
|
||||
.body(ByteStream::from_static(b"encrypted checksum"))
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.checksum_algorithm(ChecksumAlgorithm::Crc32)
|
||||
.send()
|
||||
.await
|
||||
.expect("SSE-S3 PutObject with CRC32 failed");
|
||||
let expected = put.checksum_crc32().expect("PutObject must return CRC32");
|
||||
|
||||
let head = client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key("encrypted.txt")
|
||||
.checksum_mode(ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await
|
||||
.expect("SSE-S3 HeadObject failed");
|
||||
|
||||
assert_eq!(head.checksum_crc32(), Some(expected));
|
||||
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key("encrypted-copy.txt")
|
||||
.copy_source(format!("{bucket}/encrypted.txt"))
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.send()
|
||||
.await
|
||||
.expect("SSE-S3 CopyObject failed");
|
||||
let copy_head = client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key("encrypted-copy.txt")
|
||||
.checksum_mode(ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await
|
||||
.expect("SSE-S3 copied HeadObject failed");
|
||||
|
||||
assert_eq!(copy_head.checksum_crc32(), Some(expected));
|
||||
|
||||
let multipart_key = "encrypted-multipart.txt";
|
||||
let create = client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(multipart_key)
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.checksum_algorithm(ChecksumAlgorithm::Crc32)
|
||||
.send()
|
||||
.await
|
||||
.expect("SSE-S3 CreateMultipartUpload with CRC32 failed");
|
||||
let upload_id = create.upload_id().expect("CreateMultipartUpload must return an upload ID");
|
||||
let part = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(multipart_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from_static(b"encrypted multipart checksum"))
|
||||
.checksum_algorithm(ChecksumAlgorithm::Crc32)
|
||||
.send()
|
||||
.await
|
||||
.expect("SSE-S3 UploadPart with CRC32 failed");
|
||||
let completed_part = CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(part.e_tag().expect("UploadPart must return an ETag"))
|
||||
.checksum_crc32(part.checksum_crc32().expect("UploadPart must return CRC32"))
|
||||
.build();
|
||||
let complete = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(multipart_key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().parts(completed_part).build())
|
||||
.send()
|
||||
.await
|
||||
.expect("SSE-S3 CompleteMultipartUpload with CRC32 failed");
|
||||
let expected_multipart = complete.checksum_crc32().expect("CompleteMultipartUpload must return CRC32");
|
||||
let multipart_head = client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(multipart_key)
|
||||
.checksum_mode(ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await
|
||||
.expect("SSE-S3 multipart HeadObject failed");
|
||||
|
||||
assert_eq!(multipart_head.checksum_crc32(), Some(expected_multipart));
|
||||
}
|
||||
|
||||
/// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content.
|
||||
/// Uses part size >= 5MB (server minimum) for two parts.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1699,51 +1699,6 @@ impl RustFSTestClusterEnvironment {
|
||||
process.wait()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Gracefully stop one cluster node and wait for its process to exit.
|
||||
///
|
||||
/// This is intentionally separate from [`Self::stop_node`]: the latter is
|
||||
/// a hard kill used by crash-recovery tests, while this path lets RustFS
|
||||
/// complete its normal shutdown hooks before a test restarts the node.
|
||||
pub async fn stop_node_gracefully(&mut self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.ensure_node_index(node_idx)?;
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let Some(process) = self.nodes[node_idx].process.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
let pid = process.id().to_string();
|
||||
let signal_status = Command::new("kill").args(["-TERM", &pid]).status()?;
|
||||
if !signal_status.success() {
|
||||
return Err(format!("failed to send SIGTERM to cluster node {node_idx} (pid {pid})").into());
|
||||
}
|
||||
|
||||
let mut process = self.nodes[node_idx]
|
||||
.process
|
||||
.take()
|
||||
.ok_or_else(|| format!("cluster node {node_idx} process disappeared while stopping"))?;
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(45);
|
||||
loop {
|
||||
if let Some(status) = process.try_wait()? {
|
||||
info!("Cluster node {} stopped gracefully with {}", node_idx, status);
|
||||
return Ok(());
|
||||
}
|
||||
if std::time::Instant::now() >= deadline {
|
||||
let _ = process.kill();
|
||||
let _ = process.wait();
|
||||
return Err(format!("cluster node {node_idx} did not stop gracefully within 45 seconds").into());
|
||||
}
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = node_idx;
|
||||
Err("graceful cluster-node stop is only supported on Unix E2E hosts".into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RustFSTestClusterEnvironment {
|
||||
|
||||
@@ -1,15 +1,11 @@
|
||||
# Programmable fake S3 target
|
||||
|
||||
This module is the shared failure-injection boundary for replication end-to-end tests and the programmable external source for on-demand-migration (ODM) tests. It runs an in-process, path-style S3 endpoint backed by `s3s`; no production crate depends on it.
|
||||
This module is the shared failure-injection boundary for replication end-to-end tests. It runs an in-process, path-style S3 endpoint backed by `s3s`; no production crate depends on it.
|
||||
|
||||
`FakeS3Target::start()` creates the listener. Add target buckets with `create_bucket`, point a RustFS remote target at `address()`, use `FAKE_ACCESS_KEY` / `FAKE_SECRET_KEY`, then enqueue per-operation faults with `inject`. Faults for one operation are consumed in FIFO order and do not consume faults queued for another operation. A fault is consumed only after `s3s` verifies the full request signature, so anonymous, other-access-key, and bad-signature traffic cannot disturb a script.
|
||||
|
||||
Supported data operations are HeadBucket, GetBucketVersioning, ListObjectsV2, PUT/GET/HEAD/DELETE Object, Get/Put/Delete ObjectTagging (tags live per version; Put replaces the whole set, Delete clears it), and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets created with `create_bucket` are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
|
||||
Supported data operations are HeadBucket, GetBucketVersioning, PUT/GET/HEAD/DELETE Object, Get/Put/Delete ObjectTagging (tags live per version; Put replaces the whole set, Delete clears it), and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
|
||||
|
||||
`create_bucket_with_mode(name, BucketMode::Unversioned)` models a plain migration source: PUT overwrites in place, DELETE removes the key without a delete marker, GetBucketVersioning reports no status, and no `x-amz-version-id` is returned by PUT, GET, HEAD, tagging, or multipart completion. The only `versionId` such a bucket accepts is `null`; any other value is rejected with `InvalidArgument`. The mode is fixed at creation.
|
||||
Fault actions cover HTTP 401/403/503 responses, pre-dispatch delay, connection abort when a logical request-body threshold is reached, streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions. Each record also journals a `ProxyHeaderSnapshot` — the read-proxy anti-loop marker (`x-{rustfs,minio}-source-proxy-request`), the replication-check exemption header, and the client SSE-C header family (algorithm and key-MD5 values; for the key itself only its presence) — so proxy tests can pin the exact wire contract.
|
||||
|
||||
ListObjectsV2 lists current versions only (a key whose newest version is a delete marker is hidden) in byte order and supports `prefix`, `delimiter`, `max-keys` (clamped to 1000), `start-after`, and `continuation-token`; common prefixes count toward `max-keys`, `IsTruncated` / `NextContinuationToken` / `KeyCount` follow S3, and continuation tokens are opaque. `encoding-type` and `fetch-owner` are accepted but ignored, and ListObjects (v1) is not implemented. GET and HEAD honor `Range` in the `bytes=first-last`, `bytes=first-`, and `bytes=-suffix` forms with a 206 status, exact `Content-Range`, and `Accept-Ranges: bytes`; unsatisfiable ranges answer 416 `InvalidRange` with `Content-Range: bytes */<length>`. PUT and CreateMultipartUpload accept `Content-Type`, `Content-Encoding`, `Content-Disposition`, `Content-Language`, `Cache-Control`, `Expires`, and `x-amz-meta-*` (names stored lowercased), and HEAD/GET replay them verbatim together with `Last-Modified` and the ETag (hex MD5 for single PUTs, `<md5-of-part-md5s>-<parts>` for multipart objects). `put_seed_object` stores an object directly, bypassing the wire, the fault script, and the journal, so a source can be seeded without polluting the assertions a scenario later makes.
|
||||
|
||||
Fault actions cover HTTP 401/403/503 responses (`Status`), any 4xx/5xx status paired with the matching S3 error code (`ResponseStatus`), pre-dispatch delay, holding a fully computed successful response before its first byte (`Stall`), connection abort when a logical request-body threshold is reached, GetObject bodies cut off after N bytes while `Content-Length` announces the full size (`TruncateBodyAt`), streaming slow drain, and a deliberately wrong response ETag (including multipart-complete XML). `requests()` returns the ordered, credential-free request journal for assertions and `count_requests(operation, key)` counts entries for one exact key. Each record journals the `Range` and `User-Agent` request headers, the ListObjectsV2 `prefix` and `continuation-token` query values, and a `ProxyHeaderSnapshot` — the read-proxy anti-loop marker (`x-{rustfs,minio}-source-proxy-request`), the replication-check exemption header, and the client SSE-C header family (algorithm and key-MD5 values; for the key itself only its presence) — so proxy tests can pin the exact wire contract.
|
||||
|
||||
The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type and each standard object header at 1 KiB. By default a PUT or uploaded part is capped at 64 MiB and a completed multipart object and all stored object/part data are capped at 128 MiB; `FakeS3Target::start_with_options(FakeS3TargetOptions { max_object_bytes })` raises the object cap up to 256 MiB, and the total budget then becomes twice the object cap (never below 128 MiB). Body drain, body-permit waits, delay, stall, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound.
|
||||
The listener is loopback-only. It admits at most 64 active connections and two concurrently buffered request bodies; authenticated multipart-complete XML collection and assembly take both body permits. Keep-alive is disabled, request-header reads are bounded to 30 seconds, a parsed request is bounded to 65 seconds, and the complete connection lifetime is bounded to 100 seconds. It retains at most 256 buckets, 4,096 journal entries, 4,096 scripted faults, 4,096 object versions, 256 multipart uploads, and 10,000 multipart parts. Retained identifiers are capped at 1 KiB, user metadata at 2 KiB, and content type at 1 KiB. A PUT or uploaded part is capped at 64 MiB; a completed multipart object and all stored object/part data are capped at 128 MiB. Body drain, body-permit waits, delay, and slow-drain execution are bounded to 30 seconds; each slow-drain slice delay must be below that bound.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -17,115 +17,15 @@
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
|
||||
use crate::common::{
|
||||
FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging,
|
||||
};
|
||||
use crate::storage_api::RUSTFS_META_BUCKET;
|
||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::Method;
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||
use tracing::info;
|
||||
|
||||
const POOL_METADATA_OBJECT: &str = "pool.bin";
|
||||
|
||||
struct TcpPortBlackhole {
|
||||
port: u16,
|
||||
comment: String,
|
||||
use_sudo: bool,
|
||||
active: bool,
|
||||
}
|
||||
|
||||
impl TcpPortBlackhole {
|
||||
fn install(address: &str) -> Result<Self, Box<dyn Error + Send + Sync>> {
|
||||
let address = address.parse::<SocketAddr>()?;
|
||||
if !address.ip().is_loopback() {
|
||||
return Err(format!("refusing to install a test firewall rule for non-loopback address {address}").into());
|
||||
}
|
||||
|
||||
let id = Command::new("id").arg("-u").output()?;
|
||||
if !id.status.success() {
|
||||
return Err(format!("failed to determine the test process uid: {}", String::from_utf8_lossy(&id.stderr)).into());
|
||||
}
|
||||
let use_sudo = String::from_utf8_lossy(&id.stdout).trim() != "0";
|
||||
let mut blackhole = Self {
|
||||
port: address.port(),
|
||||
comment: format!("rustfs-e2e-{}", uuid::Uuid::new_v4()),
|
||||
use_sudo,
|
||||
active: false,
|
||||
};
|
||||
blackhole.run_iptables(true)?;
|
||||
blackhole.active = true;
|
||||
Ok(blackhole)
|
||||
}
|
||||
|
||||
fn restore(&mut self) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
if !self.active {
|
||||
return Ok(());
|
||||
}
|
||||
self.run_iptables(false)?;
|
||||
self.active = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_iptables(&self, insert: bool) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let mut command = if self.use_sudo {
|
||||
let mut command = Command::new("sudo");
|
||||
command.args(["-n", "iptables"]);
|
||||
command
|
||||
} else {
|
||||
Command::new("iptables")
|
||||
};
|
||||
command.args(["-w", "5"]);
|
||||
if insert {
|
||||
command.args(["-I", "OUTPUT", "1"]);
|
||||
} else {
|
||||
command.args(["-D", "OUTPUT"]);
|
||||
}
|
||||
let port = self.port.to_string();
|
||||
let output = command
|
||||
.args([
|
||||
"-p",
|
||||
"tcp",
|
||||
"-d",
|
||||
"127.0.0.1/32",
|
||||
"--dport",
|
||||
&port,
|
||||
"-m",
|
||||
"comment",
|
||||
"--comment",
|
||||
&self.comment,
|
||||
"-j",
|
||||
"DROP",
|
||||
])
|
||||
.output()?;
|
||||
if !output.status.success() {
|
||||
let action = if insert { "install" } else { "remove" };
|
||||
return Err(format!(
|
||||
"failed to {action} endpoint blackhole rule for port {}: stdout={}, stderr={}",
|
||||
self.port,
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TcpPortBlackhole {
|
||||
fn drop(&mut self) {
|
||||
if let Err(error) = self.restore() {
|
||||
eprintln!("failed to remove {} firewall rule during test cleanup: {error}", self.comment);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn has_file_under(path: &Path) -> bool {
|
||||
let Ok(entries) = std::fs::read_dir(path) else {
|
||||
return false;
|
||||
@@ -191,62 +91,6 @@ mod tests {
|
||||
.count()
|
||||
}
|
||||
|
||||
async fn assert_all_nodes_list_exact_keys(
|
||||
clients: &[aws_sdk_s3::Client],
|
||||
bucket: &str,
|
||||
expected_keys: &HashSet<String>,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
const PAGE_SIZE: i32 = 10;
|
||||
for (node_index, client) in clients.iter().enumerate() {
|
||||
let mut listed_keys = Vec::new();
|
||||
let mut continuation_token = None;
|
||||
let max_pages = expected_keys.len().div_ceil(PAGE_SIZE as usize) + 1;
|
||||
let mut page_count = 0;
|
||||
loop {
|
||||
page_count += 1;
|
||||
if page_count > max_pages {
|
||||
return Err(format!("node {node_index} listing exceeded the bounded {max_pages}-page budget").into());
|
||||
}
|
||||
let response = timeout(
|
||||
Duration::from_secs(15),
|
||||
client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.max_keys(PAGE_SIZE)
|
||||
.set_continuation_token(continuation_token.clone())
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
listed_keys.extend(
|
||||
response
|
||||
.contents()
|
||||
.iter()
|
||||
.filter_map(|object| object.key().map(str::to_owned)),
|
||||
);
|
||||
if !response.is_truncated().unwrap_or(false) {
|
||||
break;
|
||||
}
|
||||
let next_token = response
|
||||
.next_continuation_token()
|
||||
.filter(|token| Some(*token) != continuation_token.as_deref())
|
||||
.ok_or_else(|| format!("node {node_index} returned a truncated listing without a new continuation token"))?;
|
||||
continuation_token = Some(next_token.to_owned());
|
||||
}
|
||||
|
||||
let listed_key_set = listed_keys.iter().cloned().collect::<HashSet<_>>();
|
||||
assert_eq!(
|
||||
listed_keys.len(),
|
||||
listed_key_set.len(),
|
||||
"node {node_index} returned duplicate keys after recovery: {listed_keys:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
&listed_key_set, expected_keys,
|
||||
"node {node_index} did not expose the complete recovered namespace"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn heal_task_status_diagnostic(body: &str) -> String {
|
||||
let Ok(status) = serde_json::from_str::<serde_json::Value>(body) else {
|
||||
return body.to_string();
|
||||
@@ -281,12 +125,11 @@ mod tests {
|
||||
&& operations["retryingTasks"].as_u64() == Some(0)
|
||||
}
|
||||
|
||||
// Queued low-priority repairs cannot execute while the single admin slot is
|
||||
// occupied; ownership is determined by active and retrying tasks only.
|
||||
fn only_admin_heal_is_active(status: &serde_json::Value) -> bool {
|
||||
let operations = &status["healOperations"];
|
||||
status["clusterStatusComplete"] == serde_json::Value::Bool(true)
|
||||
&& status["state"].as_str() == Some("active")
|
||||
&& operations["queueLength"].as_u64() == Some(0)
|
||||
&& operations["activeTasks"].as_u64() == Some(1)
|
||||
&& operations["retryingTasks"].as_u64() == Some(0)
|
||||
&& operations["activeBySource"]["admin"].as_u64() == Some(1)
|
||||
@@ -704,144 +547,41 @@ mod tests {
|
||||
.into())
|
||||
}
|
||||
|
||||
async fn wait_for_scanner_cycle_after(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
previous_cycle_end: u64,
|
||||
) -> Result<u64, Box<dyn Error + Send + Sync>> {
|
||||
let deadline = Instant::now() + Duration::from_secs(60);
|
||||
loop {
|
||||
let mut latest_cycle_end = 0;
|
||||
let mut versions_observed = false;
|
||||
let mut observations = Vec::with_capacity(cluster.nodes.len());
|
||||
for (node_index, node) in cluster.nodes.iter().enumerate() {
|
||||
let (status, body) = timeout(
|
||||
Duration::from_secs(5),
|
||||
admin_request(
|
||||
&node.url,
|
||||
Method::GET,
|
||||
"/rustfs/admin/v3/scanner/status",
|
||||
None,
|
||||
&cluster.access_key,
|
||||
&cluster.secret_key,
|
||||
),
|
||||
)
|
||||
.await??;
|
||||
assert_eq!(status, 200, "scanner status must be available: {body}");
|
||||
let status: serde_json::Value = serde_json::from_str(&body)?;
|
||||
assert_eq!(status["enabled"].as_bool(), Some(true), "scanner must stay enabled: {status}");
|
||||
let metrics = &status["metrics"];
|
||||
let cycle_end = metrics["last_cycle_end_unix_secs"]
|
||||
.as_u64()
|
||||
.ok_or("scanner status is missing its completed-cycle timestamp")?;
|
||||
let versions_scanned = metrics["versions_scanned"]
|
||||
.as_u64()
|
||||
.ok_or("scanner status is missing its version-coverage counter")?;
|
||||
latest_cycle_end = latest_cycle_end.max(cycle_end);
|
||||
versions_observed |= versions_scanned > 0;
|
||||
observations.push(format!(
|
||||
"node{node_index}: end={cycle_end}, versions={versions_scanned}, cycle={}, active={}, leader={}, result={}",
|
||||
metrics["current_cycle"],
|
||||
metrics["current_cycle_active"],
|
||||
metrics["leader_lock_state"],
|
||||
metrics["last_cycle_result"],
|
||||
));
|
||||
}
|
||||
// The coordinator records cycle completion, but remote workers
|
||||
// record scanned versions. Both witnesses need not share a node.
|
||||
if latest_cycle_end > previous_cycle_end && versions_observed {
|
||||
return Ok(latest_cycle_end);
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!(
|
||||
"enabled scanner did not complete an object-scanning cycle after {previous_cycle_end}: {observations:?}"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the original unformatted-disk scenario above. This case retains the
|
||||
// format identity so only the explicit admin task can rebuild missing data.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_resumes_missing_remote_shards_after_node_restart() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::IsolatedTargetRestart).await
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_remote_shards_after_coordinator_restart() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
timeout(
|
||||
Duration::from_secs(420),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundCoordinatorRestart),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_after_target_endpoint_blackhole() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
timeout(
|
||||
Duration::from_secs(420),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::TargetEndpointBlackhole),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum InterruptionScenario {
|
||||
IsolatedTargetRestart,
|
||||
BackgroundCoordinatorRestart,
|
||||
TargetEndpointBlackhole,
|
||||
}
|
||||
|
||||
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let (background_enabled, interruption_node, interruption_kind) = match scenario {
|
||||
InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
|
||||
InterruptionScenario::BackgroundCoordinatorRestart => (true, 0, "coordinator_restart"),
|
||||
InterruptionScenario::TargetEndpointBlackhole => (false, 1, "target_endpoint_blackhole"),
|
||||
};
|
||||
init_logging();
|
||||
info!(
|
||||
event = "heal_interruption_started",
|
||||
event = "heal_restart_started",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
background_enabled,
|
||||
interruption_node,
|
||||
interruption_kind,
|
||||
"Starting root-heal interruption test"
|
||||
"Starting root-heal restart test"
|
||||
);
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
|
||||
cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
|
||||
// Heal control uses the first lexicographically sorted grid host.
|
||||
// Keep that coordinator distinct from the remote target at index 1.
|
||||
cluster.nodes.sort_by(|left, right| left.url.cmp(&right.url));
|
||||
cluster.set_env("RUSTFS_HEAL_AUTO_HEAL_ENABLE", background_enabled.to_string());
|
||||
cluster.set_env("RUSTFS_HEAL_MRF_ENABLE", background_enabled.to_string());
|
||||
cluster.set_env("RUSTFS_SCANNER_ENABLED", background_enabled.to_string());
|
||||
if background_enabled {
|
||||
// Only the scanner cadence is accelerated. Keep normal Heal
|
||||
// concurrency and every automatic recovery owner enabled.
|
||||
for &(key, value) in FAST_DATA_USAGE_SCANNER_ENV {
|
||||
cluster.set_env(key, value);
|
||||
}
|
||||
} else {
|
||||
cluster.set_env("RUSTFS_HEAL_MAX_CONCURRENT_HEALS", "1");
|
||||
cluster.set_env("RUSTFS_HEAL_MAX_CONCURRENT_PER_SET", "1");
|
||||
cluster.set_env("RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY", "1");
|
||||
cluster.set_env("RUSTFS_HEAL_PAGE_PARALLEL_ENABLE", "false");
|
||||
}
|
||||
// Keep every node's Heal runtime enabled for normal disk registration.
|
||||
cluster.set_env("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "false");
|
||||
cluster.set_env("RUSTFS_HEAL_MRF_ENABLE", "false");
|
||||
cluster.set_env("RUSTFS_SCANNER_ENABLED", "false");
|
||||
cluster.set_env("RUSTFS_HEAL_MAX_CONCURRENT_HEALS", "1");
|
||||
cluster.set_env("RUSTFS_HEAL_MAX_CONCURRENT_PER_SET", "1");
|
||||
cluster.set_env("RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY", "1");
|
||||
cluster.set_env("RUSTFS_HEAL_PAGE_PARALLEL_ENABLE", "false");
|
||||
// Keep all storage nodes' Heal runtimes enabled so their disk services
|
||||
// complete normal registration after restart. Scanner, auto-heal and
|
||||
// MRF are disabled; the pre-root idle barrier below drains the direct
|
||||
// outage-object repair before the explicit admin task starts.
|
||||
let server_rust_log = std::env::var("RUSTFS_HEAL_CHAOS_SERVER_RUST_LOG")
|
||||
.unwrap_or_else(|_| "rustfs::heal::task=info,rustfs=error".to_string());
|
||||
cluster.set_env("RUST_LOG", server_rust_log);
|
||||
let log_dir = std::env::var("RUSTFS_HEAL_CHAOS_LOG_DIR").unwrap_or_else(|_| format!("{}/logs", cluster.temp_dir));
|
||||
std::fs::create_dir_all(&log_dir)?;
|
||||
for node_index in 0..cluster.nodes.len() {
|
||||
cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
|
||||
if let Ok(log_dir) = std::env::var("RUSTFS_HEAL_CHAOS_LOG_DIR") {
|
||||
std::fs::create_dir_all(&log_dir)?;
|
||||
for node_index in 0..cluster.nodes.len() {
|
||||
cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
|
||||
}
|
||||
}
|
||||
cluster.start().await?;
|
||||
let clients = cluster.create_all_clients()?;
|
||||
@@ -894,18 +634,6 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
let expected_pool_metadata = if background_enabled {
|
||||
let census = census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?;
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"target must hold complete pool metadata before the fault: {census:?}"
|
||||
);
|
||||
wait_for_scanner_cycle_after(&cluster, 0).await?;
|
||||
Some(census)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
cluster.stop_node(1)?;
|
||||
std::fs::remove_dir_all(&replaced_disk)?;
|
||||
std::fs::create_dir_all(
|
||||
@@ -963,25 +691,25 @@ mod tests {
|
||||
.find(|index| !outage_peer_erasure_indices.contains(index))
|
||||
.ok_or("online outage-object shards leave no erasure index for the replacement target")?;
|
||||
|
||||
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
if !background_enabled {
|
||||
// The PUT path may have admitted a direct Internal object repair while
|
||||
// node 1 was offline. Cancel the isolated bucket path before the target
|
||||
// returns; otherwise it could rebuild the outage object and invalidate
|
||||
// the explicit-root ownership assertion below.
|
||||
let cancel_outage_heal_path = format!("/rustfs/admin/v3/heal/{bucket}?forceStop=true");
|
||||
let (cancel_status, cancel_body) = admin_request(
|
||||
&cluster.nodes[0].url,
|
||||
Method::POST,
|
||||
&cancel_outage_heal_path,
|
||||
Some(heal_body.to_string()),
|
||||
&cluster.access_key,
|
||||
&cluster.secret_key,
|
||||
)
|
||||
.await?;
|
||||
if !cancel_status.is_success() {
|
||||
return Err(format!("cancel outage heal failed: {cancel_status} {cancel_body}").into());
|
||||
}
|
||||
// The PUT path may have admitted a direct Internal object repair while
|
||||
// node 1 was offline. Cancel the isolated bucket path before the target
|
||||
// returns; otherwise it could rebuild the outage object and invalidate
|
||||
// the explicit-root ownership assertion below.
|
||||
let cancel_outage_heal_path = format!("/rustfs/admin/v3/heal/{bucket}?forceStop=true");
|
||||
let (cancel_status, cancel_body) = admin_request(
|
||||
&cluster.nodes[0].url,
|
||||
Method::POST,
|
||||
&cancel_outage_heal_path,
|
||||
Some(
|
||||
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#
|
||||
.to_string(),
|
||||
),
|
||||
&cluster.access_key,
|
||||
&cluster.secret_key,
|
||||
)
|
||||
.await?;
|
||||
if !cancel_status.is_success() {
|
||||
return Err(format!("cancel outage heal failed: {cancel_status} {cancel_body}").into());
|
||||
}
|
||||
|
||||
cluster.start_node(1).await?;
|
||||
@@ -996,12 +724,7 @@ mod tests {
|
||||
);
|
||||
let recovered: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
let ready = if background_enabled {
|
||||
recovered["clusterStatusComplete"] == serde_json::Value::Bool(true)
|
||||
} else {
|
||||
cluster_heal_is_idle(&recovered)
|
||||
};
|
||||
if ready {
|
||||
if cluster_heal_is_idle(&recovered) {
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= recovery_deadline {
|
||||
@@ -1009,24 +732,23 @@ mod tests {
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
assert_eq!(
|
||||
matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?,
|
||||
0,
|
||||
"non-admin Heal is disabled, so the replacement target must remain empty before the explicit root heal"
|
||||
);
|
||||
assert!(
|
||||
!census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?.has_xl_meta,
|
||||
"the object written during the outage must be absent before the explicit root heal"
|
||||
);
|
||||
let pre_heal_replacement = replacement_recovery_status(&cluster).await?;
|
||||
if !background_enabled {
|
||||
assert_eq!(
|
||||
matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?,
|
||||
0,
|
||||
"non-admin Heal is disabled, so the replacement target must remain empty before the explicit root heal"
|
||||
);
|
||||
assert!(
|
||||
!census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?.has_xl_meta,
|
||||
"the object written during the outage must be absent before the explicit root heal"
|
||||
);
|
||||
assert_eq!(
|
||||
pre_heal_replacement["cluster"]["records"].as_array().map(Vec::len),
|
||||
Some(0),
|
||||
"isolated target must not retain an automatic replacement generation: {pre_heal_replacement}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
pre_heal_replacement["cluster"]["records"].as_array().map(Vec::len),
|
||||
Some(0),
|
||||
"isolated target must not retain an automatic replacement generation: {pre_heal_replacement}"
|
||||
);
|
||||
|
||||
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
let heal_url = format!("{}/rustfs/admin/v3/heal/?forceStart=true", cluster.nodes[0].url);
|
||||
let heal_start_body = signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?;
|
||||
let heal_start: serde_json::Value = serde_json::from_str(&heal_start_body)
|
||||
@@ -1042,39 +764,24 @@ mod tests {
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.unwrap_or(60);
|
||||
let partial_deadline = Instant::now() + Duration::from_secs(partial_timeout_secs);
|
||||
loop {
|
||||
let pre_interrupt_status = loop {
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let active_status: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
let active = if background_enabled {
|
||||
active_status["state"].as_str() == Some("active")
|
||||
&& active_status["healOperations"]["activeBySource"]["admin"].as_u64() == Some(1)
|
||||
} else {
|
||||
only_admin_heal_is_active(&active_status)
|
||||
};
|
||||
if active {
|
||||
break;
|
||||
if only_admin_heal_is_active(&active_status) {
|
||||
break active_status;
|
||||
}
|
||||
if Instant::now() >= partial_deadline {
|
||||
return Err(format!("root heal never became active within {partial_timeout_secs}s: {active_status}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
let (partial_count, partial_manifest) = loop {
|
||||
// Hash one committed shard to prove progress without letting a
|
||||
// full-corpus hash pass consume the interruption window.
|
||||
let materialized = metadata_count(&replaced_disk, bucket, &expected_manifests);
|
||||
if materialized > 0
|
||||
&& materialized < expected_manifests.len()
|
||||
&& let Some(expected) = expected_manifests
|
||||
.iter()
|
||||
.find(|expected| object_metadata_exists_on_disk(&replaced_disk, bucket, &expected.key))
|
||||
&& census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?
|
||||
.matches_manifest(&expected.shard_census)
|
||||
{
|
||||
break (materialized, expected);
|
||||
};
|
||||
let partial_count = loop {
|
||||
let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
if matching > 0 && matching < expected_manifests.len() {
|
||||
break matching;
|
||||
}
|
||||
if materialized == expected_manifests.len() {
|
||||
if matching == expected_manifests.len() {
|
||||
return Err(format!(
|
||||
"root heal rebuilt all {} baseline objects before the target could be interrupted",
|
||||
expected_manifests.len()
|
||||
@@ -1089,195 +796,30 @@ mod tests {
|
||||
}
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
};
|
||||
|
||||
let pre_interrupt_status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let pre_interrupt_status: serde_json::Value = serde_json::from_str(&pre_interrupt_status_body)
|
||||
.map_err(|err| format!("pre-interrupt background heal status is not JSON ({err}): {pre_interrupt_status_body}"))?;
|
||||
let pre_interrupt_replacement = replacement_recovery_status(&cluster).await?;
|
||||
let coordinator_log = std::fs::read_to_string(format!("{log_dir}/node0.log"))?;
|
||||
assert!(
|
||||
coordinator_log
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.any(|event| {
|
||||
event["event"] == "heal_task_state"
|
||||
&& event["task_id"] == client_token
|
||||
&& event["heal_type"] == "cluster"
|
||||
&& event["state"] == "started"
|
||||
}),
|
||||
"node 0 must have started the exact admin task before interruption"
|
||||
);
|
||||
let pre_interrupt_operations = &pre_interrupt_status["healOperations"];
|
||||
assert_eq!(
|
||||
pre_interrupt_operations["activeBySource"]["admin"].as_u64(),
|
||||
Some(1),
|
||||
"interruption must occur while the single admin task is active: {pre_interrupt_status}"
|
||||
);
|
||||
if !background_enabled {
|
||||
assert!(
|
||||
only_admin_heal_is_active(&pre_interrupt_status),
|
||||
"isolated interruption must retain only the admin task: {pre_interrupt_status}"
|
||||
);
|
||||
assert_eq!(
|
||||
pre_interrupt_replacement["cluster"]["records"].as_array().map(Vec::len),
|
||||
Some(0),
|
||||
"root-heal interruption point must not retain an automatic replacement generation: {pre_interrupt_replacement}"
|
||||
);
|
||||
}
|
||||
info!(
|
||||
event = "heal_interruption_checkpoint",
|
||||
event = "heal_restart_checkpoint",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
background_enabled,
|
||||
interruption_node,
|
||||
interruption_kind,
|
||||
partial_metadata_count = partial_count,
|
||||
verified_key = partial_manifest.key,
|
||||
"Observed partial rebuild before interruption"
|
||||
partial_count,
|
||||
"Verified unique admin owner before target interruption"
|
||||
);
|
||||
|
||||
let target_pid = cluster.nodes[1].process.as_ref().ok_or("target process is not running")?.id();
|
||||
if scenario == InterruptionScenario::TargetEndpointBlackhole {
|
||||
let node_pids = cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.map(|node| {
|
||||
node.process
|
||||
.as_ref()
|
||||
.ok_or("cluster process is not running")
|
||||
.map(std::process::Child::id)
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
timeout(Duration::from_secs(2), TcpStream::connect(&cluster.nodes[1].address)).await??;
|
||||
let mut blackhole = TcpPortBlackhole::install(&cluster.nodes[1].address)?;
|
||||
let blocked_connect = timeout(Duration::from_millis(500), TcpStream::connect(&cluster.nodes[1].address)).await;
|
||||
assert!(
|
||||
blocked_connect.is_err(),
|
||||
"target endpoint connection must time out while the OUTPUT DROP rule is active: {blocked_connect:?}"
|
||||
);
|
||||
|
||||
let stable_window_secs = std::env::var("RUSTFS_HEAL_CHAOS_BLACKHOLE_STABLE_SECS")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.unwrap_or(2)
|
||||
.clamp(1, 5);
|
||||
let blackhole_timeout_secs = std::env::var("RUSTFS_HEAL_CHAOS_BLACKHOLE_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.unwrap_or(20)
|
||||
.clamp(stable_window_secs + 1, 60);
|
||||
let blackhole_deadline = Instant::now() + Duration::from_secs(blackhole_timeout_secs);
|
||||
let mut stable_count = metadata_count(&replaced_disk, bucket, &expected_manifests);
|
||||
let mut stable_since = Instant::now();
|
||||
loop {
|
||||
for (node_index, (node, expected_pid)) in cluster.nodes.iter_mut().zip(&node_pids).enumerate() {
|
||||
let process = node
|
||||
.process
|
||||
.as_mut()
|
||||
.ok_or_else(|| format!("node {node_index} process disappeared"))?;
|
||||
assert_eq!(process.id(), *expected_pid, "node {node_index} PID changed during endpoint blackhole");
|
||||
assert!(process.try_wait()?.is_none(), "node {node_index} exited during endpoint blackhole");
|
||||
}
|
||||
|
||||
let current_count = metadata_count(&replaced_disk, bucket, &expected_manifests);
|
||||
if current_count == expected_manifests.len() {
|
||||
return Err("root heal completed before the endpoint blackhole became observable".into());
|
||||
}
|
||||
if current_count != stable_count {
|
||||
stable_count = current_count;
|
||||
stable_since = Instant::now();
|
||||
}
|
||||
if stable_since.elapsed() >= Duration::from_secs(stable_window_secs) {
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= blackhole_deadline {
|
||||
return Err(format!(
|
||||
"target rebuild never remained stable for {stable_window_secs}s during the endpoint blackhole: last_count={stable_count}, total={}",
|
||||
expected_manifests.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
|
||||
let blocked_task_body = timeout(
|
||||
Duration::from_secs(5),
|
||||
signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key),
|
||||
)
|
||||
.await??;
|
||||
let blocked_task: serde_json::Value = serde_json::from_str(&blocked_task_body)
|
||||
.map_err(|err| format!("blackholed task status is not JSON ({err}): {blocked_task_body}"))?;
|
||||
assert_eq!(
|
||||
blocked_task["summary"].as_str(),
|
||||
Some("running"),
|
||||
"the original admin task must remain resumable during the endpoint blackhole: {blocked_task}"
|
||||
);
|
||||
assert!(
|
||||
census_object_version_on_disk(&replaced_disk, bucket, &partial_manifest.key, None)?
|
||||
.matches_manifest(&partial_manifest.shard_census),
|
||||
"the witnessed complete shard must survive the endpoint blackhole"
|
||||
);
|
||||
|
||||
blackhole.restore()?;
|
||||
timeout(Duration::from_secs(2), TcpStream::connect(&cluster.nodes[1].address)).await??;
|
||||
info!(
|
||||
event = "heal_endpoint_blackhole_restored",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
interruption_kind,
|
||||
stable_metadata_count = stable_count,
|
||||
stable_window_secs,
|
||||
"Restored target endpoint forwarding"
|
||||
);
|
||||
} else {
|
||||
cluster.stop_node(interruption_node)?;
|
||||
let stopped_count = metadata_count(&replaced_disk, bucket, &expected_manifests);
|
||||
assert!(
|
||||
stopped_count > 0 && stopped_count < expected_manifests.len(),
|
||||
"node {interruption_node} must stop during a partial rebuild, observed before stop={partial_count}, after stop={stopped_count}, total={}",
|
||||
expected_manifests.len()
|
||||
);
|
||||
assert!(
|
||||
census_object_version_on_disk(&replaced_disk, bucket, &partial_manifest.key, None)?
|
||||
.matches_manifest(&partial_manifest.shard_census),
|
||||
"the witnessed complete shard must survive interruption"
|
||||
);
|
||||
let unclean_shutdown_marker = Path::new(&cluster.nodes[interruption_node].data_dir)
|
||||
.join(".rustfs.sys")
|
||||
.join("unclean-shutdown");
|
||||
if background_enabled {
|
||||
assert!(
|
||||
unclean_shutdown_marker.is_file(),
|
||||
"background restart must retain the real unclean-shutdown marker"
|
||||
);
|
||||
} else {
|
||||
match std::fs::remove_file(&unclean_shutdown_marker) {
|
||||
Ok(()) => {}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(error) => {
|
||||
return Err(
|
||||
format!("failed to isolate unclean recovery marker {unclean_shutdown_marker:?}: {error}").into()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
cluster.start_node(interruption_node).await?;
|
||||
if interruption_node == 0 {
|
||||
let target = cluster.nodes[1]
|
||||
.process
|
||||
.as_mut()
|
||||
.ok_or("target process disappeared during coordinator restart")?;
|
||||
assert_eq!(target.id(), target_pid, "coordinator restart must not replace the target process");
|
||||
assert!(target.try_wait()?.is_none(), "the target must remain alive during coordinator restart");
|
||||
cluster.stop_node(1)?;
|
||||
let stopped_count = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
assert!(
|
||||
stopped_count > 0 && stopped_count < expected_manifests.len(),
|
||||
"the target must stop after a partial rebuild, observed before stop={partial_count}, after stop={stopped_count}, total={}",
|
||||
expected_manifests.len()
|
||||
);
|
||||
let unclean_shutdown_marker = replaced_disk.join(".rustfs.sys").join("unclean-shutdown");
|
||||
match std::fs::remove_file(&unclean_shutdown_marker) {
|
||||
Ok(()) => {}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(error) => {
|
||||
return Err(format!("failed to isolate unclean recovery marker {unclean_shutdown_marker:?}: {error}").into());
|
||||
}
|
||||
}
|
||||
|
||||
let scanner_cycle_floor = if background_enabled {
|
||||
Some(std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)?.as_secs())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
cluster.start_node(1).await?;
|
||||
|
||||
let heal_timeout_secs = std::env::var("RUSTFS_HEAL_REPLACED_DISK_TIMEOUT_SECS")
|
||||
.ok()
|
||||
@@ -1290,22 +832,13 @@ mod tests {
|
||||
{
|
||||
let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
let pool_metadata_matches = match &expected_pool_metadata {
|
||||
Some(expected) => {
|
||||
census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?
|
||||
.matches_manifest(expected)
|
||||
}
|
||||
None => true,
|
||||
};
|
||||
if matching == expected_manifests.len() && outage_census.is_complete() && pool_metadata_matches {
|
||||
if matching == expected_manifests.len() && outage_census.is_complete() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if Instant::now() >= heal_deadline {
|
||||
let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
let pool_metadata =
|
||||
census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?;
|
||||
let final_status = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key)
|
||||
.await
|
||||
.unwrap_or_else(|err| format!("status request failed: {err}"));
|
||||
@@ -1325,7 +858,7 @@ mod tests {
|
||||
Err(_) => "replacement status request exceeded 5s diagnostic budget".to_string(),
|
||||
};
|
||||
return Err(format!(
|
||||
"root heal did not recover after {interruption_kind} within {heal_timeout_secs}s: baseline={matching}/{}, outage={outage_census:?}, pool_metadata={pool_metadata:?}, status={final_status}, task_status={task_status}, pre_interrupt_status={pre_interrupt_status}, pre_heal_replacement={pre_heal_replacement}, pre_interrupt_replacement={pre_interrupt_replacement}, replacement_status={replacement_status}",
|
||||
"root heal did not resume after target restart within {heal_timeout_secs}s: baseline={matching}/{}, outage={outage_census:?}, status={final_status}, task_status={task_status}, pre_interrupt_status={pre_interrupt_status}, pre_heal_replacement={pre_heal_replacement}, replacement_status={replacement_status}",
|
||||
expected_manifests.len()
|
||||
)
|
||||
.into());
|
||||
@@ -1352,17 +885,6 @@ mod tests {
|
||||
"the outage object must be rebuilt into its own missing erasure slot"
|
||||
);
|
||||
|
||||
if let Some(cycle_end) = scanner_cycle_floor {
|
||||
wait_for_scanner_cycle_after(&cluster, cycle_end).await?;
|
||||
}
|
||||
|
||||
let mut expected_keys = expected_manifests
|
||||
.iter()
|
||||
.map(|manifest| manifest.key.clone())
|
||||
.collect::<HashSet<_>>();
|
||||
assert!(expected_keys.insert(outage_key.to_string()));
|
||||
assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||
|
||||
let target_client = cluster.create_s3_client(1)?;
|
||||
for expected in &expected_manifests {
|
||||
let response = target_client.get_object().bucket(bucket).key(&expected.key).send().await?;
|
||||
@@ -1392,30 +914,6 @@ mod tests {
|
||||
let task_status_body = signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let task_status: serde_json::Value = serde_json::from_str(&task_status_body)
|
||||
.map_err(|err| format!("heal task status is not JSON ({err}): {task_status_body}"))?;
|
||||
if interruption_node == 0 {
|
||||
// Admin tasks are process-local. Physical and queue convergence
|
||||
// above establish recovery; a lost task must not report success.
|
||||
assert_eq!(
|
||||
task_status["summary"].as_str(),
|
||||
Some("notFound"),
|
||||
"interrupted task status: {task_status}"
|
||||
);
|
||||
assert_eq!(
|
||||
task_status["detail"].as_str(),
|
||||
Some("heal task not found or expired"),
|
||||
"interrupted admin task must be explicitly unavailable: {task_status}"
|
||||
);
|
||||
info!(
|
||||
event = "heal_interruption_recovered",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
interruption_node,
|
||||
interruption_kind,
|
||||
task_state = "not_found",
|
||||
"Physical recovery completed after coordinator restart"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
if task_status["summary"].as_str() != Some("finished") {
|
||||
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
|
||||
}
|
||||
|
||||
@@ -560,12 +560,18 @@ async fn test_multipart_encryption_type(
|
||||
.set_parts(Some(completed_parts))
|
||||
.build();
|
||||
|
||||
let complete_request = s3_client
|
||||
let mut complete_request = s3_client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(object_key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(completed_multipart_upload);
|
||||
if matches!(encryption_type, EncryptionType::SSEC) {
|
||||
complete_request = complete_request
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(sse_c_key.as_ref().unwrap())
|
||||
.sse_customer_key_md5(sse_c_md5.as_ref().unwrap());
|
||||
}
|
||||
let _complete_output = complete_request.send().await?;
|
||||
|
||||
// Download and verify
|
||||
|
||||
@@ -23,17 +23,10 @@ pub mod common;
|
||||
#[cfg(test)]
|
||||
pub mod chaos;
|
||||
|
||||
// Programmable S3 target for replication failure-path tests (backlog#1147 repl-8)
|
||||
// and on-demand-migration source scenarios (backlog#2151).
|
||||
// Programmable S3 target for replication failure-path tests (backlog#1147 repl-8).
|
||||
#[cfg(test)]
|
||||
pub mod fake_s3_target;
|
||||
|
||||
// On-demand migration (backlog#2147): shared two-server environment, admin
|
||||
// wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
|
||||
// added by later ODM tasks.
|
||||
#[cfg(test)]
|
||||
pub mod on_demand_migration;
|
||||
|
||||
// Socket-level network fault-injection proxy for black-box cluster tests
|
||||
// (backlog#1325 network fault-injection block): latency / blackhole / one-way
|
||||
// partition on the wire between nodes. Serves #1312/#1319 (lock-plane one-way
|
||||
|
||||
@@ -225,8 +225,8 @@ fn encode_unsigned_aws_chunked_with_sha256_trailer(decoded: &[u8]) -> Vec<u8> {
|
||||
let checksum = sha256_base64(decoded);
|
||||
let mut encoded = format!("{:x}\r\n", decoded.len()).into_bytes();
|
||||
encoded.extend_from_slice(decoded);
|
||||
encoded.extend_from_slice(b"\r\n0\r\n");
|
||||
encoded.extend_from_slice(format!("x-amz-checksum-sha256:{checksum}\r\n\r\n").as_bytes());
|
||||
encoded.extend_from_slice(b"\r\n0\r\n\r\n");
|
||||
encoded.extend_from_slice(format!("x-amz-checksum-sha256:{checksum}").as_bytes());
|
||||
encoded
|
||||
}
|
||||
|
||||
@@ -549,68 +549,6 @@ async fn tampered_upload_part_payload_is_rejected() -> Result<(), Box<dyn std::e
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// s3s v0.16 validates the aws-chunked decoded length while RustFS consumes the
|
||||
/// body stream. Mismatches are client body errors and must not leak as 500s.
|
||||
#[tokio::test]
|
||||
async fn aws_chunked_decoded_length_mismatch_returns_incomplete_body() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
setup(&mut env).await?;
|
||||
|
||||
for (key, declared_len) in [
|
||||
("decoded-length-overrun.bin", 3_usize),
|
||||
("decoded-length-shortfall.bin", 9_usize),
|
||||
] {
|
||||
let decoded = b"decoded";
|
||||
assert_ne!(declared_len, decoded.len(), "test case must exercise a mismatch");
|
||||
let encoded_body = encode_unsigned_aws_chunked_with_sha256_trailer(decoded);
|
||||
let decoded_content_length = declared_len.to_string();
|
||||
let path = format!("/{BUCKET}/{key}");
|
||||
let signer = SigV4::new(&env);
|
||||
let extra_signed_headers = [
|
||||
("content-encoding", "aws-chunked"),
|
||||
("x-amz-decoded-content-length", decoded_content_length.as_str()),
|
||||
("x-amz-trailer", "x-amz-checksum-sha256"),
|
||||
];
|
||||
let headers = signer.sign_with_extra_headers("PUT", &path, "", UNSIGNED_PAYLOAD_TRAILER, &extra_signed_headers);
|
||||
|
||||
let response = local_http_client()
|
||||
.put(format!("{}{}", env.url, path))
|
||||
.header("authorization", &headers.authorization)
|
||||
.header("content-encoding", "aws-chunked")
|
||||
.header("x-amz-content-sha256", &headers.content_sha256)
|
||||
.header("x-amz-date", &headers.amz_date)
|
||||
.header("x-amz-decoded-content-length", &decoded_content_length)
|
||||
.header("x-amz-trailer", "x-amz-checksum-sha256")
|
||||
.body(encoded_body)
|
||||
.timeout(std::time::Duration::from_secs(10))
|
||||
.send()
|
||||
.await?;
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"decoded length mismatch must be a client error, body:\n{body}"
|
||||
);
|
||||
assert_error_code(&body, "IncompleteBody");
|
||||
|
||||
let absent = env
|
||||
.create_s3_client()
|
||||
.get_object()
|
||||
.bucket(BUCKET)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("decoded length mismatch must not publish an object");
|
||||
assert_eq!(absent.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
assert_eq!(absent.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey"));
|
||||
}
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// (e) A request whose `x-amz-date` is skewed beyond the server's tolerance
|
||||
/// (s3s default 900s / 15 min) must be rejected with RequestTimeTooSkewed /
|
||||
/// 403. The signature is otherwise valid: the credential-scope date and
|
||||
|
||||
@@ -1,452 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Shared environment for on-demand migration (ODM) end-to-end tests.
|
||||
//!
|
||||
//! [`OdmTestEnv`] pairs one RustFS server under test with one in-process
|
||||
//! programmable S3 source ([`FakeS3Target`]). Admin calls target the route
|
||||
//! convention fixed by the tracking plan
|
||||
//! (`/rustfs/admin/v3/on-demand-migration/{bucket}`, JSON bodies); the
|
||||
//! server side lands with ODM-07, so until then the wrappers compile but are
|
||||
//! not exercised by the harness self-test.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, signed_request};
|
||||
use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FakeS3TargetOptions, SeedMetadata};
|
||||
use aws_config::retry::RetryConfig;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use bytes::Bytes;
|
||||
use serde::Serialize;
|
||||
use std::fmt;
|
||||
|
||||
pub type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
/// Module switch the server reads at startup (`false` before GA). The harness
|
||||
/// turns it on so scenario tests exercise the feature without repeating it.
|
||||
pub const ODM_MODULE_SWITCH_ENV: &str = "RUSTFS_ON_DEMAND_MIGRATION_ENABLED";
|
||||
/// Admin route prefix; the bucket name is appended as one path segment.
|
||||
pub const ODM_ADMIN_ROUTE: &str = "/rustfs/admin/v3/on-demand-migration";
|
||||
/// Region the fake source is addressed with (it accepts any SigV4 region).
|
||||
pub const FAKE_SOURCE_REGION: &str = "us-east-1";
|
||||
|
||||
/// Wire form of the bucket-level ODM configuration (ODM-01 model). Every
|
||||
/// field is public so a scenario can tweak one knob and serialize the rest
|
||||
/// with the documented defaults.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct OdmSourceSpec {
|
||||
pub version: u32,
|
||||
pub enabled: bool,
|
||||
pub source: OdmSource,
|
||||
pub filter: OdmFilter,
|
||||
pub policy: OdmPolicy,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct OdmSource {
|
||||
pub provider: String,
|
||||
pub endpoint: String,
|
||||
pub region: String,
|
||||
pub bucket: String,
|
||||
pub path_style: String,
|
||||
pub credentials: Option<OdmCredentials>,
|
||||
pub tls: OdmTls,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize)]
|
||||
pub struct OdmCredentials {
|
||||
pub access_key: String,
|
||||
pub secret_key: String,
|
||||
pub session_token: Option<String>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for OdmCredentials {
|
||||
/// Test logs are captured into CI artifacts; keep the secret out of them.
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("OdmCredentials")
|
||||
.field("access_key", &self.access_key)
|
||||
.field("secret_key", &"REDACTED")
|
||||
.field("session_token", &self.session_token.as_ref().map(|_| "REDACTED"))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize)]
|
||||
pub struct OdmTls {
|
||||
pub skip_verify: bool,
|
||||
pub ca_cert_pem: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize)]
|
||||
pub struct OdmFilter {
|
||||
pub prefix: Option<String>,
|
||||
pub source_prefix: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct OdmPolicy {
|
||||
pub head: String,
|
||||
pub range_get: String,
|
||||
pub source_error: String,
|
||||
pub respect_local_delete_marker: bool,
|
||||
pub preserve_etag: bool,
|
||||
pub copy_tags: bool,
|
||||
pub emit_events: bool,
|
||||
pub negative_cache_ttl_secs: u64,
|
||||
pub inline_max_bytes: u64,
|
||||
pub multipart_part_size_bytes: u64,
|
||||
pub max_concurrent_pulls: u32,
|
||||
pub pull_queue_capacity: u32,
|
||||
pub source_timeout: OdmSourceTimeout,
|
||||
pub bandwidth_limit_bytes_per_sec: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct OdmSourceTimeout {
|
||||
pub connect_ms: u64,
|
||||
pub first_byte_ms: u64,
|
||||
pub idle_ms: u64,
|
||||
}
|
||||
|
||||
impl Default for OdmPolicy {
|
||||
/// The ODM-01 defaults verbatim.
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
head: "proxy".to_string(),
|
||||
range_get: "serve_and_backfill".to_string(),
|
||||
source_error: "propagate".to_string(),
|
||||
respect_local_delete_marker: true,
|
||||
preserve_etag: true,
|
||||
copy_tags: false,
|
||||
emit_events: true,
|
||||
negative_cache_ttl_secs: 30,
|
||||
inline_max_bytes: 16 * 1024 * 1024,
|
||||
multipart_part_size_bytes: 64 * 1024 * 1024,
|
||||
max_concurrent_pulls: 8,
|
||||
pull_queue_capacity: 1024,
|
||||
source_timeout: OdmSourceTimeout {
|
||||
connect_ms: 5_000,
|
||||
first_byte_ms: 15_000,
|
||||
idle_ms: 30_000,
|
||||
},
|
||||
bandwidth_limit_bytes_per_sec: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl OdmSourceSpec {
|
||||
/// Enabled configuration pointing at a bucket on the fake source with the
|
||||
/// fixture credentials, path-style addressing, and default policy.
|
||||
pub fn for_fake_source(source: &FakeS3Target, source_bucket: impl Into<String>) -> Self {
|
||||
Self::new(
|
||||
"s3",
|
||||
source.endpoint(),
|
||||
FAKE_SOURCE_REGION,
|
||||
source_bucket,
|
||||
FAKE_ACCESS_KEY,
|
||||
FAKE_SECRET_KEY,
|
||||
)
|
||||
}
|
||||
|
||||
/// Enabled configuration pointing at a bucket on a second RustFS server
|
||||
/// (see [`start_source_rustfs`]).
|
||||
pub fn for_rustfs_source(source: &RustFSTestEnvironment, source_bucket: impl Into<String>) -> Self {
|
||||
Self::new(
|
||||
"rustfs",
|
||||
&source.url,
|
||||
FAKE_SOURCE_REGION,
|
||||
source_bucket,
|
||||
&source.access_key,
|
||||
&source.secret_key,
|
||||
)
|
||||
}
|
||||
|
||||
fn new(
|
||||
provider: &str,
|
||||
endpoint: &str,
|
||||
region: &str,
|
||||
source_bucket: impl Into<String>,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Self {
|
||||
Self {
|
||||
version: 1,
|
||||
enabled: true,
|
||||
source: OdmSource {
|
||||
provider: provider.to_string(),
|
||||
endpoint: endpoint.to_string(),
|
||||
region: region.to_string(),
|
||||
bucket: source_bucket.into(),
|
||||
path_style: "path".to_string(),
|
||||
credentials: Some(OdmCredentials {
|
||||
access_key: access_key.to_string(),
|
||||
secret_key: secret_key.to_string(),
|
||||
session_token: None,
|
||||
}),
|
||||
tls: OdmTls::default(),
|
||||
},
|
||||
filter: OdmFilter::default(),
|
||||
policy: OdmPolicy::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_json(&self) -> serde_json::Value {
|
||||
serde_json::to_value(self).expect("ODM source spec serializes")
|
||||
}
|
||||
}
|
||||
|
||||
/// Backfill job control (ODM-12 route shape).
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum BackfillOp {
|
||||
Start(BackfillRequest),
|
||||
Cancel,
|
||||
Status,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize)]
|
||||
pub struct BackfillRequest {
|
||||
pub prefix: Option<String>,
|
||||
pub skip_existing: Option<String>,
|
||||
pub dry_run: bool,
|
||||
}
|
||||
|
||||
/// Status plus raw body of an admin call, so a scenario can assert on the
|
||||
/// HTTP status first and only then parse the JSON.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AdminResponse {
|
||||
pub status: u16,
|
||||
pub body: String,
|
||||
}
|
||||
|
||||
impl AdminResponse {
|
||||
pub fn json(&self) -> Result<serde_json::Value, BoxError> {
|
||||
Ok(serde_json::from_str(&self.body)?)
|
||||
}
|
||||
}
|
||||
|
||||
/// One object to seed into the source.
|
||||
#[derive(Clone)]
|
||||
pub struct SeedObject {
|
||||
pub key: String,
|
||||
pub body: Bytes,
|
||||
pub metadata: SeedMetadata,
|
||||
}
|
||||
|
||||
impl SeedObject {
|
||||
pub fn new(key: impl Into<String>, body: impl Into<Bytes>) -> Self {
|
||||
Self {
|
||||
key: key.into(),
|
||||
body: body.into(),
|
||||
metadata: SeedMetadata::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_metadata(mut self, metadata: SeedMetadata) -> Self {
|
||||
self.metadata = metadata;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// RustFS under test plus its fake S3 source.
|
||||
pub struct OdmTestEnv {
|
||||
pub rustfs: RustFSTestEnvironment,
|
||||
pub source: FakeS3Target,
|
||||
/// S3 client for the RustFS under test.
|
||||
pub client: Client,
|
||||
}
|
||||
|
||||
impl OdmTestEnv {
|
||||
/// Start a fake source with default limits and a RustFS server with the
|
||||
/// ODM module switch enabled.
|
||||
pub async fn start() -> Result<Self, BoxError> {
|
||||
Self::start_with_options(FakeS3TargetOptions::default()).await
|
||||
}
|
||||
|
||||
pub async fn start_with_options(options: FakeS3TargetOptions) -> Result<Self, BoxError> {
|
||||
let source = FakeS3Target::start_with_options(options).await?;
|
||||
let mut rustfs = RustFSTestEnvironment::new().await?;
|
||||
rustfs
|
||||
.start_rustfs_server_with_env(vec![], &[(ODM_MODULE_SWITCH_ENV, "true")])
|
||||
.await?;
|
||||
let client = rustfs.create_s3_client();
|
||||
Ok(Self { rustfs, source, client })
|
||||
}
|
||||
|
||||
/// S3 client addressing the fake source directly, for assertions on the
|
||||
/// source's own state. Retries are off so a scripted fault is consumed by
|
||||
/// exactly the request the test issued.
|
||||
pub fn source_client(&self) -> Client {
|
||||
fake_source_client(&self.source)
|
||||
}
|
||||
|
||||
/// Enabled ODM configuration for `source_bucket` on the fake source.
|
||||
pub fn fake_source_spec(&self, source_bucket: impl Into<String>) -> OdmSourceSpec {
|
||||
OdmSourceSpec::for_fake_source(&self.source, source_bucket)
|
||||
}
|
||||
|
||||
/// `PUT /rustfs/admin/v3/on-demand-migration/{bucket}` with the JSON spec.
|
||||
pub async fn configure_source(&self, bucket: &str, spec: &OdmSourceSpec) -> Result<AdminResponse, BoxError> {
|
||||
self.admin(http::Method::PUT, &format!("/{bucket}"), Some(spec.to_json()))
|
||||
.await
|
||||
}
|
||||
|
||||
/// Same as [`Self::configure_source`] with `dry-run=true`: validate and
|
||||
/// probe without persisting.
|
||||
pub async fn validate_source(&self, bucket: &str, spec: &OdmSourceSpec) -> Result<AdminResponse, BoxError> {
|
||||
self.admin(http::Method::PUT, &format!("/{bucket}?dry-run=true"), Some(spec.to_json()))
|
||||
.await
|
||||
}
|
||||
|
||||
/// `GET .../{bucket}`: redacted configuration, 404 when unconfigured.
|
||||
pub async fn get_config(&self, bucket: &str) -> Result<AdminResponse, BoxError> {
|
||||
self.admin(http::Method::GET, &format!("/{bucket}"), None).await
|
||||
}
|
||||
|
||||
/// `DELETE .../{bucket}`: remove the configuration (idempotent).
|
||||
pub async fn disable(&self, bucket: &str) -> Result<AdminResponse, BoxError> {
|
||||
self.admin(http::Method::DELETE, &format!("/{bucket}"), None).await
|
||||
}
|
||||
|
||||
/// `GET .../{bucket}/status`: runtime snapshot.
|
||||
pub async fn status(&self, bucket: &str) -> Result<AdminResponse, BoxError> {
|
||||
self.admin(http::Method::GET, &format!("/{bucket}/status"), None).await
|
||||
}
|
||||
|
||||
/// Backfill control: `POST .../{bucket}/backfill?op=start|cancel` or
|
||||
/// `GET .../{bucket}/backfill` for the checkpoint.
|
||||
pub async fn backfill(&self, bucket: &str, op: BackfillOp) -> Result<AdminResponse, BoxError> {
|
||||
match op {
|
||||
BackfillOp::Start(request) => {
|
||||
self.admin(
|
||||
http::Method::POST,
|
||||
&format!("/{bucket}/backfill?op=start"),
|
||||
Some(serde_json::to_value(request)?),
|
||||
)
|
||||
.await
|
||||
}
|
||||
BackfillOp::Cancel => {
|
||||
self.admin(http::Method::POST, &format!("/{bucket}/backfill?op=cancel"), None)
|
||||
.await
|
||||
}
|
||||
BackfillOp::Status => self.admin(http::Method::GET, &format!("/{bucket}/backfill"), None).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn admin(
|
||||
&self,
|
||||
method: http::Method,
|
||||
path_and_query: &str,
|
||||
body: Option<serde_json::Value>,
|
||||
) -> Result<AdminResponse, BoxError> {
|
||||
let url = format!("{}{ODM_ADMIN_ROUTE}{path_and_query}", self.rustfs.url);
|
||||
let body = body.map(|value| serde_json::to_vec(&value)).transpose()?;
|
||||
let content_type = body.is_some().then_some("application/json");
|
||||
let response = signed_request(method, &url, &self.rustfs.access_key, &self.rustfs.secret_key, body, content_type).await?;
|
||||
Ok(AdminResponse {
|
||||
status: response.status().as_u16(),
|
||||
body: response.text().await?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Store objects directly in the fake source (no wire traffic, no journal
|
||||
/// entries). Returns the ETags in input order.
|
||||
pub fn seed_source(&self, source_bucket: &str, objects: &[SeedObject]) -> Vec<String> {
|
||||
objects
|
||||
.iter()
|
||||
.map(|object| {
|
||||
self.source
|
||||
.put_seed_object(source_bucket, object.key.clone(), object.body.clone(), &object.metadata)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Whether `key` is listed by the RustFS under test. Listing is served from
|
||||
/// local state only, so this does not trigger a migration the way GET or
|
||||
/// HEAD would.
|
||||
pub async fn local_key_listed(&self, bucket: &str, key: &str) -> Result<bool, BoxError> {
|
||||
let listed = self
|
||||
.client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.prefix(key)
|
||||
.max_keys(1)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(listed.contents().iter().any(|object| object.key() == Some(key)))
|
||||
}
|
||||
|
||||
/// Panics unless `key` is stored locally with exactly `expected` bytes.
|
||||
/// Presence is checked through listing first so a missing object fails
|
||||
/// here instead of being pulled from the source by the GET.
|
||||
pub async fn assert_local_present(&self, bucket: &str, key: &str, expected: &[u8]) {
|
||||
assert!(
|
||||
self.local_key_listed(bucket, key)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("listing {bucket}/{key} failed: {error}")),
|
||||
"{bucket}/{key} must be present locally"
|
||||
);
|
||||
let body = self
|
||||
.client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("GET {bucket}/{key} failed: {error}"))
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("reading {bucket}/{key} failed: {error}"))
|
||||
.into_bytes();
|
||||
assert_eq!(body.as_ref(), expected, "{bucket}/{key} local content mismatch");
|
||||
}
|
||||
|
||||
/// Panics if `key` is listed locally.
|
||||
pub async fn assert_local_absent(&self, bucket: &str, key: &str) {
|
||||
assert!(
|
||||
!self
|
||||
.local_key_listed(bucket, key)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("listing {bucket}/{key} failed: {error}")),
|
||||
"{bucket}/{key} must be absent locally"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// S3 client for the fake source with retries disabled (see
|
||||
/// [`OdmTestEnv::source_client`]).
|
||||
pub fn fake_source_client(source: &FakeS3Target) -> Client {
|
||||
let credentials = Credentials::new(FAKE_ACCESS_KEY, FAKE_SECRET_KEY, None, None, "odm-fake-source");
|
||||
Client::from_conf(
|
||||
aws_sdk_s3::Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new(FAKE_SOURCE_REGION))
|
||||
.endpoint_url(source.endpoint())
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.retry_config(RetryConfig::standard().with_max_attempts(1))
|
||||
.http_client(SmithyHttpClientBuilder::new().build_http())
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Start a second, fully independent RustFS process (own port, data
|
||||
/// directory, and default credentials) to act as a real S3 source. It is
|
||||
/// spawned the same way `reliant::tiering` starts its cold tier; the process
|
||||
/// is stopped and its directory removed when the returned environment drops.
|
||||
pub async fn start_source_rustfs() -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut source = RustFSTestEnvironment::new().await?;
|
||||
source.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
Ok(source)
|
||||
}
|
||||
@@ -1,606 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Self-test of the ODM harness (rustfs/backlog#2151): the fake source's
|
||||
//! migration-facing surface (ListObjectsV2 paging, `Range`, unversioned
|
||||
//! buckets, metadata replay, fault actions) and the two-server environment.
|
||||
//! No ODM behavior is exercised here.
|
||||
|
||||
use super::common::{OdmTestEnv, SeedObject, fake_source_client, start_source_rustfs};
|
||||
use crate::fake_s3_target::{BucketMode, FakeS3Target, FakeS3TargetOptions, FaultAction, Operation, SeedMetadata};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::{ByteStream, DateTime};
|
||||
use bytes::Bytes;
|
||||
use std::collections::BTreeSet;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
const SOURCE_BUCKET: &str = "odm-source";
|
||||
|
||||
/// Position-dependent payload so a misaligned range read is caught.
|
||||
fn payload(len: usize) -> Bytes {
|
||||
(0..len).map(|index| (index % 251) as u8).collect::<Vec<u8>>().into()
|
||||
}
|
||||
|
||||
async fn fake_source() -> Result<(FakeS3Target, Client), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let source = FakeS3Target::start().await?;
|
||||
source.create_bucket(SOURCE_BUCKET);
|
||||
let client = fake_source_client(&source);
|
||||
Ok((source, client))
|
||||
}
|
||||
|
||||
/// Full ListObjectsV2 traversal. Returns `(keys, common prefixes, pages)` and
|
||||
/// checks the page shape on the way: every page except the last is full and
|
||||
/// truncated, the last carries no continuation token.
|
||||
async fn list_all(
|
||||
client: &Client,
|
||||
prefix: Option<&str>,
|
||||
delimiter: Option<&str>,
|
||||
start_after: Option<&str>,
|
||||
max_keys: i32,
|
||||
) -> Result<(Vec<String>, Vec<String>, usize), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut keys = Vec::new();
|
||||
let mut prefixes = Vec::new();
|
||||
let mut pages = 0usize;
|
||||
let mut token: Option<String> = None;
|
||||
loop {
|
||||
let page = client
|
||||
.list_objects_v2()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.set_prefix(prefix.map(str::to_string))
|
||||
.set_delimiter(delimiter.map(str::to_string))
|
||||
.set_start_after(start_after.map(str::to_string))
|
||||
.max_keys(max_keys)
|
||||
.set_continuation_token(token.clone())
|
||||
.send()
|
||||
.await?;
|
||||
pages += 1;
|
||||
let page_keys: Vec<String> = page
|
||||
.contents()
|
||||
.iter()
|
||||
.filter_map(|object| object.key().map(str::to_string))
|
||||
.collect();
|
||||
let page_prefixes: Vec<String> = page
|
||||
.common_prefixes()
|
||||
.iter()
|
||||
.filter_map(|common| common.prefix().map(str::to_string))
|
||||
.collect();
|
||||
let entries = page_keys.len() + page_prefixes.len();
|
||||
assert_eq!(page.key_count(), Some(entries as i32), "KeyCount must count keys and prefixes");
|
||||
assert_eq!(page.continuation_token(), token.as_deref(), "the request token must be echoed");
|
||||
keys.extend(page_keys);
|
||||
prefixes.extend(page_prefixes);
|
||||
if page.is_truncated() == Some(true) {
|
||||
assert_eq!(entries as i32, max_keys, "every truncated page must be full");
|
||||
token = Some(
|
||||
page.next_continuation_token()
|
||||
.expect("truncated page must carry a continuation token")
|
||||
.to_string(),
|
||||
);
|
||||
} else {
|
||||
assert!(page.next_continuation_token().is_none(), "final page must not carry a token");
|
||||
return Ok((keys, prefixes, pages));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_source_list_objects_v2_paginates_with_delimiter() -> TestResult {
|
||||
let (source, client) = fake_source().await?;
|
||||
let mut expected_keys = BTreeSet::new();
|
||||
for directory in 0..30 {
|
||||
for file in 0..30 {
|
||||
expected_keys.insert(format!("d{directory:02}/k{file:03}"));
|
||||
}
|
||||
}
|
||||
for index in 0..100 {
|
||||
expected_keys.insert(format!("top-{index:03}"));
|
||||
}
|
||||
assert_eq!(expected_keys.len(), 1000);
|
||||
for key in &expected_keys {
|
||||
source.put_seed_object(SOURCE_BUCKET, key.clone(), Bytes::from(key.clone()), &SeedMetadata::new());
|
||||
}
|
||||
// A key whose current version is a delete marker must stay hidden.
|
||||
client
|
||||
.put_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key("hidden/marker")
|
||||
.body(ByteStream::from_static(b"gone"))
|
||||
.send()
|
||||
.await?;
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key("hidden/marker")
|
||||
.send()
|
||||
.await?;
|
||||
let expected_sorted: Vec<String> = expected_keys.iter().cloned().collect();
|
||||
let expected_prefixes: Vec<String> = (0..30).map(|directory| format!("d{directory:02}/")).collect();
|
||||
let expected_top: Vec<String> = (0..100).map(|index| format!("top-{index:03}")).collect();
|
||||
|
||||
// Flat traversal in byte order, 1000 keys in pages of 7.
|
||||
let (keys, prefixes, pages) = list_all(&client, None, None, None, 7).await?;
|
||||
assert_eq!(keys, expected_sorted);
|
||||
assert!(prefixes.is_empty());
|
||||
assert_eq!(pages, 143);
|
||||
|
||||
// Delimiter folding: 30 common prefixes then 100 top-level keys, pages of 7.
|
||||
let (keys, prefixes, pages) = list_all(&client, None, Some("/"), None, 7).await?;
|
||||
assert_eq!(prefixes, expected_prefixes);
|
||||
assert_eq!(keys, expected_top);
|
||||
assert_eq!(pages, 19);
|
||||
|
||||
// Empty prefix equals no prefix.
|
||||
let (keys, _, _) = list_all(&client, Some(""), None, None, 1000).await?;
|
||||
assert_eq!(keys, expected_sorted);
|
||||
|
||||
// No match: empty, not truncated, no token.
|
||||
let (keys, prefixes, pages) = list_all(&client, Some("zzz/"), Some("/"), None, 7).await?;
|
||||
assert!(keys.is_empty() && prefixes.is_empty());
|
||||
assert_eq!(pages, 1);
|
||||
let (keys, _, _) = list_all(&client, Some("hidden/"), None, None, 7).await?;
|
||||
assert!(keys.is_empty(), "a current delete marker must hide its key");
|
||||
|
||||
// Exact page boundary: 30 keys under one directory, max-keys=30 -> one
|
||||
// untruncated page.
|
||||
let (keys, prefixes, pages) = list_all(&client, Some("d05/"), Some("/"), None, 30).await?;
|
||||
assert_eq!(keys.len(), 30);
|
||||
assert!(prefixes.is_empty());
|
||||
assert_eq!(pages, 1);
|
||||
|
||||
// start-after skips keys at or before the marker.
|
||||
let (keys, _, _) = list_all(&client, None, None, Some("top-097"), 1000).await?;
|
||||
assert_eq!(keys, ["top-098", "top-099"]);
|
||||
|
||||
// max-keys is clamped to 1000; exactly 1000 keys fit in one page.
|
||||
let (keys, _, pages) = list_all(&client, None, None, None, 5000).await?;
|
||||
assert_eq!(keys.len(), 1000);
|
||||
assert_eq!(pages, 1);
|
||||
|
||||
let listings: Vec<_> = source
|
||||
.requests()
|
||||
.into_iter()
|
||||
.filter(|record| record.operation == Operation::ListObjectsV2)
|
||||
.collect();
|
||||
assert!(listings.len() >= 143 + 19);
|
||||
assert!(listings.iter().any(|record| record.prefix.as_deref() == Some("d05/")));
|
||||
assert!(
|
||||
listings.iter().any(|record| record.continuation_token.is_some()),
|
||||
"resumed pages must journal their continuation token"
|
||||
);
|
||||
assert!(listings.iter().all(|record| record.user_agent.is_some()));
|
||||
source.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_source_range_get_variants_and_416() -> TestResult {
|
||||
let (source, client) = fake_source().await?;
|
||||
let body = payload(1000);
|
||||
source.put_seed_object(SOURCE_BUCKET, "ranged", body.clone(), &SeedMetadata::new());
|
||||
|
||||
for (range, expected_range, expected_slice) in [
|
||||
("bytes=10-19", "bytes 10-19/1000", &body[10..20]),
|
||||
("bytes=990-", "bytes 990-999/1000", &body[990..]),
|
||||
("bytes=-5", "bytes 995-999/1000", &body[995..]),
|
||||
("bytes=0-5000", "bytes 0-999/1000", &body[..]),
|
||||
] {
|
||||
let output = client
|
||||
.get_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key("ranged")
|
||||
.range(range)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(output.content_range(), Some(expected_range), "{range}");
|
||||
assert_eq!(output.accept_ranges(), Some("bytes"), "{range}");
|
||||
assert_eq!(output.content_length(), Some(expected_slice.len() as i64), "{range}");
|
||||
let collected = output.body.collect().await?.into_bytes();
|
||||
assert_eq!(collected.as_ref(), expected_slice, "{range}");
|
||||
}
|
||||
let head = client
|
||||
.head_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key("ranged")
|
||||
.range("bytes=10-19")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(head.content_range(), Some("bytes 10-19/1000"));
|
||||
assert_eq!(head.content_length(), Some(10));
|
||||
|
||||
for range in ["bytes=1000-", "bytes=-0"] {
|
||||
let error = client
|
||||
.get_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key("ranged")
|
||||
.range(range)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("unsatisfiable range must fail");
|
||||
let response = error.raw_response().expect("416 must retain the raw response");
|
||||
assert_eq!(response.status().as_u16(), 416, "{range}");
|
||||
assert_eq!(response.headers().get("content-range"), Some("bytes */1000"), "{range}");
|
||||
assert_eq!(error.code(), Some("InvalidRange"), "{range}");
|
||||
}
|
||||
|
||||
let ranged = source
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == Operation::GetObject && record.range.as_deref() == Some("bytes=10-19"))
|
||||
.expect("the Range header must be journaled verbatim");
|
||||
assert_eq!(ranged.key.as_deref(), Some("ranged"));
|
||||
assert!(source.count_requests(Operation::GetObject, "ranged") >= 6);
|
||||
source.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_source_unversioned_bucket_overwrites_and_deletes() -> TestResult {
|
||||
let (source, client) = fake_source().await?;
|
||||
source.create_bucket_with_mode("plain-source", BucketMode::Unversioned);
|
||||
let versioning = client.get_bucket_versioning().bucket("plain-source").send().await?;
|
||||
assert!(versioning.status().is_none(), "unversioned bucket must report no versioning status");
|
||||
|
||||
let first = client
|
||||
.put_object()
|
||||
.bucket("plain-source")
|
||||
.key("doc")
|
||||
.body(ByteStream::from_static(b"first"))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(first.version_id().is_none());
|
||||
let second = client
|
||||
.put_object()
|
||||
.bucket("plain-source")
|
||||
.key("doc")
|
||||
.body(ByteStream::from_static(b"second"))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(second.version_id().is_none());
|
||||
let get = client.get_object().bucket("plain-source").key("doc").send().await?;
|
||||
assert!(get.version_id().is_none(), "GET must not return x-amz-version-id");
|
||||
assert_eq!(get.body.collect().await?.into_bytes().as_ref(), b"second");
|
||||
let head = client.head_object().bucket("plain-source").key("doc").send().await?;
|
||||
assert!(head.version_id().is_none(), "HEAD must not return x-amz-version-id");
|
||||
assert_eq!(source.stored_versions("plain-source", "doc").len(), 1, "overwrite must replace in place");
|
||||
|
||||
let deleted = client.delete_object().bucket("plain-source").key("doc").send().await?;
|
||||
assert!(deleted.delete_marker().is_none() && deleted.version_id().is_none());
|
||||
let missing = client
|
||||
.get_object()
|
||||
.bucket("plain-source")
|
||||
.key("doc")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("deleted object must be gone");
|
||||
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
assert_eq!(missing.code(), Some("NoSuchKey"));
|
||||
let missing_head = client
|
||||
.head_object()
|
||||
.bucket("plain-source")
|
||||
.key("doc")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("deleted object must fail HEAD");
|
||||
assert_eq!(missing_head.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
assert!(source.stored_versions("plain-source", "doc").is_empty(), "DELETE must not leave a marker");
|
||||
|
||||
// The versioned bucket on the same target keeps its version ids.
|
||||
let versioned = client
|
||||
.put_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key("doc")
|
||||
.body(ByteStream::from_static(b"versioned"))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(versioned.version_id().is_some());
|
||||
source.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_source_replays_standard_and_user_metadata() -> TestResult {
|
||||
let (source, client) = fake_source().await?;
|
||||
let body = payload(4096);
|
||||
let expected_etag = format!("\"{}\"", {
|
||||
use md5::Digest as _;
|
||||
hex_simd::encode_to_string(md5::Md5::digest(&body), hex_simd::AsciiCase::Lower)
|
||||
});
|
||||
// 2026-01-01T00:00:00Z rendered as an HTTP date by the SDK.
|
||||
let expires = DateTime::from_secs(1_767_225_600);
|
||||
client
|
||||
.put_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key("meta")
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.content_type("application/x-odm")
|
||||
.content_encoding("gzip")
|
||||
.content_disposition("attachment; filename=\"meta.bin\"")
|
||||
.content_language("en-US")
|
||||
.cache_control("max-age=60")
|
||||
.expires(expires)
|
||||
.metadata("Foo-Bar", "mixed case name")
|
||||
.metadata("UPPER", "upper name")
|
||||
.metadata("already-lower", "lower name")
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let head = client.head_object().bucket(SOURCE_BUCKET).key("meta").send().await?;
|
||||
let get = client.get_object().bucket(SOURCE_BUCKET).key("meta").send().await?;
|
||||
for (label, content_type, content_encoding, content_disposition, content_language, cache_control, expires_string, e_tag) in [
|
||||
(
|
||||
"HEAD",
|
||||
head.content_type(),
|
||||
head.content_encoding(),
|
||||
head.content_disposition(),
|
||||
head.content_language(),
|
||||
head.cache_control(),
|
||||
head.expires_string(),
|
||||
head.e_tag(),
|
||||
),
|
||||
(
|
||||
"GET",
|
||||
get.content_type(),
|
||||
get.content_encoding(),
|
||||
get.content_disposition(),
|
||||
get.content_language(),
|
||||
get.cache_control(),
|
||||
get.expires_string(),
|
||||
get.e_tag(),
|
||||
),
|
||||
] {
|
||||
assert_eq!(content_type, Some("application/x-odm"), "{label}");
|
||||
assert_eq!(content_encoding, Some("gzip"), "{label}");
|
||||
assert_eq!(content_disposition, Some("attachment; filename=\"meta.bin\""), "{label}");
|
||||
assert_eq!(content_language, Some("en-US"), "{label}");
|
||||
assert_eq!(cache_control, Some("max-age=60"), "{label}");
|
||||
assert_eq!(expires_string, Some("Thu, 01 Jan 2026 00:00:00 GMT"), "{label}");
|
||||
assert_eq!(e_tag, Some(expected_etag.as_str()), "{label}");
|
||||
}
|
||||
for metadata in [head.metadata(), get.metadata()] {
|
||||
let metadata = metadata.expect("user metadata must be replayed");
|
||||
assert_eq!(metadata.get("foo-bar").map(String::as_str), Some("mixed case name"));
|
||||
assert_eq!(metadata.get("upper").map(String::as_str), Some("upper name"));
|
||||
assert_eq!(metadata.get("already-lower").map(String::as_str), Some("lower name"));
|
||||
assert!(!metadata.contains_key("Foo-Bar") && !metadata.contains_key("UPPER"));
|
||||
}
|
||||
assert!(head.last_modified().is_some());
|
||||
assert_eq!(head.last_modified(), get.last_modified());
|
||||
assert_eq!(head.content_length(), Some(4096));
|
||||
assert_eq!(get.body.collect().await?.into_bytes(), body);
|
||||
|
||||
// Seeded objects replay the same way.
|
||||
let seeded_etag = source.put_seed_object(
|
||||
SOURCE_BUCKET,
|
||||
"seeded",
|
||||
Bytes::from_static(b"seeded"),
|
||||
&SeedMetadata::new()
|
||||
.content_type("text/plain")
|
||||
.content_encoding("identity")
|
||||
.cache_control("no-store")
|
||||
.user_metadata("Origin", "seed"),
|
||||
);
|
||||
let seeded = client.head_object().bucket(SOURCE_BUCKET).key("seeded").send().await?;
|
||||
assert_eq!(seeded.e_tag(), Some(format!("\"{seeded_etag}\"").as_str()));
|
||||
assert_eq!(seeded.content_type(), Some("text/plain"));
|
||||
assert_eq!(seeded.content_encoding(), Some("identity"));
|
||||
assert_eq!(seeded.cache_control(), Some("no-store"));
|
||||
assert_eq!(
|
||||
seeded
|
||||
.metadata()
|
||||
.and_then(|metadata| metadata.get("origin"))
|
||||
.map(String::as_str),
|
||||
Some("seed")
|
||||
);
|
||||
source.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_source_fault_actions_truncate_stall_and_status() -> TestResult {
|
||||
let (source, client) = fake_source().await?;
|
||||
let body = payload(4096);
|
||||
source.put_seed_object(SOURCE_BUCKET, "faulty", body.clone(), &SeedMetadata::new());
|
||||
|
||||
// TruncateBodyAt: headers promise 4096 bytes, the body ends after 100.
|
||||
source.inject_for_key(Operation::GetObject, "faulty", FaultAction::TruncateBodyAt(100), 1);
|
||||
let truncated = client.get_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
|
||||
assert_eq!(truncated.content_length(), Some(4096));
|
||||
let short_read = truncated
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect_err("a truncated body must fail to collect");
|
||||
let short_read = short_read.to_string();
|
||||
assert!(!short_read.is_empty());
|
||||
|
||||
// ResponseStatus: arbitrary status with the matching S3 error code.
|
||||
for (code, expected_code) in [
|
||||
(429u16, "SlowDown"),
|
||||
(404, "NoSuchKey"),
|
||||
(500, "InternalError"),
|
||||
(503, "ServiceUnavailable"),
|
||||
] {
|
||||
source.inject(Operation::GetObject, FaultAction::ResponseStatus(code), 1);
|
||||
let error = client
|
||||
.get_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key("faulty")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("scripted status must fail");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(code));
|
||||
assert_eq!(error.code(), Some(expected_code));
|
||||
}
|
||||
|
||||
// Stall: the fully computed response is held before its first byte.
|
||||
source.inject(Operation::HeadObject, FaultAction::Stall(Duration::from_millis(400)), 1);
|
||||
let started = Instant::now();
|
||||
let stalled = client.head_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
|
||||
assert!(started.elapsed() >= Duration::from_millis(350), "stall must delay the first byte");
|
||||
assert_eq!(stalled.content_length(), Some(4096));
|
||||
let post_stall_started = Instant::now();
|
||||
client.head_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
|
||||
assert!(post_stall_started.elapsed() < Duration::from_millis(350), "stall is consumed once");
|
||||
|
||||
// The object is intact once the script is drained.
|
||||
let intact = client.get_object().bucket(SOURCE_BUCKET).key("faulty").send().await?;
|
||||
assert_eq!(intact.body.collect().await?.into_bytes(), body);
|
||||
|
||||
assert_eq!(source.count_requests(Operation::GetObject, "faulty"), 6);
|
||||
assert_eq!(source.count_requests(Operation::HeadObject, "faulty"), 2);
|
||||
assert_eq!(source.count_requests(Operation::GetObject, "other"), 0);
|
||||
let records = source.requests();
|
||||
assert!(
|
||||
records.iter().all(|record| record
|
||||
.user_agent
|
||||
.as_deref()
|
||||
.is_some_and(|agent| agent.contains("aws-sdk-rust"))),
|
||||
"the SDK user agent must be journaled"
|
||||
);
|
||||
assert!(
|
||||
records
|
||||
.iter()
|
||||
.any(|record| record.fault == Some(FaultAction::TruncateBodyAt(100)))
|
||||
);
|
||||
assert!(
|
||||
records
|
||||
.iter()
|
||||
.any(|record| record.fault == Some(FaultAction::Stall(Duration::from_millis(400))))
|
||||
);
|
||||
source.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fake_source_raised_object_cap_accepts_large_put() -> TestResult {
|
||||
let source = FakeS3Target::start_with_options(FakeS3TargetOptions {
|
||||
max_object_bytes: 96 * 1024 * 1024,
|
||||
})
|
||||
.await?;
|
||||
source.create_bucket(SOURCE_BUCKET);
|
||||
let client = fake_source_client(&source);
|
||||
let len = 64 * 1024 * 1024 + 1;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key("large")
|
||||
.body(ByteStream::from(vec![7u8; len]))
|
||||
.send()
|
||||
.await?;
|
||||
let head = client.head_object().bucket(SOURCE_BUCKET).key("large").send().await?;
|
||||
assert_eq!(head.content_length(), Some(len as i64));
|
||||
let tail = client
|
||||
.get_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key("large")
|
||||
.range("bytes=-1")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(tail.content_range(), Some(format!("bytes {}-{}/{len}", len - 1, len - 1).as_str()));
|
||||
source.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn odm_env_starts_rustfs_and_fake_source() -> TestResult {
|
||||
let env = OdmTestEnv::start().await?;
|
||||
env.source.create_bucket(SOURCE_BUCKET);
|
||||
let local_bucket = "odm-local";
|
||||
env.rustfs.create_test_bucket(local_bucket).await?;
|
||||
|
||||
let etags = env.seed_source(
|
||||
SOURCE_BUCKET,
|
||||
&[
|
||||
SeedObject::new("seed/a", Bytes::from_static(b"alpha")),
|
||||
SeedObject::new("seed/b", Bytes::from_static(b"beta"))
|
||||
.with_metadata(SeedMetadata::new().content_type("text/plain").user_metadata("Kind", "seed")),
|
||||
],
|
||||
);
|
||||
assert_eq!(etags.len(), 2);
|
||||
assert!(env.source.requests().is_empty(), "seeding must not touch the journal");
|
||||
let source_client = env.source_client();
|
||||
let seeded = source_client.head_object().bucket(SOURCE_BUCKET).key("seed/b").send().await?;
|
||||
assert_eq!(seeded.content_type(), Some("text/plain"));
|
||||
assert_eq!(seeded.e_tag(), Some(format!("\"{}\"", etags[1]).as_str()));
|
||||
assert_eq!(env.source.count_requests(Operation::HeadObject, "seed/b"), 1);
|
||||
|
||||
env.assert_local_absent(local_bucket, "seed/a").await;
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(local_bucket)
|
||||
.key("seed/a")
|
||||
.body(ByteStream::from_static(b"alpha"))
|
||||
.send()
|
||||
.await?;
|
||||
env.assert_local_present(local_bucket, "seed/a", b"alpha").await;
|
||||
env.assert_local_absent(local_bucket, "seed/b").await;
|
||||
|
||||
let spec = env.fake_source_spec(SOURCE_BUCKET).to_json();
|
||||
assert_eq!(spec["version"], 1);
|
||||
assert_eq!(spec["enabled"], true);
|
||||
assert_eq!(spec["source"]["provider"], "s3");
|
||||
assert_eq!(spec["source"]["endpoint"], env.source.endpoint());
|
||||
assert_eq!(spec["source"]["bucket"], SOURCE_BUCKET);
|
||||
assert_eq!(spec["source"]["credentials"]["secret_key"], "fake-secret");
|
||||
assert_eq!(spec["policy"]["source_timeout"]["first_byte_ms"], 15_000);
|
||||
assert!(spec["policy"]["bandwidth_limit_bytes_per_sec"].is_null());
|
||||
let debug = format!("{:?}", env.fake_source_spec(SOURCE_BUCKET));
|
||||
assert!(!debug.contains("fake-secret"), "Debug output must redact the secret");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn start_source_rustfs_round_trips_put_get() -> TestResult {
|
||||
let env = OdmTestEnv::start().await?;
|
||||
let source = start_source_rustfs().await?;
|
||||
assert_ne!(source.url, env.rustfs.url, "the source must be a separate instance");
|
||||
|
||||
source.create_test_bucket(SOURCE_BUCKET).await?;
|
||||
let source_client = source.create_s3_client();
|
||||
let body = payload(70_000);
|
||||
let put = source_client
|
||||
.put_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key("real/object")
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.content_type("application/octet-stream")
|
||||
.send()
|
||||
.await?;
|
||||
assert!(put.e_tag().is_some());
|
||||
let get = source_client
|
||||
.get_object()
|
||||
.bucket(SOURCE_BUCKET)
|
||||
.key("real/object")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(get.content_type(), Some("application/octet-stream"));
|
||||
assert_eq!(get.body.collect().await?.into_bytes(), body);
|
||||
|
||||
let visible_to_primary = env
|
||||
.client
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await?
|
||||
.buckets()
|
||||
.iter()
|
||||
.any(|bucket| bucket.name() == Some(SOURCE_BUCKET));
|
||||
assert!(!visible_to_primary, "the two servers must not share state");
|
||||
let spec = super::common::OdmSourceSpec::for_rustfs_source(&source, SOURCE_BUCKET).to_json();
|
||||
assert_eq!(spec["source"]["provider"], "rustfs");
|
||||
assert_eq!(spec["source"]["endpoint"], source.url);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! On-demand migration (ODM) end-to-end suite (rustfs/backlog#2147).
|
||||
//!
|
||||
//! `common` is the shared environment: one RustFS under test, one programmable
|
||||
//! fake S3 source, admin-API wrappers, seeding and local-state assertions.
|
||||
//! `harness_self_test` proves the harness itself; ODM behavior scenarios are
|
||||
//! separate modules wired by later tasks.
|
||||
|
||||
pub mod common;
|
||||
|
||||
mod harness_self_test;
|
||||
@@ -40,10 +40,10 @@ mod tests {
|
||||
|
||||
const ENABLE_ENV: &str = "RUSTFS_PRIVILEGED_REPLACEMENT_E2E";
|
||||
const NAMESPACE_ENV: &str = "RUSTFS_PRIVILEGED_REPLACEMENT_E2E_IN_NAMESPACE";
|
||||
const LOG_DIR_ENV: &str = "RUSTFS_PRIVILEGED_REPLACEMENT_LOG_DIR";
|
||||
const TARGET_NODE: usize = 1;
|
||||
const TARGET_DRIVE: usize = 0;
|
||||
const MOUNT_SIZE: &str = "size=128m,mode=0700";
|
||||
const ABSENT_SCANNER_OBSERVATION_TIMEOUT_SECS: u64 = 180;
|
||||
const REPLACEMENT_RECOVERY_DIR: &str = ".rustfs.sys/buckets/ahm-replacement";
|
||||
const REPLACEMENT_INTENT_SUFFIX: &str = "_ahm_replacement_intent.json";
|
||||
const REPLACEMENT_COMPLETION_PROOF_SUFFIX: &str = "_ahm_replacement_completion_proof.json";
|
||||
@@ -142,23 +142,6 @@ mod tests {
|
||||
run_command("dmsetup", &["resume", &self.dm_name])
|
||||
}
|
||||
|
||||
fn verify_raw_io_is_unavailable(&self) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let mapper = format!("/dev/mapper/{}", self.dm_name);
|
||||
let output = Command::new("dd")
|
||||
.env("LC_ALL", "C")
|
||||
.arg(format!("if={mapper}"))
|
||||
.args(["of=/dev/null", "bs=4096", "count=1", "iflag=direct", "status=none"])
|
||||
.output()?;
|
||||
if output.status.success() {
|
||||
return Err(format!("dm-error target unexpectedly allowed a raw read from {mapper}").into());
|
||||
}
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
if !stderr.contains("Input/output error") {
|
||||
return Err(format!("raw read from dm-error target failed unexpectedly: {stderr}").into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn restore_available(&self) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let sectors = run_command_stdout("blockdev", &["--getsz", &self.loop_device])?;
|
||||
let linear_table = format!("0 {sectors} linear {} 0", self.loop_device);
|
||||
@@ -211,72 +194,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
struct ZramBlockMount {
|
||||
target: PathBuf,
|
||||
device: String,
|
||||
mounted: bool,
|
||||
}
|
||||
|
||||
impl ZramBlockMount {
|
||||
fn reserve(target: &Path) -> Result<Self, Box<dyn Error + Send + Sync>> {
|
||||
if !Path::new("/dev/zram-control").exists() {
|
||||
run_command("modprobe", &["zram"])?;
|
||||
}
|
||||
let device = run_command_stdout("zramctl", &["--find", "--size", "256M"])?;
|
||||
if device.is_empty() {
|
||||
return Err("zramctl --find --size returned an empty device".into());
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
target: target.to_path_buf(),
|
||||
device,
|
||||
mounted: false,
|
||||
})
|
||||
}
|
||||
|
||||
fn mount_target(&mut self) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let result = (|| {
|
||||
run_command("mkfs.ext4", &["-F", &self.device])?;
|
||||
let target_arg = path_to_string(&self.target, "zram replacement mount target")?;
|
||||
run_command("mount", &[&self.device, &target_arg])
|
||||
})();
|
||||
if let Err(error) = result {
|
||||
let _ = self.cleanup();
|
||||
return Err(error);
|
||||
}
|
||||
self.mounted = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cleanup(&mut self) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let mut first_error: Option<Box<dyn Error + Send + Sync>> = None;
|
||||
if self.mounted {
|
||||
if let Err(error) = detach_mount(&self.target) {
|
||||
first_error.get_or_insert(error);
|
||||
} else {
|
||||
self.mounted = false;
|
||||
}
|
||||
}
|
||||
if !self.device.is_empty() {
|
||||
if let Err(error) = run_command("zramctl", &["--reset", &self.device]) {
|
||||
first_error.get_or_insert(error);
|
||||
} else {
|
||||
self.device.clear();
|
||||
}
|
||||
}
|
||||
if let Some(error) = first_error {
|
||||
return Err(error);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ZramBlockMount {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
fn checked_command_output(program: &str, args: &[&str]) -> Result<std::process::Output, Box<dyn Error + Send + Sync>> {
|
||||
let output = Command::new(program).args(args).output()?;
|
||||
if output.status.success() {
|
||||
@@ -381,18 +298,6 @@ mod tests {
|
||||
Err(format!("{ENABLE_ENV}=1 requires root or CAP_SYS_ADMIN; unshare exited with status {status}").into())
|
||||
}
|
||||
|
||||
fn replacement_node_log_path(
|
||||
cluster_temp_dir: &str,
|
||||
parity: usize,
|
||||
node_index: usize,
|
||||
) -> Result<PathBuf, Box<dyn Error + Send + Sync>> {
|
||||
let log_dir = std::env::var_os(LOG_DIR_ENV)
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from(cluster_temp_dir));
|
||||
fs::create_dir_all(&log_dir)?;
|
||||
Ok(log_dir.join(format!("replacement-ec{parity}-node{node_index}-{}.log", std::process::id())))
|
||||
}
|
||||
|
||||
fn payload(len: usize, seed: u8) -> Vec<u8> {
|
||||
let mut next = seed;
|
||||
(0..len)
|
||||
@@ -567,20 +472,8 @@ mod tests {
|
||||
if let Some(version_id) = &version.version_id {
|
||||
request = request.version_id(version_id);
|
||||
}
|
||||
let response = request.send().await.map_err(|error| {
|
||||
format!("body GET failed for {}/{}@{:?}: {error}", version.bucket, version.key, version.version_id)
|
||||
})?;
|
||||
let body = response
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.map_err(|error| {
|
||||
format!(
|
||||
"body stream failed for {}/{}@{:?}: {error}",
|
||||
version.bucket, version.key, version.version_id
|
||||
)
|
||||
})?
|
||||
.into_bytes();
|
||||
let response = request.send().await?;
|
||||
let body = response.body.collect().await?.into_bytes();
|
||||
assert_eq!(
|
||||
sha256_hex(&body),
|
||||
*expected_sha256,
|
||||
@@ -689,6 +582,81 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn log_tail(log: &str) -> String {
|
||||
let mut lines = log.lines().rev().take(80).collect::<Vec<_>>();
|
||||
lines.reverse();
|
||||
lines.join("\n")
|
||||
}
|
||||
|
||||
fn log_len(path: &Path) -> Result<u64, Box<dyn Error + Send + Sync>> {
|
||||
match fs::metadata(path) {
|
||||
Ok(metadata) => Ok(metadata.len()),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(0),
|
||||
Err(error) => Err(format!("failed to stat target node log {path:?}: {error}").into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn log_from_offset(path: &Path, offset: u64) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let log = match fs::read(path) {
|
||||
Ok(log) => log,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(),
|
||||
Err(error) => return Err(format!("failed to read target node log {path:?}: {error}").into()),
|
||||
};
|
||||
let start = usize::try_from(offset).unwrap_or(usize::MAX).min(log.len());
|
||||
Ok(String::from_utf8_lossy(&log[start..]).into_owned())
|
||||
}
|
||||
|
||||
fn live_disk_loss_scan_completed(log: &str, target_disk: &Path) -> bool {
|
||||
let target = target_disk.to_string_lossy();
|
||||
let mut saw_live_loss = false;
|
||||
for line in log.lines() {
|
||||
if line.contains("Heal auto-scan disk inspection failed")
|
||||
&& line.contains("check_failed")
|
||||
&& line.contains(target.as_ref())
|
||||
{
|
||||
saw_live_loss = true;
|
||||
continue;
|
||||
}
|
||||
if saw_live_loss && (line.contains("Heal auto disk scanner idle") || line.contains("Heal auto-scan cycle completed"))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn live_disk_loss_scan_completed_from_path(
|
||||
log_path: &Path,
|
||||
start_offset: u64,
|
||||
target_disk: &Path,
|
||||
) -> Result<bool, Box<dyn Error + Send + Sync>> {
|
||||
Ok(live_disk_loss_scan_completed(&log_from_offset(log_path, start_offset)?, target_disk))
|
||||
}
|
||||
|
||||
async fn wait_for_live_disk_loss_observation(
|
||||
log_path: &Path,
|
||||
target_disk: &Path,
|
||||
start_offset: u64,
|
||||
timeout_secs: u64,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let deadline = Instant::now() + Duration::from_secs(timeout_secs);
|
||||
let mut tick = interval(Duration::from_secs(1));
|
||||
loop {
|
||||
if live_disk_loss_scan_completed_from_path(log_path, start_offset, target_disk)? {
|
||||
return Ok(());
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
let log = log_from_offset(log_path, start_offset)?;
|
||||
return Err(format!(
|
||||
"scanner did not finish a live target-loss scan for {target_disk:?} within {timeout_secs}s; log tail:\n{}",
|
||||
log_tail(&log)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
tick.tick().await;
|
||||
}
|
||||
}
|
||||
|
||||
fn cluster_status_is_definitive(status: &serde_json::Value) -> Result<bool, Box<dyn Error + Send + Sync>> {
|
||||
status["cluster"]["definitive"]
|
||||
.as_bool()
|
||||
@@ -739,13 +707,6 @@ mod tests {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn is_transient_recovery_version_absence(error: &(dyn Error + 'static)) -> bool {
|
||||
matches!(
|
||||
error.downcast_ref::<rustfs_filemeta::Error>(),
|
||||
Some(rustfs_filemeta::Error::FileVersionNotFound)
|
||||
)
|
||||
}
|
||||
|
||||
fn incomplete_versions(
|
||||
target_disk: &Path,
|
||||
versions: &[BaselineVersion],
|
||||
@@ -753,21 +714,7 @@ mod tests {
|
||||
let mut missing = BTreeSet::new();
|
||||
for version in versions {
|
||||
let actual =
|
||||
match census_object_version_on_disk(target_disk, &version.bucket, &version.key, version.version_id.as_deref()) {
|
||||
Ok(actual) => actual,
|
||||
// During replacement recovery, xl.meta may arrive before this
|
||||
// particular historical version. The generic census helper
|
||||
// correctly reports that as an error; this progress poll must
|
||||
// instead wait for the version to be restored.
|
||||
Err(error) if is_transient_recovery_version_absence(error.as_ref()) => {
|
||||
missing.insert(format!(
|
||||
"{}/{}@{:?}: version metadata not yet present on replacement",
|
||||
version.bucket, version.key, version.version_id
|
||||
));
|
||||
continue;
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
census_object_version_on_disk(target_disk, &version.bucket, &version.key, version.version_id.as_deref())?;
|
||||
if !actual.matches_manifest(&version.expected) {
|
||||
missing.insert(format!("{}/{}@{:?}: {actual:?}", version.bucket, version.key, version.version_id));
|
||||
}
|
||||
@@ -875,15 +822,13 @@ mod tests {
|
||||
|
||||
let mut mount_ns = MountNamespaceGuard::new()?;
|
||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(3, 4)).await?;
|
||||
for node_index in 0..cluster.nodes.len() {
|
||||
let node_log_path = replacement_node_log_path(&cluster.temp_dir, parity, node_index)?;
|
||||
cluster.set_node_capture_log_path(node_index, node_log_path.to_string_lossy())?;
|
||||
}
|
||||
let target_log_path = PathBuf::from(&cluster.temp_dir).join(format!("replacement-node{TARGET_NODE}.log"));
|
||||
cluster.set_node_capture_log_path(TARGET_NODE, target_log_path.to_string_lossy())?;
|
||||
let target_disk = PathBuf::from(&cluster.nodes[TARGET_NODE].data_dirs[TARGET_DRIVE]);
|
||||
// The blank target uses a temporary zram block device, so the
|
||||
// replacement readiness fence sees no root or sibling alias.
|
||||
// Each drive below is an independent tmpfs mount, so this privileged
|
||||
// path must exercise the production distinct-device/readiness fences.
|
||||
cluster.extra_env.retain(|(key, _)| key != "RUSTFS_UNSAFE_BYPASS_DISK_CHECK");
|
||||
let image_root = PathBuf::from(&cluster.temp_dir).join("replacement-block-images");
|
||||
let image_root = PathBuf::from(&cluster.temp_dir).join("replacement-faultable-images");
|
||||
let mut target_mount = None;
|
||||
for (node_index, node) in cluster.nodes.iter().enumerate() {
|
||||
for (drive_index, drive) in node.data_dirs.iter().enumerate() {
|
||||
@@ -900,7 +845,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
let mut target_mount = target_mount.ok_or("target drive was not mounted with the faultable block fixture")?;
|
||||
let mut replacement_mount = ZramBlockMount::reserve(&target_disk)?;
|
||||
|
||||
cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_SCANNER_ENABLED", "true");
|
||||
@@ -908,34 +852,28 @@ mod tests {
|
||||
cluster.set_env("RUSTFS_SCANNER_CYCLE", "1");
|
||||
cluster.set_env("RUSTFS_SCANNER_START_DELAY_SECS", "0");
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{parity}"));
|
||||
for node_index in 0..cluster.nodes.len() {
|
||||
cluster.set_node_env(node_index, "RUST_LOG", "rustfs=info,rustfs::heal::manager=debug,rustfs_notify=debug")?;
|
||||
}
|
||||
cluster.set_node_env(TARGET_NODE, "RUST_LOG", "rustfs=info,rustfs::heal::manager=debug,rustfs_notify=debug")?;
|
||||
cluster.start().await?;
|
||||
|
||||
let clients = cluster.create_all_clients()?;
|
||||
let versions = seed_baseline(&clients[0], &target_disk)
|
||||
.await
|
||||
.map_err(|error| format!("pre-fault baseline seeding failed: {error}"))?;
|
||||
verify_bodies(&clients[0], &versions)
|
||||
.await
|
||||
.map_err(|error| format!("pre-fault body verification failed: {error}"))?;
|
||||
let versions = seed_baseline(&clients[0], &target_disk).await?;
|
||||
verify_bodies(&clients[0], &versions).await?;
|
||||
|
||||
target_mount
|
||||
.make_unavailable()
|
||||
.map_err(|error| format!("failed to install the dm-error target: {error}"))?;
|
||||
target_mount
|
||||
.verify_raw_io_is_unavailable()
|
||||
.map_err(|error| format!("dm-error target was not proven by a direct raw read: {error}"))?;
|
||||
assert_no_replacement_status_records(&cluster, &target_disk)
|
||||
.await
|
||||
.map_err(|error| format!("live-fault replacement status check failed: {error}"))?;
|
||||
assert_no_replacement_admission_artifacts(&cluster, &target_disk)
|
||||
.map_err(|error| format!("live-fault replacement artifact check failed: {error}"))?;
|
||||
let live_loss_log_offset = log_len(&target_log_path)?;
|
||||
target_mount.make_unavailable()?;
|
||||
wait_for_live_disk_loss_observation(
|
||||
&target_log_path,
|
||||
&target_disk,
|
||||
live_loss_log_offset,
|
||||
ABSENT_SCANNER_OBSERVATION_TIMEOUT_SECS,
|
||||
)
|
||||
.await?;
|
||||
assert_no_replacement_status_records(&cluster, &target_disk).await?;
|
||||
assert_no_replacement_admission_artifacts(&cluster, &target_disk)?;
|
||||
|
||||
cluster.stop_node_gracefully(TARGET_NODE).await?;
|
||||
cluster.stop_node(TARGET_NODE)?;
|
||||
target_mount.cleanup()?;
|
||||
replacement_mount.mount_target()?;
|
||||
mount_ns.mount_tmpfs(&target_disk, &format!("rustfs-e2e-p{parity}-replacement"))?;
|
||||
let missing_before_restart = incomplete_versions(&target_disk, &versions)?;
|
||||
assert_eq!(
|
||||
missing_before_restart.len(),
|
||||
@@ -944,26 +882,46 @@ mod tests {
|
||||
);
|
||||
cluster.start_node(TARGET_NODE).await?;
|
||||
|
||||
let recovery_result = async {
|
||||
wait_for_completed_replacement_with_census(&cluster, &target_disk, &versions, 420).await?;
|
||||
verify_bodies(&clients[0], &versions).await
|
||||
}
|
||||
.await;
|
||||
let stop_result = cluster.stop_node_gracefully(TARGET_NODE).await;
|
||||
let replacement_cleanup_result = replacement_mount.cleanup();
|
||||
wait_for_completed_replacement_with_census(&cluster, &target_disk, &versions, 420).await?;
|
||||
verify_bodies(&clients[0], &versions).await?;
|
||||
|
||||
if let Err(error) = recovery_result {
|
||||
if let Err(stop_error) = stop_result {
|
||||
info!(%stop_error, "replacement target stop failed while preserving recovery failure");
|
||||
}
|
||||
if let Err(cleanup_error) = replacement_cleanup_result {
|
||||
info!(%cleanup_error, "replacement zram cleanup failed while preserving recovery failure");
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
stop_result?;
|
||||
replacement_cleanup_result?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_loss_barrier_requires_scanner_failure_after_log_offset() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let target = Path::new("/mnt/target");
|
||||
assert!(live_disk_loss_scan_completed(
|
||||
"Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto-scan cycle completed",
|
||||
target
|
||||
));
|
||||
assert!(live_disk_loss_scan_completed(
|
||||
"Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle",
|
||||
target
|
||||
));
|
||||
assert!(!live_disk_loss_scan_completed(
|
||||
"Heal auto disk scanner idle\nHeal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed",
|
||||
target
|
||||
));
|
||||
assert!(!live_disk_loss_scan_completed(
|
||||
"event=disk_health_check_failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle",
|
||||
target
|
||||
));
|
||||
assert!(!live_disk_loss_scan_completed(
|
||||
"Heal auto-scan disk inspection failed endpoint=/mnt/other disk_state=check_failed\nHeal auto disk scanner idle",
|
||||
target
|
||||
));
|
||||
let path = std::env::temp_dir().join(format!("rustfs-replacement-scan-{}.log", std::process::id()));
|
||||
let stale =
|
||||
"Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle\n";
|
||||
fs::write(&path, stale)?;
|
||||
let offset = log_len(&path)?;
|
||||
assert!(!live_disk_loss_scan_completed_from_path(&path, offset, target)?);
|
||||
let fresh =
|
||||
"Heal auto-scan disk inspection failed endpoint=/mnt/target disk_state=check_failed\nHeal auto disk scanner idle\n";
|
||||
fs::write(&path, format!("{stale}{fresh}"))?;
|
||||
assert!(live_disk_loss_scan_completed_from_path(&path, offset, target)?);
|
||||
fs::remove_file(path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -996,15 +954,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_census_only_treats_missing_version_as_transient() {
|
||||
let missing_version: Box<dyn Error + Send + Sync> = Box::new(rustfs_filemeta::Error::FileVersionNotFound);
|
||||
let missing_file: Box<dyn Error + Send + Sync> = Box::new(rustfs_filemeta::Error::FileNotFound);
|
||||
|
||||
assert!(is_transient_recovery_version_absence(missing_version.as_ref()));
|
||||
assert!(!is_transient_recovery_version_absence(missing_file.as_ref()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_poll_samples_census_before_status() {
|
||||
let order = std::rc::Rc::new(std::cell::RefCell::new(Vec::new()));
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{RUSTFS_META_BUCKET, VolumeInfo, WalkDirOptions};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{VolumeInfo, WalkDirOptions};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
|
||||
@@ -128,6 +128,7 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod metadata {
|
||||
pub use crate::bucket::metadata::BUCKET_DURABILITY_CONFIG;
|
||||
pub use crate::bucket::metadata::{
|
||||
BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG,
|
||||
BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_QUOTA_CONFIG_FILE,
|
||||
@@ -136,7 +137,6 @@ pub mod bucket {
|
||||
BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BUCKET_WEBSITE_CONFIG, BucketMetadata, OBJECT_LOCK_CONFIG,
|
||||
load_bucket_metadata, table_catalog_path_hash,
|
||||
};
|
||||
pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
|
||||
}
|
||||
|
||||
pub mod durability {
|
||||
@@ -145,21 +145,6 @@ pub mod bucket {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod on_demand_migration {
|
||||
pub use crate::bucket::on_demand_migration::{
|
||||
ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION,
|
||||
OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy,
|
||||
SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
|
||||
};
|
||||
pub mod source_client {
|
||||
pub use crate::bucket::on_demand_migration::source_client::{
|
||||
SourceClient, SourceClientSpec, SourceError, SourceGet, SourceHead, SourceObject, SourcePage, SourceProbe,
|
||||
SourceProvider, SourceSse, SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value,
|
||||
resolve_path_style,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
pub mod metadata_sys {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
|
||||
@@ -169,11 +154,11 @@ pub mod bucket {
|
||||
delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
|
||||
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, get_public_access_block_config,
|
||||
get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config,
|
||||
get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata,
|
||||
remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock,
|
||||
update_config_with, update_if_incarnation, update_quota_if_incarnation, update_under_transaction_lock,
|
||||
get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config,
|
||||
get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config,
|
||||
get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata,
|
||||
set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
|
||||
update_quota_if_incarnation, update_under_transaction_lock,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -214,13 +199,6 @@ pub mod bucket {
|
||||
}
|
||||
}
|
||||
|
||||
pub mod remote_s3_client {
|
||||
pub use crate::bucket::remote_s3_client::{
|
||||
PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, build_remote_s3_client,
|
||||
validate_remote_endpoint,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod replication {
|
||||
pub use crate::bucket::replication::replication_pool::{
|
||||
DurableMrfBacklogSummary, DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBacklogObservabilitySummary,
|
||||
@@ -462,11 +440,6 @@ pub mod object {
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
SnapshotConsistencyError,
|
||||
};
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
pub use crate::store::DeleteAfterObjectLockSnapshotBarrier;
|
||||
}
|
||||
}
|
||||
|
||||
pub mod rebalance {
|
||||
|
||||
@@ -15,13 +15,17 @@
|
||||
use crate::bucket::metadata::BucketMetadata;
|
||||
use crate::bucket::metadata_sys::get_bucket_targets_config;
|
||||
use crate::bucket::metadata_sys::get_replication_config;
|
||||
use crate::bucket::remote_s3_client::{PathStyle, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client};
|
||||
use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
|
||||
use crate::bucket::target::ARN;
|
||||
use crate::bucket::target::BucketTargetType;
|
||||
use crate::bucket::target::{self, BucketTarget, BucketTargets, Credentials};
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use aws_credential_types::Credentials as SdkCredentials;
|
||||
use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future};
|
||||
use aws_sdk_s3::config::Region as SdkRegion;
|
||||
use aws_sdk_s3::config::RequestChecksumCalculation;
|
||||
use aws_sdk_s3::config::SharedHttpClient;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::operation::complete_multipart_upload::CompleteMultipartUploadOutput;
|
||||
@@ -33,17 +37,28 @@ use aws_sdk_s3::operation::head_object::HeadObjectError;
|
||||
use aws_sdk_s3::operation::put_object_tagging::{PutObjectTaggingError, PutObjectTaggingOutput};
|
||||
use aws_sdk_s3::operation::upload_part::UploadPartOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::BucketVersioningStatus;
|
||||
use aws_sdk_s3::types::Tagging as SdkTagging;
|
||||
use aws_sdk_s3::types::{
|
||||
ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
ServerSideEncryption,
|
||||
};
|
||||
use aws_sdk_s3::{Client as S3Client, operation::head_object::HeadObjectOutput};
|
||||
use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
|
||||
use aws_sdk_s3::{Client as S3Client, Config as S3Config, operation::head_object::HeadObjectOutput};
|
||||
use aws_sdk_s3::{config::SharedCredentialsProvider, types::BucketVersioningStatus};
|
||||
use aws_smithy_http_client::{Builder as SmithyHttpClientBuilder, tls as smithy_tls};
|
||||
use aws_smithy_runtime_api::box_error::BoxError;
|
||||
use aws_smithy_runtime_api::client::http::{
|
||||
HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
|
||||
};
|
||||
use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
|
||||
use aws_smithy_runtime_api::client::result::ConnectorError;
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use futures::{StreamExt, stream};
|
||||
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
|
||||
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode, Uri};
|
||||
use hyper_util::client::legacy::Client as HyperClient;
|
||||
use hyper_util::rt::{TokioExecutor, TokioTimer};
|
||||
use reqwest::Client as HttpClient;
|
||||
use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RUSTFS_CA_CERT, RUSTFS_TLS_CERT};
|
||||
use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url};
|
||||
use rustfs_utils::http::{
|
||||
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
|
||||
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING_LOWER, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header,
|
||||
@@ -55,10 +70,12 @@ use rustfs_utils::http::{
|
||||
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID,
|
||||
insert_header,
|
||||
};
|
||||
use rustls_pki_types::pem::PemObject;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error;
|
||||
use std::fmt;
|
||||
use std::path::Path;
|
||||
use std::str::FromStr as _;
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
@@ -67,6 +84,7 @@ use std::time::{Duration, Instant, SystemTime};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::sync::RwLock;
|
||||
use tower::Service;
|
||||
use tracing::error;
|
||||
use tracing::warn;
|
||||
use url::Url;
|
||||
@@ -74,50 +92,72 @@ use uuid::Uuid;
|
||||
|
||||
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
|
||||
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||
const EXPIRED_REMOTE_TARGET_CREDENTIALS: &str = "remote target credentials have expired";
|
||||
|
||||
fn remote_credentials(credentials: &Credentials, account_id: &str) -> RemoteCredentials {
|
||||
RemoteCredentials {
|
||||
access_key: credentials.access_key.clone(),
|
||||
secret_key: credentials.secret_key.clone(),
|
||||
session_token: credentials.effective_session_token().map(str::to_string),
|
||||
expiration: credentials.effective_expiration().map(SystemTime::from),
|
||||
account_id: account_id.to_string(),
|
||||
}
|
||||
#[derive(Clone)]
|
||||
struct RemoteTargetCredentialsProvider {
|
||||
credentials: SdkCredentials,
|
||||
}
|
||||
|
||||
fn target_path_style(path: &str) -> PathStyle {
|
||||
match path.trim().to_ascii_lowercase().as_str() {
|
||||
// Explicit DNS/virtual-hosted-style requested by user.
|
||||
"dns" | "off" | "false" => PathStyle::VirtualHost,
|
||||
// Explicit path-style or legacy boolean-like values.
|
||||
"path" | "on" | "true" => PathStyle::Path,
|
||||
// `auto` and empty are defaulted to path-style for custom S3-compatible endpoints.
|
||||
"auto" | "" => PathStyle::Auto,
|
||||
// Unknown values: prefer compatibility with S3-compatible services.
|
||||
_ => PathStyle::Path,
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&BucketTarget> for RemoteS3EndpointSpec {
|
||||
fn from(target: &BucketTarget) -> Self {
|
||||
RemoteS3EndpointSpec {
|
||||
endpoint: target.endpoint.clone(),
|
||||
secure: target.secure,
|
||||
region: target.region.clone(),
|
||||
path_style: target_path_style(&target.path),
|
||||
credentials: target
|
||||
.credentials
|
||||
.as_ref()
|
||||
.map(|credentials| remote_credentials(credentials, &target.reset_id)),
|
||||
skip_tls_verify: target.skip_tls_verify,
|
||||
ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
|
||||
connect_timeout: None,
|
||||
read_timeout: None,
|
||||
user_agent_suffix: "",
|
||||
impl RemoteTargetCredentialsProvider {
|
||||
fn resolve_at(&self, now: SystemTime) -> aws_credential_types::provider::Result {
|
||||
if self.credentials.expiry().is_some_and(|expiration| expiration <= now) {
|
||||
return Err(CredentialsError::provider_error(std::io::Error::other(EXPIRED_REMOTE_TARGET_CREDENTIALS)));
|
||||
}
|
||||
Ok(self.credentials.clone())
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for RemoteTargetCredentialsProvider {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("RemoteTargetCredentialsProvider")
|
||||
.field("temporary", &self.credentials.session_token().is_some())
|
||||
.field("expiration", &self.credentials.expiry())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProvideCredentials for RemoteTargetCredentialsProvider {
|
||||
fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a>
|
||||
where
|
||||
Self: 'a,
|
||||
{
|
||||
future::ProvideCredentials::ready(self.resolve_at(SystemTime::now()))
|
||||
}
|
||||
|
||||
fn fallback_on_interrupt(&self) -> Option<SdkCredentials> {
|
||||
self.resolve_at(SystemTime::now()).ok()
|
||||
}
|
||||
}
|
||||
|
||||
fn remote_target_sdk_credentials(
|
||||
credentials: &Credentials,
|
||||
account_id: &str,
|
||||
now: SystemTime,
|
||||
) -> Result<SdkCredentials, &'static str> {
|
||||
let session_token = credentials.effective_session_token();
|
||||
let expiration = credentials.effective_expiration().map(SystemTime::from);
|
||||
if expiration.is_some() && session_token.is_none() {
|
||||
return Err("remote target credential expiration requires a session token");
|
||||
}
|
||||
if expiration.is_some_and(|expiration| expiration <= now) {
|
||||
return Err(EXPIRED_REMOTE_TARGET_CREDENTIALS);
|
||||
}
|
||||
|
||||
let mut builder = SdkCredentials::builder()
|
||||
.access_key_id(credentials.access_key.clone())
|
||||
.secret_access_key(credentials.secret_key.clone())
|
||||
.account_id(account_id.to_string())
|
||||
.provider_name("bucket_target_sys");
|
||||
if let Some(session_token) = session_token {
|
||||
builder = builder.session_token(session_token.to_string());
|
||||
}
|
||||
if let Some(expiration) = expiration {
|
||||
builder = builder.expiry(expiration);
|
||||
}
|
||||
Ok(builder.build())
|
||||
}
|
||||
|
||||
pub type HeadObjectSdkError = Box<SdkError<HeadObjectError>>;
|
||||
pub type GetObjectSdkError = Box<SdkError<GetObjectError>>;
|
||||
pub type GetObjectTaggingSdkError = Box<SdkError<GetObjectTaggingError>>;
|
||||
@@ -445,19 +485,6 @@ impl BucketTargetSys {
|
||||
mutex
|
||||
}
|
||||
|
||||
/// Snapshot the heartbeat-tracked health of `url`'s endpoint.
|
||||
///
|
||||
/// Returns `None` when the heartbeat has never seen the endpoint. Unlike
|
||||
/// [`Self::is_offline`] this deliberately does not call `init_hc`: a caller
|
||||
/// that only reports metrics must not create health entries as a side
|
||||
/// effect, or merely rendering a status page would mark an unknown peer
|
||||
/// online.
|
||||
pub async fn endpoint_health(&self, url: &Url) -> Option<EpHealth> {
|
||||
let key = endpoint_health_key(url);
|
||||
let health_map = self.h_mutex.read().await;
|
||||
health_map.get(&key).cloned()
|
||||
}
|
||||
|
||||
pub async fn is_offline(&self, url: &Url) -> bool {
|
||||
let key = endpoint_health_key(url);
|
||||
{
|
||||
@@ -1018,17 +1045,57 @@ impl BucketTargetSys {
|
||||
});
|
||||
};
|
||||
|
||||
let spec = RemoteS3EndpointSpec::from(target);
|
||||
let client = build_remote_s3_client(&spec)
|
||||
.await
|
||||
.map_err(|err| BucketTargetError::RemoteTargetConnectionErr {
|
||||
let creds = remote_target_sdk_credentials(credentials, &target.reset_id, SystemTime::now()).map_err(|error| {
|
||||
BucketTargetError::RemoteTargetConnectionErr {
|
||||
bucket: target.target_bucket.clone(),
|
||||
access_key: credentials.access_key.clone(),
|
||||
error: err.to_string(),
|
||||
})?;
|
||||
error: error.to_string(),
|
||||
}
|
||||
})?;
|
||||
|
||||
let endpoint = if target.secure {
|
||||
format!("https://{}", target.endpoint)
|
||||
} else {
|
||||
format!("http://{}", target.endpoint)
|
||||
};
|
||||
let parsed_endpoint = Url::parse(&endpoint).map_err(|err| BucketTargetError::RemoteTargetConnectionErr {
|
||||
bucket: target.target_bucket.clone(),
|
||||
access_key: credentials.access_key.clone(),
|
||||
error: format!("invalid target endpoint: {err}"),
|
||||
})?;
|
||||
validate_replication_target_endpoint(&parsed_endpoint).map_err(|err| BucketTargetError::RemoteTargetConnectionErr {
|
||||
bucket: target.target_bucket.clone(),
|
||||
access_key: credentials.access_key.clone(),
|
||||
error: format!("target endpoint is not allowed: {err}"),
|
||||
})?;
|
||||
|
||||
let mut config_builder = S3Config::builder()
|
||||
.endpoint_url(endpoint.clone())
|
||||
.credentials_provider(SharedCredentialsProvider::new(RemoteTargetCredentialsProvider { credentials: creds }))
|
||||
.region(SdkRegion::new(target.region.clone()))
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.request_checksum_calculation(replication_request_checksum_calculation());
|
||||
|
||||
if should_force_path_style(target) {
|
||||
config_builder = config_builder.force_path_style(true);
|
||||
}
|
||||
|
||||
if let Some(http_client) =
|
||||
build_aws_s3_http_client_for_target(target)
|
||||
.await
|
||||
.map_err(|err| BucketTargetError::RemoteTargetConnectionErr {
|
||||
bucket: target.target_bucket.clone(),
|
||||
access_key: credentials.access_key.clone(),
|
||||
error: err.to_string(),
|
||||
})?
|
||||
{
|
||||
config_builder = config_builder.http_client(http_client);
|
||||
}
|
||||
|
||||
let config = config_builder.build();
|
||||
|
||||
Ok(TargetClient {
|
||||
endpoint: spec.endpoint_url(),
|
||||
endpoint,
|
||||
credentials: target.credentials.clone(),
|
||||
bucket: target.target_bucket.clone(),
|
||||
storage_class: target.storage_class.clone(),
|
||||
@@ -1038,7 +1105,7 @@ impl BucketTargetSys {
|
||||
secure: target.secure,
|
||||
health_check_duration: target.health_check_duration,
|
||||
replicate_sync: target.replication_sync,
|
||||
client: Arc::new(client),
|
||||
client: Arc::new(S3Client::from_conf(config)),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1201,6 +1268,327 @@ impl BucketTargetSys {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct AcceptAnyServerCertVerifier;
|
||||
|
||||
impl rustls::client::danger::ServerCertVerifier for AcceptAnyServerCertVerifier {
|
||||
fn verify_server_cert(
|
||||
&self,
|
||||
_end_entity: &rustls_pki_types::CertificateDer<'_>,
|
||||
_intermediates: &[rustls_pki_types::CertificateDer<'_>],
|
||||
_server_name: &rustls_pki_types::ServerName<'_>,
|
||||
_ocsp_response: &[u8],
|
||||
_now: rustls_pki_types::UnixTime,
|
||||
) -> Result<rustls::client::danger::ServerCertVerified, rustls::Error> {
|
||||
Ok(rustls::client::danger::ServerCertVerified::assertion())
|
||||
}
|
||||
|
||||
fn verify_tls12_signature(
|
||||
&self,
|
||||
_message: &[u8],
|
||||
_cert: &rustls_pki_types::CertificateDer<'_>,
|
||||
_dss: &rustls::DigitallySignedStruct,
|
||||
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
|
||||
Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
|
||||
}
|
||||
|
||||
fn verify_tls13_signature(
|
||||
&self,
|
||||
_message: &[u8],
|
||||
_cert: &rustls_pki_types::CertificateDer<'_>,
|
||||
_dss: &rustls::DigitallySignedStruct,
|
||||
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
|
||||
Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
|
||||
}
|
||||
|
||||
fn supported_verify_schemes(&self) -> Vec<rustls::SignatureScheme> {
|
||||
rustls::crypto::aws_lc_rs::default_provider()
|
||||
.signature_verification_algorithms
|
||||
.supported_schemes()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TargetHyperHttpConnector<C> {
|
||||
client: HyperClient<C, SdkBody>,
|
||||
}
|
||||
|
||||
impl<C> fmt::Debug for TargetHyperHttpConnector<C> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("TargetHyperHttpConnector")
|
||||
.field("client", &"** hyper client **")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl<C> SmithyHttpConnector for TargetHyperHttpConnector<C>
|
||||
where
|
||||
C: Clone + Send + Sync + 'static,
|
||||
C: Service<Uri>,
|
||||
C::Response:
|
||||
hyper::rt::Read + hyper::rt::Write + hyper_util::client::legacy::connect::Connection + Send + Sync + Unpin + 'static,
|
||||
C::Future: Unpin + Send + 'static,
|
||||
C::Error: Into<BoxError>,
|
||||
{
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
let request = match request.try_into_http1x() {
|
||||
Ok(request) => request,
|
||||
Err(err) => return HttpConnectorFuture::ready(Err(ConnectorError::user(err.into()))),
|
||||
};
|
||||
|
||||
let mut client = self.client.clone();
|
||||
let fut = client.call(request);
|
||||
HttpConnectorFuture::new(async move {
|
||||
let response = fut
|
||||
.await
|
||||
.map_err(|err| ConnectorError::io(err.into()))?
|
||||
.map(SdkBody::from_body_1_x);
|
||||
HttpResponse::try_from(response).map_err(|err| ConnectorError::other(err.into(), None))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_rustls_crypto_provider() {
|
||||
if rustls::crypto::CryptoProvider::get_default().is_none() {
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
}
|
||||
}
|
||||
|
||||
fn has_custom_ca_pem(target: &BucketTarget) -> bool {
|
||||
!target.ca_cert_pem.trim().is_empty()
|
||||
}
|
||||
|
||||
/// Env opt-in that re-enables loopback replication targets. Loopback (`127.0.0.1`,
|
||||
/// `::1`, `localhost`) is a classic SSRF vector and stays rejected by default, but
|
||||
/// single-host multi-instance dev setups and the e2e harness legitimately replicate
|
||||
/// over loopback. Never set this in production.
|
||||
const ALLOW_LOOPBACK_REPLICATION_TARGET_ENV: &str = "RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET";
|
||||
|
||||
fn loopback_replication_targets_allowed() -> bool {
|
||||
std::env::var(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV)
|
||||
.map(|v| v.eq_ignore_ascii_case("true") || v == "1")
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
const REPLICATION_STREAMING_CHECKSUMS_ENV: &str = "RUSTFS_REPLICATION_STREAMING_CHECKSUMS";
|
||||
|
||||
/// Streaming trailer checksums make the SDK frame request bodies as
|
||||
/// `aws-chunked`; a target that does not decode that framing stores the frames
|
||||
/// verbatim, silently corrupting every replica while the transfer itself
|
||||
/// succeeds (#6853). Plain signed payloads are the compatible default; the env
|
||||
/// knob restores trailer checksums for fleets whose targets are all known to
|
||||
/// decode them.
|
||||
fn replication_request_checksum_calculation() -> RequestChecksumCalculation {
|
||||
if std::env::var(REPLICATION_STREAMING_CHECKSUMS_ENV)
|
||||
.map(|v| v.eq_ignore_ascii_case("true") || v == "1")
|
||||
.unwrap_or(false)
|
||||
{
|
||||
RequestChecksumCalculation::WhenSupported
|
||||
} else {
|
||||
RequestChecksumCalculation::WhenRequired
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_replication_target_endpoint(url: &Url) -> Result<(), OutboundUrlError> {
|
||||
validate_replication_target_endpoint_inner(url, loopback_replication_targets_allowed())
|
||||
}
|
||||
|
||||
fn validate_replication_target_endpoint_inner(url: &Url, allow_loopback: bool) -> Result<(), OutboundUrlError> {
|
||||
match validate_outbound_url(url) {
|
||||
Ok(()) => Ok(()),
|
||||
// Replication targets are trusted infrastructure the operator configures, and
|
||||
// legitimately live on private networks, so private addresses are always allowed.
|
||||
Err(OutboundUrlError::ForbiddenHost {
|
||||
reason: "private address",
|
||||
..
|
||||
}) => Ok(()),
|
||||
// Loopback is far higher SSRF risk, so it is allowed only under the explicit,
|
||||
// off-by-default opt-in above (single-host multi-instance / the e2e harness).
|
||||
Err(OutboundUrlError::ForbiddenHost {
|
||||
reason: "loopback address" | "loopback host",
|
||||
..
|
||||
}) if allow_loopback => Ok(()),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn build_insecure_aws_s3_http_client() -> SharedHttpClient {
|
||||
ensure_rustls_crypto_provider();
|
||||
|
||||
let tls_config = rustls::ClientConfig::builder()
|
||||
.dangerous()
|
||||
.with_custom_certificate_verifier(Arc::new(AcceptAnyServerCertVerifier))
|
||||
.with_no_client_auth();
|
||||
|
||||
let https = hyper_rustls::HttpsConnectorBuilder::new()
|
||||
.with_tls_config(tls_config)
|
||||
.https_or_http()
|
||||
.enable_http1()
|
||||
.enable_http2()
|
||||
.build();
|
||||
let mut client_builder = HyperClient::builder(TokioExecutor::new());
|
||||
client_builder.pool_timer(TokioTimer::new());
|
||||
let client = client_builder.build(https);
|
||||
let connector = SharedHttpConnector::new(TargetHyperHttpConnector { client });
|
||||
|
||||
http_client_fn(move |_settings, _components| connector.clone())
|
||||
}
|
||||
|
||||
fn validate_ca_pem_bundle(ca_cert_pem: &[u8]) -> Result<(), String> {
|
||||
let certs = rustls_pki_types::CertificateDer::pem_slice_iter(ca_cert_pem)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|err| format!("invalid PEM encoding: {err}"))?;
|
||||
|
||||
if certs.is_empty() {
|
||||
return Err("no certificates found".to_string());
|
||||
}
|
||||
|
||||
// Smithy's rustls adapter defers parsing custom certificates and assumes
|
||||
// they are valid when the HTTPS connector is built. Validate every DER
|
||||
// certificate first so malformed configuration is reported rather than
|
||||
// reaching an `expect` in the dependency.
|
||||
let mut validation_store = rustls::RootCertStore::empty();
|
||||
for cert in certs {
|
||||
validation_store
|
||||
.add(cert)
|
||||
.map_err(|err| format!("invalid X.509 certificate: {err}"))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), BucketTargetError> {
|
||||
validate_ca_pem_bundle(ca_cert_pem.as_bytes())
|
||||
.map_err(|err| BucketTargetError::Io(std::io::Error::other(format!("invalid target CA PEM: {err}"))))
|
||||
}
|
||||
|
||||
fn compose_replication_trust_store(certificate_bundles: impl IntoIterator<Item = Vec<u8>>) -> (smithy_tls::TrustStore, usize) {
|
||||
// `TrustStore::default()` keeps the platform-native roots enabled. Target
|
||||
// and RUSTFS_TLS_PATH certificates extend that baseline instead of
|
||||
// replacing it with a target-specific trust island.
|
||||
let mut trust_store = smithy_tls::TrustStore::default();
|
||||
let mut custom_bundle_count = 0;
|
||||
for pem in certificate_bundles {
|
||||
trust_store.add_pem_certificate(pem);
|
||||
custom_bundle_count += 1;
|
||||
}
|
||||
|
||||
(trust_store, custom_bundle_count)
|
||||
}
|
||||
|
||||
fn build_aws_s3_http_client_with_trust_store(trust_store: smithy_tls::TrustStore) -> Result<SharedHttpClient, BucketTargetError> {
|
||||
let tls_context = smithy_tls::TlsContext::builder()
|
||||
.with_trust_store(trust_store)
|
||||
.build()
|
||||
.map_err(|err| BucketTargetError::Io(std::io::Error::other(format!("invalid target CA PEM: {err}"))))?;
|
||||
|
||||
Ok(SmithyHttpClientBuilder::new()
|
||||
.tls_provider(smithy_tls::Provider::rustls(smithy_tls::rustls_provider::CryptoMode::AwsLc))
|
||||
.tls_context(tls_context)
|
||||
.build_https())
|
||||
}
|
||||
|
||||
async fn load_tls_path_ca_bundles(tls_dir: &Path, trust_leaf_cert_as_ca: bool) -> Vec<Vec<u8>> {
|
||||
let mut certificate_bundles = Vec::new();
|
||||
|
||||
let ca_path = tls_dir.join(RUSTFS_CA_CERT);
|
||||
match tokio::fs::read(&ca_path).await {
|
||||
Ok(pem) => match validate_ca_pem_bundle(&pem) {
|
||||
Ok(()) => certificate_bundles.push(pem),
|
||||
Err(err) => warn!("ignoring invalid custom CA bundle {:?} for replication client: {}", ca_path, err),
|
||||
},
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(e) => warn!("failed to read custom CA bundle {:?} for replication client: {}", ca_path, e),
|
||||
}
|
||||
|
||||
if trust_leaf_cert_as_ca {
|
||||
let leaf_cert_path = tls_dir.join(RUSTFS_TLS_CERT);
|
||||
match tokio::fs::read(&leaf_cert_path).await {
|
||||
Ok(pem) => match validate_ca_pem_bundle(&pem) {
|
||||
Ok(()) => certificate_bundles.push(pem),
|
||||
Err(err) => warn!(
|
||||
"ignoring invalid leaf certificate {:?} for replication client trust store: {}",
|
||||
leaf_cert_path, err
|
||||
),
|
||||
},
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(e) => warn!("failed to read leaf cert {:?} for replication client trust store: {}", leaf_cert_path, e),
|
||||
}
|
||||
}
|
||||
|
||||
certificate_bundles
|
||||
}
|
||||
|
||||
async fn load_configured_tls_ca_bundles() -> Vec<Vec<u8>> {
|
||||
let tls_path = rustfs_utils::get_env_str(rustfs_config::ENV_RUSTFS_TLS_PATH, rustfs_config::DEFAULT_RUSTFS_TLS_PATH);
|
||||
if tls_path.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
load_tls_path_ca_bundles(
|
||||
Path::new(&tls_path),
|
||||
rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn build_aws_s3_http_client_from_target_ca_pem(ca_cert_pem: &str) -> Result<SharedHttpClient, BucketTargetError> {
|
||||
validate_target_ca_pem(ca_cert_pem)?;
|
||||
|
||||
let mut certificate_bundles = load_configured_tls_ca_bundles().await;
|
||||
certificate_bundles.push(ca_cert_pem.as_bytes().to_vec());
|
||||
let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
|
||||
|
||||
build_aws_s3_http_client_with_trust_store(trust_store)
|
||||
}
|
||||
|
||||
async fn build_aws_s3_http_client_for_target(target: &BucketTarget) -> Result<Option<SharedHttpClient>, BucketTargetError> {
|
||||
if !target.secure {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if target.skip_tls_verify {
|
||||
return Ok(Some(build_insecure_aws_s3_http_client()));
|
||||
}
|
||||
|
||||
if has_custom_ca_pem(target) {
|
||||
return build_aws_s3_http_client_from_target_ca_pem(&target.ca_cert_pem)
|
||||
.await
|
||||
.map(Some);
|
||||
}
|
||||
|
||||
Ok(build_aws_s3_http_client_from_tls_path().await)
|
||||
}
|
||||
|
||||
async fn build_aws_s3_http_client_from_tls_path() -> Option<aws_sdk_s3::config::SharedHttpClient> {
|
||||
let certificate_bundles = load_configured_tls_ca_bundles().await;
|
||||
if certificate_bundles.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
|
||||
match build_aws_s3_http_client_with_trust_store(trust_store) {
|
||||
Ok(client) => Some(client),
|
||||
Err(e) => {
|
||||
warn!("failed to build AWS SDK TLS context for replication client: {}", e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn should_force_path_style(target: &BucketTarget) -> bool {
|
||||
match target.path.trim().to_ascii_lowercase().as_str() {
|
||||
// Explicit DNS/virtual-hosted-style requested by user.
|
||||
"dns" | "off" | "false" => false,
|
||||
// Explicit path-style or legacy boolean-like values.
|
||||
"path" | "on" | "true" => true,
|
||||
// `auto` and empty are defaulted to path-style for custom S3-compatible endpoints.
|
||||
"auto" | "" => true,
|
||||
// Unknown values: prefer compatibility with S3-compatible services.
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
// generate ARN that is unique to this target type
|
||||
fn generate_arn(t: &BucketTarget, depl_id: &str) -> String {
|
||||
let uuid = if depl_id.is_empty() {
|
||||
@@ -2306,24 +2694,7 @@ impl Error for BucketTargetError {}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::remote_s3_client::{
|
||||
EXPIRED_REMOTE_TARGET_CREDENTIALS, RemoteTargetCredentialsProvider, build_aws_s3_http_client_for_spec,
|
||||
build_aws_s3_http_client_from_target_ca_pem, build_aws_s3_http_client_with_trust_store,
|
||||
build_insecure_aws_s3_http_client, compose_replication_trust_store, ensure_rustls_crypto_provider,
|
||||
load_tls_path_ca_bundles, remote_sdk_credentials, replication_request_checksum_calculation,
|
||||
validate_remote_endpoint_inner, validate_target_ca_pem,
|
||||
};
|
||||
use aws_credential_types::Credentials as SdkCredentials;
|
||||
use aws_sdk_s3::Config as S3Config;
|
||||
use aws_sdk_s3::config::{Region as SdkRegion, RequestChecksumCalculation, SharedCredentialsProvider, SharedHttpClient};
|
||||
use aws_smithy_runtime_api::client::http::{
|
||||
HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
|
||||
};
|
||||
use aws_smithy_runtime_api::client::orchestrator::HttpResponse;
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use rcgen::generate_simple_self_signed;
|
||||
use rustfs_config::{RUSTFS_CA_CERT, RUSTFS_TLS_CERT};
|
||||
use rustfs_utils::egress::OutboundUrlError;
|
||||
|
||||
// The startup panic fix for hosts without a CA bundle (issue #6734) rests
|
||||
// on two properties: the health-check client constructor never panics, and
|
||||
@@ -2550,8 +2921,8 @@ mod tests {
|
||||
expiration: Some(jiff::Timestamp::try_from(expiration).expect("test expiration should convert")),
|
||||
};
|
||||
|
||||
let sdk_credentials = remote_sdk_credentials(&remote_credentials(&credentials, "account"), now)
|
||||
.expect("unexpired temporary credentials should build");
|
||||
let sdk_credentials =
|
||||
remote_target_sdk_credentials(&credentials, "account", now).expect("unexpired temporary credentials should build");
|
||||
|
||||
assert_eq!(sdk_credentials.session_token(), Some("temporary-session-token"));
|
||||
assert_eq!(sdk_credentials.expiry(), Some(expiration));
|
||||
@@ -2567,7 +2938,7 @@ mod tests {
|
||||
expiration: Some("0001-01-01T00:00:00Z".parse().expect("Go zero time should parse")),
|
||||
};
|
||||
|
||||
let sdk_credentials = remote_sdk_credentials(&remote_credentials(&credentials, ""), SystemTime::now())
|
||||
let sdk_credentials = remote_target_sdk_credentials(&credentials, "", SystemTime::now())
|
||||
.expect("Go zero expiration should remain compatible with static credentials");
|
||||
|
||||
assert!(sdk_credentials.session_token().is_none());
|
||||
@@ -2585,14 +2956,14 @@ mod tests {
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
remote_sdk_credentials(&remote_credentials(&credentials, ""), SystemTime::UNIX_EPOCH + Duration::from_secs(1_000))
|
||||
remote_target_sdk_credentials(&credentials, "", SystemTime::UNIX_EPOCH + Duration::from_secs(1_000))
|
||||
.expect_err("expiration without a session token must fail"),
|
||||
"remote target credential expiration requires a session token"
|
||||
);
|
||||
|
||||
credentials.session_token = Some("temporary-session-token".to_string());
|
||||
assert_eq!(
|
||||
remote_sdk_credentials(&remote_credentials(&credentials, ""), expiration)
|
||||
remote_target_sdk_credentials(&credentials, "", expiration)
|
||||
.expect_err("credentials expire at the exact expiration boundary"),
|
||||
EXPIRED_REMOTE_TARGET_CREDENTIALS
|
||||
);
|
||||
@@ -2652,7 +3023,7 @@ mod tests {
|
||||
session_token: Some("temporary-session-token".to_string()),
|
||||
expiration: Some("2099-01-01T00:00:00Z".parse().expect("future expiration should parse")),
|
||||
};
|
||||
let sdk_credentials = remote_sdk_credentials(&remote_credentials(&credentials, ""), SystemTime::now())
|
||||
let sdk_credentials = remote_target_sdk_credentials(&credentials, "", SystemTime::now())
|
||||
.expect("unexpired temporary credentials should build");
|
||||
let client = S3Client::from_conf(
|
||||
S3Config::builder()
|
||||
@@ -2827,46 +3198,6 @@ mod tests {
|
||||
assert!(!replication_target_versioning_enabled(None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_endpoint_spec_from_target_keeps_legacy_path_style_and_trust_semantics() {
|
||||
for (path, expected) in [
|
||||
("dns", PathStyle::VirtualHost),
|
||||
("OFF", PathStyle::VirtualHost),
|
||||
("false", PathStyle::VirtualHost),
|
||||
("path", PathStyle::Path),
|
||||
("on", PathStyle::Path),
|
||||
("true", PathStyle::Path),
|
||||
(" auto ", PathStyle::Auto),
|
||||
("", PathStyle::Auto),
|
||||
("something-else", PathStyle::Path),
|
||||
] {
|
||||
assert_eq!(target_path_style(path), expected, "path={path:?}");
|
||||
}
|
||||
|
||||
let spec = RemoteS3EndpointSpec::from(&BucketTarget {
|
||||
endpoint: "192.168.1.10:9000".to_string(),
|
||||
secure: true,
|
||||
region: "us-east-1".to_string(),
|
||||
ca_cert_pem: " ".to_string(),
|
||||
reset_id: "reset-1".to_string(),
|
||||
credentials: Some(Credentials {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
session_token: Some(" ".to_string()),
|
||||
expiration: Some("0001-01-01T00:00:00Z".parse().expect("Go zero time should parse")),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
assert_eq!(spec.endpoint_url(), "https://192.168.1.10:9000");
|
||||
assert!(spec.ca_cert_pem.is_none(), "whitespace-only CA PEM means unset");
|
||||
assert!(spec.connect_timeout.is_none() && spec.read_timeout.is_none());
|
||||
assert_eq!(spec.user_agent_suffix, "");
|
||||
let credentials = spec.credentials.expect("credentials carry over");
|
||||
assert_eq!(credentials.account_id, "reset-1");
|
||||
assert!(credentials.session_token.is_none(), "blank session token is absent");
|
||||
assert!(credentials.expiration.is_none(), "Go zero expiration is absent");
|
||||
}
|
||||
|
||||
fn parse_url(raw: &str) -> Url {
|
||||
Url::parse(raw).expect("test URL should parse")
|
||||
}
|
||||
@@ -2876,16 +3207,16 @@ mod tests {
|
||||
// Public hosts and private-network targets are allowed regardless of the
|
||||
// loopback opt-in — replication commonly runs across trusted private infra.
|
||||
for allow_loopback in [false, true] {
|
||||
assert!(validate_remote_endpoint_inner(&parse_url("https://s3.example.com"), allow_loopback).is_ok());
|
||||
assert!(validate_remote_endpoint_inner(&parse_url("http://10.0.0.5:9000"), allow_loopback).is_ok());
|
||||
assert!(validate_remote_endpoint_inner(&parse_url("http://192.168.1.20"), allow_loopback).is_ok());
|
||||
assert!(validate_replication_target_endpoint_inner(&parse_url("https://s3.example.com"), allow_loopback).is_ok());
|
||||
assert!(validate_replication_target_endpoint_inner(&parse_url("http://10.0.0.5:9000"), allow_loopback).is_ok());
|
||||
assert!(validate_replication_target_endpoint_inner(&parse_url("http://192.168.1.20"), allow_loopback).is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_endpoint_rejects_loopback_without_opt_in() {
|
||||
// Default (production) behaviour: loopback IP and localhost host both rejected.
|
||||
let err = validate_remote_endpoint_inner(&parse_url("http://127.0.0.1:9000"), false)
|
||||
let err = validate_replication_target_endpoint_inner(&parse_url("http://127.0.0.1:9000"), false)
|
||||
.expect_err("loopback IP must be rejected by default");
|
||||
assert!(matches!(
|
||||
err,
|
||||
@@ -2894,7 +3225,7 @@ mod tests {
|
||||
..
|
||||
}
|
||||
));
|
||||
let err = validate_remote_endpoint_inner(&parse_url("http://localhost:9000"), false)
|
||||
let err = validate_replication_target_endpoint_inner(&parse_url("http://localhost:9000"), false)
|
||||
.expect_err("localhost must be rejected by default");
|
||||
assert!(matches!(
|
||||
err,
|
||||
@@ -2909,15 +3240,15 @@ mod tests {
|
||||
fn replication_endpoint_allows_loopback_with_opt_in() {
|
||||
// e2e harness / single-host multi-instance: opt-in re-enables loopback in
|
||||
// both IP (127.0.0.1, ::1) and hostname (localhost) forms.
|
||||
assert!(validate_remote_endpoint_inner(&parse_url("http://127.0.0.1:9000"), true).is_ok());
|
||||
assert!(validate_remote_endpoint_inner(&parse_url("http://[::1]:9000"), true).is_ok());
|
||||
assert!(validate_remote_endpoint_inner(&parse_url("http://localhost:9000"), true).is_ok());
|
||||
assert!(validate_replication_target_endpoint_inner(&parse_url("http://127.0.0.1:9000"), true).is_ok());
|
||||
assert!(validate_replication_target_endpoint_inner(&parse_url("http://[::1]:9000"), true).is_ok());
|
||||
assert!(validate_replication_target_endpoint_inner(&parse_url("http://localhost:9000"), true).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_endpoint_opt_in_does_not_open_other_ssrf_targets() {
|
||||
// The loopback opt-in must not widen into link-local / metadata endpoints.
|
||||
let err = validate_remote_endpoint_inner(&parse_url("http://169.254.169.254/latest/meta-data"), true)
|
||||
let err = validate_replication_target_endpoint_inner(&parse_url("http://169.254.169.254/latest/meta-data"), true)
|
||||
.expect_err("metadata endpoint must stay rejected even with loopback opt-in");
|
||||
assert!(matches!(
|
||||
err,
|
||||
@@ -2926,7 +3257,7 @@ mod tests {
|
||||
..
|
||||
}
|
||||
));
|
||||
let err = validate_remote_endpoint_inner(&parse_url("http://[fe80::1]:9000"), true)
|
||||
let err = validate_replication_target_endpoint_inner(&parse_url("http://[fe80::1]:9000"), true)
|
||||
.expect_err("link-local must stay rejected even with loopback opt-in");
|
||||
assert!(matches!(
|
||||
err,
|
||||
@@ -3935,12 +4266,12 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn skip_tls_verify_takes_priority_over_invalid_custom_ca_pem() {
|
||||
let client = build_aws_s3_http_client_for_spec(&RemoteS3EndpointSpec::from(&BucketTarget {
|
||||
let client = build_aws_s3_http_client_for_target(&BucketTarget {
|
||||
secure: true,
|
||||
skip_tls_verify: true,
|
||||
ca_cert_pem: "not a pem".to_string(),
|
||||
..Default::default()
|
||||
}))
|
||||
})
|
||||
.await
|
||||
.expect("skip verification should bypass custom CA parsing");
|
||||
|
||||
|
||||
@@ -921,7 +921,7 @@ impl ExpiryState {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn enqueue_free_version(&self, oi: ObjectInfo) -> bool {
|
||||
pub fn enqueue_free_version(&mut self, oi: ObjectInfo) -> bool {
|
||||
let task = FreeVersionTask(oi);
|
||||
let wrkr = self.get_worker_ch(task.op_hash());
|
||||
if wrkr.is_none() {
|
||||
@@ -1215,22 +1215,6 @@ impl ExpiryState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn enqueue_committed_free_versions(api: &ECStore, free_versions: Vec<ObjectInfo>) -> usize {
|
||||
if free_versions.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let expiry_state = api.ctx.expiry_state();
|
||||
let state = expiry_state.read().await;
|
||||
let mut queued = 0;
|
||||
for free_version in free_versions {
|
||||
if state.enqueue_free_version(free_version) {
|
||||
queued += 1;
|
||||
}
|
||||
}
|
||||
queued
|
||||
}
|
||||
|
||||
async fn enqueue_recovered_free_version_with_state(state: &Arc<RwLock<ExpiryState>>, oi: ObjectInfo) -> bool {
|
||||
let task = FreeVersionTask(oi);
|
||||
let hash = task.op_hash();
|
||||
@@ -6643,7 +6627,7 @@ mod tests {
|
||||
async fn enqueue_free_version_reports_false_without_worker_channel() {
|
||||
let state = ExpiryState::new();
|
||||
let recovery_notify = Arc::clone(&state.read().await.recovery_notify);
|
||||
let state = state.write().await;
|
||||
let mut state = state.write().await;
|
||||
let oi = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
@@ -6835,7 +6819,7 @@ mod tests {
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let state = state.write().await;
|
||||
let mut state = state.write().await;
|
||||
|
||||
assert!(state.enqueue_free_version(oi.clone()));
|
||||
assert!(recovery_notify.notified().now_or_never().is_none());
|
||||
|
||||
@@ -270,7 +270,6 @@ pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
|
||||
pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
|
||||
pub const BUCKET_TABLE_CONFIG: &str = "table-bucket.json";
|
||||
pub const BUCKET_DURABILITY_CONFIG: &str = "durability.json";
|
||||
pub const BUCKET_ON_DEMAND_MIGRATION_CONFIG: &str = "on-demand-migration.json";
|
||||
pub const BUCKET_TABLE_RESERVED_PREFIX: &str = ".rustfs-table";
|
||||
pub const BUCKET_TABLE_CATALOG_META_PREFIX: &str = "s3tables/catalog";
|
||||
pub const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str = "table-buckets";
|
||||
@@ -322,7 +321,6 @@ pub struct BucketMetadata {
|
||||
pub bucket_acl_config_json: Vec<u8>,
|
||||
pub table_bucket_config_json: Vec<u8>,
|
||||
pub durability_config_json: Vec<u8>,
|
||||
pub on_demand_migration_config_json: Vec<u8>,
|
||||
|
||||
pub policy_config_updated_at: OffsetDateTime,
|
||||
pub object_lock_config_updated_at: OffsetDateTime,
|
||||
@@ -344,7 +342,6 @@ pub struct BucketMetadata {
|
||||
pub bucket_acl_config_updated_at: OffsetDateTime,
|
||||
pub table_bucket_config_updated_at: OffsetDateTime,
|
||||
pub durability_config_updated_at: OffsetDateTime,
|
||||
pub on_demand_migration_config_updated_at: OffsetDateTime,
|
||||
|
||||
pub new_field_updated_at: OffsetDateTime,
|
||||
|
||||
@@ -396,7 +393,6 @@ impl Default for BucketMetadata {
|
||||
bucket_acl_config_json: Default::default(),
|
||||
table_bucket_config_json: Default::default(),
|
||||
durability_config_json: Default::default(),
|
||||
on_demand_migration_config_json: Default::default(),
|
||||
policy_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
@@ -417,7 +413,6 @@ impl Default for BucketMetadata {
|
||||
bucket_acl_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
table_bucket_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
durability_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
on_demand_migration_config_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
new_field_updated_at: OffsetDateTime::UNIX_EPOCH,
|
||||
policy_config: Default::default(),
|
||||
notification_config: Default::default(),
|
||||
@@ -482,23 +477,6 @@ impl BucketMetadata {
|
||||
/// Absent/empty/unparsable payloads all mean "no override" (the bucket
|
||||
/// follows the global durability mode); a parse failure is logged so a
|
||||
/// corrupted entry cannot silently change fsync behavior.
|
||||
/// Parsed on-demand migration config, if one is stored.
|
||||
///
|
||||
/// `Ok(None)` means no config (absent or cleared). A stored payload that
|
||||
/// does not parse is an error, never a default: the runtime must not
|
||||
/// pull from a source it cannot describe.
|
||||
pub fn on_demand_migration_config(
|
||||
&self,
|
||||
) -> std::result::Result<
|
||||
Option<super::on_demand_migration::OnDemandMigrationConfig>,
|
||||
super::on_demand_migration::OnDemandMigrationConfigError,
|
||||
> {
|
||||
if self.on_demand_migration_config_json.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
super::on_demand_migration::OnDemandMigrationConfig::from_json(&self.on_demand_migration_config_json).map(Some)
|
||||
}
|
||||
|
||||
pub fn durability_config(&self) -> Option<super::durability::BucketDurabilityConfig> {
|
||||
if self.durability_config_json.is_empty() {
|
||||
return None;
|
||||
@@ -577,9 +555,6 @@ impl BucketMetadata {
|
||||
"BucketAclConfigJSON" | "BucketAclConfigJson" => self.bucket_acl_config_json = read_msgp_bin(rd)?,
|
||||
"TableBucketConfigJSON" | "TableBucketConfigJson" => self.table_bucket_config_json = read_msgp_bin(rd)?,
|
||||
"DurabilityConfigJSON" | "DurabilityConfigJson" => self.durability_config_json = read_msgp_bin(rd)?,
|
||||
"OnDemandMigrationConfigJSON" | "OnDemandMigrationConfigJson" => {
|
||||
self.on_demand_migration_config_json = read_msgp_bin(rd)?
|
||||
}
|
||||
"CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"LoggingConfigUpdatedAt" => self.logging_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"WebsiteConfigUpdatedAt" => self.website_config_updated_at = read_msgp_time_value(rd)?,
|
||||
@@ -589,7 +564,6 @@ impl BucketMetadata {
|
||||
"BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
|
||||
"OnDemandMigrationConfigUpdatedAt" => self.on_demand_migration_config_updated_at = read_msgp_time_value(rd)?,
|
||||
other => {
|
||||
tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field");
|
||||
skip_msgp_value(rd)?;
|
||||
@@ -602,8 +576,8 @@ impl BucketMetadata {
|
||||
|
||||
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
|
||||
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
|
||||
// Map size: MinIO fields (25) + RustFS extensions (21)
|
||||
let map_len: u32 = 46;
|
||||
// Map size: MinIO fields (25) + RustFS extensions (19)
|
||||
let map_len: u32 = 44;
|
||||
rmp::encode::write_map_len(wr, map_len)?;
|
||||
|
||||
// MinIO field order (same as Go struct)
|
||||
@@ -663,7 +637,6 @@ impl BucketMetadata {
|
||||
write_bin_field(wr, "BucketAclConfigJSON", &self.bucket_acl_config_json)?;
|
||||
write_bin_field(wr, "TableBucketConfigJSON", &self.table_bucket_config_json)?;
|
||||
write_bin_field(wr, "DurabilityConfigJSON", &self.durability_config_json)?;
|
||||
write_bin_field(wr, "OnDemandMigrationConfigJSON", &self.on_demand_migration_config_json)?;
|
||||
rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.cors_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "LoggingConfigUpdatedAt")?;
|
||||
@@ -682,8 +655,6 @@ impl BucketMetadata {
|
||||
write_msgp_time(wr, self.table_bucket_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "DurabilityConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.durability_config_updated_at)?;
|
||||
rmp::encode::write_str(wr, "OnDemandMigrationConfigUpdatedAt")?;
|
||||
write_msgp_time(wr, self.on_demand_migration_config_updated_at)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -785,9 +756,6 @@ impl BucketMetadata {
|
||||
if self.durability_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.durability_config_updated_at = self.created
|
||||
}
|
||||
if self.on_demand_migration_config_updated_at == OffsetDateTime::UNIX_EPOCH {
|
||||
self.on_demand_migration_config_updated_at = self.created
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_config(&mut self, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
|
||||
@@ -903,17 +871,6 @@ impl BucketMetadata {
|
||||
self.durability_config_json = data;
|
||||
self.durability_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_ON_DEMAND_MIGRATION_CONFIG => {
|
||||
// Structural check only (shape, unknown fields); the
|
||||
// deployment-relative rules run in the admin handler with a
|
||||
// `ValidationContext`. A blob this build cannot read must not
|
||||
// be persisted for every later reader to trip over.
|
||||
if !data.is_empty() {
|
||||
super::on_demand_migration::OnDemandMigrationConfig::from_json(&data).map_err(Error::other)?;
|
||||
}
|
||||
self.on_demand_migration_config_json = data;
|
||||
self.on_demand_migration_config_updated_at = updated;
|
||||
}
|
||||
_ => return Err(Error::other(format!("config file not found : {config_file}"))),
|
||||
}
|
||||
|
||||
@@ -1822,117 +1779,6 @@ mod test {
|
||||
assert!(!bm.table_bucket_enabled());
|
||||
}
|
||||
|
||||
const ODM_JSON: &[u8] = br#"{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
|
||||
|
||||
/// rustfs/backlog#2148: the on-demand migration config is a RustFS
|
||||
/// extension entry that round-trips through `update_config` and the
|
||||
/// msgpack codec, clears on delete, and never parses corruption into a
|
||||
/// default.
|
||||
#[test]
|
||||
fn on_demand_migration_config_round_trips_and_tracks_updates() {
|
||||
use crate::bucket::on_demand_migration::{OnDemandMigrationConfig, OnDemandMigrationConfigError};
|
||||
|
||||
let mut bm = BucketMetadata::new("odm-bucket");
|
||||
assert_eq!(bm.on_demand_migration_config(), Ok(None), "fresh metadata carries no config");
|
||||
|
||||
let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
|
||||
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
|
||||
.expect("valid config is accepted");
|
||||
assert_ne!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(bm.on_demand_migration_config(), Ok(Some(expected.clone())));
|
||||
|
||||
let back = BucketMetadata::unmarshal(&bm.marshal_msg().unwrap()).unwrap();
|
||||
assert_eq!(back.on_demand_migration_config_json, bm.on_demand_migration_config_json);
|
||||
assert_eq!(
|
||||
back.on_demand_migration_config_updated_at.unix_timestamp(),
|
||||
bm.on_demand_migration_config_updated_at.unix_timestamp()
|
||||
);
|
||||
assert_eq!(back.on_demand_migration_config(), Ok(Some(expected)));
|
||||
|
||||
// A blob this build cannot read is rejected at the write boundary
|
||||
// rather than persisted for every reader to trip over.
|
||||
let before = bm.on_demand_migration_config_json.clone();
|
||||
assert!(
|
||||
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec())
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(bm.on_demand_migration_config_json, before, "a rejected update leaves the blob untouched");
|
||||
|
||||
// Delete clears the entry.
|
||||
let stamped = bm.on_demand_migration_config_updated_at;
|
||||
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, Vec::new()).unwrap();
|
||||
assert!(bm.on_demand_migration_config_json.is_empty());
|
||||
assert_eq!(bm.on_demand_migration_config(), Ok(None));
|
||||
assert!(bm.on_demand_migration_config_updated_at >= stamped);
|
||||
|
||||
// Corruption that bypassed `update_config` (disk, another writer)
|
||||
// is a typed error, never a default.
|
||||
bm.on_demand_migration_config_json = b"not-json".to_vec();
|
||||
assert!(matches!(bm.on_demand_migration_config(), Err(OnDemandMigrationConfigError::Malformed(_))));
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2148: a `.metadata.bin` written before the on-demand
|
||||
/// migration keys existed decodes with an empty blob and an epoch
|
||||
/// timestamp that `default_timestamps` back-fills from `created`.
|
||||
#[test]
|
||||
fn on_demand_migration_config_absent_in_legacy_blob_defaults_to_created() {
|
||||
let blob = decode_hex(include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex"));
|
||||
let mut bm = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal MinIO bucket metadata");
|
||||
assert!(bm.on_demand_migration_config_json.is_empty());
|
||||
assert_eq!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(bm.on_demand_migration_config(), Ok(None));
|
||||
|
||||
bm.default_timestamps();
|
||||
assert_ne!(bm.created, OffsetDateTime::UNIX_EPOCH, "fixture must carry a real creation time");
|
||||
assert_eq!(bm.on_demand_migration_config_updated_at, bm.created);
|
||||
|
||||
// A metadata blob from this build with no config set stays
|
||||
// indistinguishable from the legacy one for these fields.
|
||||
let fresh = BucketMetadata::unmarshal(&BucketMetadata::new("fresh").marshal_msg().unwrap()).unwrap();
|
||||
assert!(fresh.on_demand_migration_config_json.is_empty());
|
||||
assert_eq!(fresh.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2148: a reader that predates the two on-demand
|
||||
/// migration keys takes `decode_from`'s unknown-field branch, which is
|
||||
/// `skip_msgp_value`. Walk the new-format blob with exactly that
|
||||
/// primitive and prove both keys are skipped without desynchronising the
|
||||
/// stream, so the fields that follow them still decode.
|
||||
#[test]
|
||||
fn old_decoder_skips_on_demand_migration_fields_without_desync() {
|
||||
let mut bm = BucketMetadata::new("odm-skip");
|
||||
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
|
||||
.unwrap();
|
||||
bm.update_config(BUCKET_DURABILITY_CONFIG, br#"{"mode":"relaxed"}"#.to_vec())
|
||||
.unwrap();
|
||||
let buf = bm.marshal_msg().unwrap();
|
||||
|
||||
let mut rd = std::io::Cursor::new(buf.as_slice());
|
||||
let fields = rmp::decode::read_map_len(&mut rd).unwrap();
|
||||
let mut skipped = Vec::new();
|
||||
let mut durability_json = Vec::new();
|
||||
for _ in 0..fields {
|
||||
let key_len = rmp::decode::read_str_len(&mut rd).unwrap();
|
||||
let mut key = vec![0u8; key_len as usize];
|
||||
rd.read_exact(&mut key).unwrap();
|
||||
let key = String::from_utf8(key).unwrap();
|
||||
match key.as_str() {
|
||||
// The field an old reader knows that is encoded *after* the
|
||||
// unknown JSON key and *before* the unknown timestamp key.
|
||||
"DurabilityConfigJSON" => durability_json = read_msgp_bin(&mut rd).unwrap(),
|
||||
other => {
|
||||
if other.starts_with("OnDemandMigration") {
|
||||
skipped.push(other.to_string());
|
||||
}
|
||||
skip_msgp_value(&mut rd).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
assert_eq!(skipped, ["OnDemandMigrationConfigJSON", "OnDemandMigrationConfigUpdatedAt"]);
|
||||
assert_eq!(durability_json, br#"{"mode":"relaxed"}"#);
|
||||
assert_eq!(rd.position() as usize, buf.len(), "old-style walk must consume the blob exactly");
|
||||
}
|
||||
|
||||
/// HP-5b (rustfs/backlog#938): the durability override is a RustFS
|
||||
/// extension entry and must survive an encode/decode round trip.
|
||||
#[test]
|
||||
|
||||
@@ -19,10 +19,9 @@ use super::quota::BucketQuota;
|
||||
use super::target::BucketTargets;
|
||||
use crate::bucket::bucket_target_sys::BucketTargetSys;
|
||||
use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_parse_with_presence};
|
||||
use crate::bucket::on_demand_migration::{ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig};
|
||||
use crate::bucket::utils::is_meta_bucketname;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
|
||||
use crate::error::{Error, Result, is_err_bucket_not_found};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::storage_api_contracts::heal::HealOperations as _;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
@@ -385,42 +384,6 @@ fn clear_bucket_durability(bucket: &str) {
|
||||
crate::disk::local::bucket_durability::set(bucket, None);
|
||||
}
|
||||
|
||||
/// Publish the bucket's on-demand migration config (or its absence) to the
|
||||
/// runtime registered in `ON_DEMAND_MIGRATION_CONFIG_HOOK`.
|
||||
///
|
||||
/// Called from the same five cache-install paths as
|
||||
/// [`sync_bucket_durability`]. A stored payload this build cannot parse is
|
||||
/// published as `None`: the runtime must stop pulling for that bucket rather
|
||||
/// than keep an older config or guess.
|
||||
fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
|
||||
let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() else {
|
||||
return;
|
||||
};
|
||||
match bm.on_demand_migration_config() {
|
||||
Ok(config) => hook(bucket, config.as_ref()),
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %bucket,
|
||||
config = "on_demand_migration",
|
||||
error = %err,
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
hook(bucket, None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Withdraw a bucket's on-demand migration config when its metadata leaves
|
||||
/// the cache.
|
||||
fn clear_on_demand_migration(bucket: &str) {
|
||||
if let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() {
|
||||
hook(bucket, None);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get(bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
let lock = sys.read().await;
|
||||
@@ -696,12 +659,13 @@ async fn acquire_config_write_guard_for_incarnation(
|
||||
async {
|
||||
match metadata_sys
|
||||
.api
|
||||
.get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.peer_sys
|
||||
.get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) if is_err_strict_volume_not_found(&err) => Err(Error::BucketNotFound(bucket.to_string())),
|
||||
Err(err) => Err(err),
|
||||
Err(crate::disk::error::Error::VolumeNotFound) => Err(Error::BucketNotFound(bucket.to_string())),
|
||||
Err(err) => Err(err.into()),
|
||||
}
|
||||
},
|
||||
),
|
||||
@@ -1007,16 +971,6 @@ pub async fn get_durability_config(
|
||||
Ok((bm.durability_config(), bm.durability_config_updated_at))
|
||||
}
|
||||
|
||||
/// The bucket's on-demand migration config with its update time, or
|
||||
/// `Ok(None)` when the bucket has none. A stored payload that does not parse
|
||||
/// is a typed error (`OnDemandMigrationConfigError` inside `Error::Io`).
|
||||
pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
bucket_meta_sys.get_on_demand_migration_config(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -1320,6 +1274,7 @@ pub struct BucketMetadataSys {
|
||||
/// name floods while avoiding repeated namespace and erasure reads.
|
||||
missing_buckets: moka::future::Cache<String, ()>,
|
||||
api: Arc<ECStore>,
|
||||
initialized: Arc<RwLock<bool>>,
|
||||
}
|
||||
|
||||
impl BucketMetadataSys {
|
||||
@@ -1348,6 +1303,7 @@ impl BucketMetadataSys {
|
||||
.time_to_live(MISSING_BUCKET_TTL)
|
||||
.build(),
|
||||
api,
|
||||
initialized: Arc::new(RwLock::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1408,12 +1364,13 @@ impl BucketMetadataSys {
|
||||
await_bucket_namespace_operation(Some(namespace_guard), bucket, operation, async {
|
||||
match self
|
||||
.api
|
||||
.get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.peer_sys
|
||||
.get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(true),
|
||||
Err(Error::VolumeNotFound) => Ok(false),
|
||||
Err(err) => Err(err),
|
||||
Err(crate::disk::error::Error::VolumeNotFound) => Ok(false),
|
||||
Err(err) => Err(err.into()),
|
||||
}
|
||||
})
|
||||
.await
|
||||
@@ -1423,15 +1380,9 @@ impl BucketMetadataSys {
|
||||
let _ = self.init_internal(buckets).await;
|
||||
}
|
||||
async fn init_internal(&self, buckets: Vec<String>) -> Result<()> {
|
||||
let count = self
|
||||
.api
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| pool.disk_set.len())
|
||||
.sum::<usize>()
|
||||
.checked_mul(10)
|
||||
.filter(|count| *count != 0)
|
||||
.ok_or_else(|| Error::other("bucket metadata store has no erasure sets"))?;
|
||||
let count = runtime_sources::endpoint_erasure_set_count()
|
||||
.map(|count| count * 10)
|
||||
.ok_or_else(|| Error::other("endpoint pools not initialized"))?;
|
||||
|
||||
let mut failed_buckets: HashSet<String> = HashSet::new();
|
||||
let mut buckets = buckets.as_slice();
|
||||
@@ -1449,6 +1400,9 @@ impl BucketMetadataSys {
|
||||
buckets = &buckets[count..]
|
||||
}
|
||||
|
||||
let mut initialized = self.initialized.write().await;
|
||||
*initialized = true;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1525,6 +1479,14 @@ impl BucketMetadataSys {
|
||||
expected: Option<&Arc<BucketMetadata>>,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
await_bucket_namespace_operation(
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
"bucket metadata heal",
|
||||
self.api.heal_bucket(bucket, &HealOpts::default()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if !self
|
||||
.bucket_exists(bucket, namespace_guard, "bucket metadata existence check")
|
||||
.await?
|
||||
@@ -1539,26 +1501,11 @@ impl BucketMetadataSys {
|
||||
if removed {
|
||||
BucketTargetSys::get().delete(bucket).await;
|
||||
clear_bucket_durability(bucket);
|
||||
clear_on_demand_migration(bucket);
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
await_bucket_namespace_operation(
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
"bucket metadata heal",
|
||||
self.api.heal_bucket(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
recreate: true,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let (bm, persisted) = await_bucket_namespace_operation(
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
@@ -1577,7 +1524,6 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &bm).await;
|
||||
sync_bucket_durability(bucket, &bm);
|
||||
sync_on_demand_migration(bucket, &bm);
|
||||
}
|
||||
MetadataLoadMode::Initial => {
|
||||
let _publish_guard = self
|
||||
@@ -1624,7 +1570,6 @@ impl BucketMetadataSys {
|
||||
if removed {
|
||||
BucketTargetSys::get().delete(bucket).await;
|
||||
clear_bucket_durability(bucket);
|
||||
clear_on_demand_migration(bucket);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
@@ -1647,7 +1592,6 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &metadata).await;
|
||||
sync_bucket_durability(bucket, &metadata);
|
||||
sync_on_demand_migration(bucket, &metadata);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1675,7 +1619,6 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(&bucket).await;
|
||||
sync_bucket_target_sys(&bucket, &bm).await;
|
||||
sync_bucket_durability(&bucket, &bm);
|
||||
sync_on_demand_migration(&bucket, &bm);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1696,7 +1639,6 @@ impl BucketMetadataSys {
|
||||
if removed {
|
||||
BucketTargetSys::get().delete(bucket).await;
|
||||
clear_bucket_durability(bucket);
|
||||
clear_on_demand_migration(bucket);
|
||||
}
|
||||
removed || removed_fabricated
|
||||
}
|
||||
@@ -1945,13 +1887,23 @@ impl BucketMetadataSys {
|
||||
"lazy metadata IO must start while the bucket namespace read lock is held"
|
||||
);
|
||||
}
|
||||
let (bm, persisted) = await_bucket_namespace_operation(
|
||||
let (bm, persisted) = match await_bucket_namespace_operation(
|
||||
Some(&guard),
|
||||
bucket,
|
||||
"lazy bucket metadata load",
|
||||
Box::pin(load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true)),
|
||||
)
|
||||
.await?;
|
||||
.await
|
||||
{
|
||||
Ok(res) => res,
|
||||
Err(err) => {
|
||||
return if *self.initialized.read().await {
|
||||
Err(Error::other("errBucketMetadataNotInitialized"))
|
||||
} else {
|
||||
Err(err)
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let bm = Arc::new(bm);
|
||||
|
||||
@@ -1962,9 +1914,11 @@ impl BucketMetadataSys {
|
||||
"lazy bucket metadata existence check",
|
||||
Box::pin(async {
|
||||
self.api
|
||||
.get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.peer_sys
|
||||
.get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(Into::into)
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
@@ -1986,7 +1940,6 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &bm).await;
|
||||
sync_bucket_durability(bucket, &bm);
|
||||
sync_on_demand_migration(bucket, &bm);
|
||||
} else {
|
||||
let exists = self
|
||||
.bucket_exists(bucket, &guard, "lazy bucket metadata existence check")
|
||||
@@ -2244,8 +2197,10 @@ impl BucketMetadataSys {
|
||||
"legacy bucket metadata existence check",
|
||||
async {
|
||||
self.api
|
||||
.get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.peer_sys
|
||||
.get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::StorageError::from)
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -2325,7 +2280,6 @@ impl BucketMetadataSys {
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &metadata).await;
|
||||
sync_bucket_durability(bucket, &metadata);
|
||||
sync_on_demand_migration(bucket, &metadata);
|
||||
Ok(BucketMetadataAuthority::Authoritative(metadata))
|
||||
}
|
||||
|
||||
@@ -2348,8 +2302,10 @@ impl BucketMetadataSys {
|
||||
"bucket metadata snapshot existence check",
|
||||
async {
|
||||
self.api
|
||||
.get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.peer_sys
|
||||
.get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::StorageError::from)
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -2518,17 +2474,6 @@ impl BucketMetadataSys {
|
||||
Err(Error::ConfigNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
/// See [`get_on_demand_migration_config`].
|
||||
pub async fn get_on_demand_migration_config(
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
let config = bm.on_demand_migration_config().map_err(Error::other)?;
|
||||
Ok(config.map(|config| (config, bm.on_demand_migration_config_updated_at)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Test-only fixture shared with sibling modules (e.g. the quota checker
|
||||
@@ -4109,151 +4054,6 @@ mod tests {
|
||||
assert_eq!(bucket_durability::lookup(bucket), None);
|
||||
}
|
||||
|
||||
const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
|
||||
|
||||
/// Every `(bucket, config)` the recording hook has seen. Tests filter by
|
||||
/// their own bucket name; the hook is process-wide and set once.
|
||||
static ODM_HOOK_CALLS: std::sync::Mutex<Vec<(String, Option<OnDemandMigrationConfig>)>> = std::sync::Mutex::new(Vec::new());
|
||||
|
||||
fn install_recording_odm_hook() {
|
||||
ON_DEMAND_MIGRATION_CONFIG_HOOK.get_or_init(|| {
|
||||
Box::new(|bucket, config| {
|
||||
ODM_HOOK_CALLS.lock().unwrap().push((bucket.to_string(), config.cloned()));
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
fn odm_hook_calls(bucket: &str) -> Vec<Option<OnDemandMigrationConfig>> {
|
||||
ODM_HOOK_CALLS
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|(name, _)| name == bucket)
|
||||
.map(|(_, config)| config.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2148: the accessor reports absence as `Ok(None)` and a
|
||||
/// stored payload it cannot parse as a typed error, never as a default
|
||||
/// and never as `ConfigNotFound`.
|
||||
#[tokio::test]
|
||||
async fn get_on_demand_migration_config_distinguishes_absent_from_corrupt() {
|
||||
use crate::bucket::on_demand_migration::OnDemandMigrationConfigError;
|
||||
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore);
|
||||
let bucket = "odm-accessor";
|
||||
|
||||
sys.set(bucket.to_string(), Arc::new(BucketMetadata::new(bucket))).await;
|
||||
assert_eq!(sys.get_on_demand_migration_config(bucket).await.unwrap(), None);
|
||||
|
||||
let mut corrupt = BucketMetadata::new(bucket);
|
||||
corrupt.on_demand_migration_config_json = br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec();
|
||||
sys.set(bucket.to_string(), Arc::new(corrupt)).await;
|
||||
let err = sys
|
||||
.get_on_demand_migration_config(bucket)
|
||||
.await
|
||||
.expect_err("corrupt config must not read as a default");
|
||||
assert_ne!(err, Error::ConfigNotFound, "corruption must not be reported as absence");
|
||||
let typed = match &err {
|
||||
Error::Io(io) => io
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<OnDemandMigrationConfigError>()),
|
||||
_ => None,
|
||||
};
|
||||
assert!(
|
||||
matches!(typed, Some(OnDemandMigrationConfigError::Malformed(_))),
|
||||
"typed parse error must survive the Result boundary, got: {err:?}"
|
||||
);
|
||||
|
||||
let mut valid = BucketMetadata::new(bucket);
|
||||
valid
|
||||
.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
|
||||
.unwrap();
|
||||
let stamped = valid.on_demand_migration_config_updated_at;
|
||||
sys.set(bucket.to_string(), Arc::new(valid)).await;
|
||||
let (config, updated_at) = sys
|
||||
.get_on_demand_migration_config(bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("stored config is returned");
|
||||
assert_eq!(config, OnDemandMigrationConfig::from_json(ODM_JSON).unwrap());
|
||||
assert_eq!(updated_at, stamped);
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2148: the publish hook fires on every path that
|
||||
/// installs bucket metadata into the cache (set, initial load, peer
|
||||
/// reload, refresh loop, lazy load) and withdraws on removal, mirroring
|
||||
/// `sync_bucket_durability`.
|
||||
#[tokio::test]
|
||||
async fn on_demand_migration_hook_fires_on_every_cache_install_path() {
|
||||
install_recording_odm_hook();
|
||||
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "odm-hook-paths";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("physical bucket should exist");
|
||||
}
|
||||
let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
|
||||
let expect_publish = |before: usize, label: &str| {
|
||||
let calls = odm_hook_calls(bucket);
|
||||
assert_eq!(calls.len(), before + 1, "{label} must publish exactly once");
|
||||
assert_eq!(calls.last().unwrap().as_ref(), Some(&expected), "{label} must publish the stored config");
|
||||
};
|
||||
|
||||
// set (via persist_new_and_set, which installs through `set`).
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
|
||||
.unwrap();
|
||||
let writer = BucketMetadataSys::new(ecstore.clone());
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
writer.persist_new_and_set(bm).await.expect("metadata should persist");
|
||||
expect_publish(before, "set");
|
||||
|
||||
// init (initial load on a cold system).
|
||||
let mut cold = BucketMetadataSys::new(ecstore.clone());
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
cold.init(vec![bucket.to_string()]).await;
|
||||
assert!(cold.get(bucket).await.is_ok(), "initial load must cache the bucket");
|
||||
expect_publish(before, "init");
|
||||
|
||||
// peer reload.
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
cold.reload_from_store(bucket).await.expect("peer reload should publish");
|
||||
expect_publish(before, "peer reload");
|
||||
|
||||
// refresh loop.
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let mut failed = HashSet::new();
|
||||
cold.concurrent_load(&[bucket.to_string()], &mut failed, MetadataLoadMode::Refresh)
|
||||
.await;
|
||||
assert!(failed.is_empty(), "refresh must succeed");
|
||||
expect_publish(before, "refresh loop");
|
||||
|
||||
// lazy load on another cold system.
|
||||
let lazy = BucketMetadataSys::new(ecstore);
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
let (_, loaded) = lazy.get_config(bucket).await.expect("lazy load should publish");
|
||||
assert!(loaded, "the lazy path must have gone to disk");
|
||||
expect_publish(before, "lazy load");
|
||||
|
||||
// Removal withdraws the config.
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
assert!(lazy.remove(bucket).await);
|
||||
let calls = odm_hook_calls(bucket);
|
||||
assert_eq!(calls.len(), before + 1, "remove must withdraw exactly once");
|
||||
assert_eq!(calls.last().unwrap(), &None);
|
||||
|
||||
// A corrupt payload is withdrawn, never published as a config.
|
||||
let mut corrupt = BucketMetadata::new(bucket);
|
||||
corrupt.on_demand_migration_config_json = b"not-json".to_vec();
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
lazy.set(bucket.to_string(), Arc::new(corrupt)).await;
|
||||
let calls = odm_hook_calls(bucket);
|
||||
assert_eq!(calls.len(), before + 1);
|
||||
assert_eq!(calls.last().unwrap(), &None, "unreadable config must publish absence");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_wait_exits_when_cancelled() {
|
||||
let cancel_token = CancellationToken::new();
|
||||
|
||||
@@ -26,10 +26,8 @@ mod metadata_test;
|
||||
pub mod migration;
|
||||
mod msgp_decode;
|
||||
pub mod object_lock;
|
||||
pub mod on_demand_migration;
|
||||
pub mod policy_sys;
|
||||
pub mod quota;
|
||||
pub mod remote_s3_client;
|
||||
pub mod replication;
|
||||
pub mod tagging;
|
||||
pub mod target;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,28 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! On-Demand Migration (ODM): a bucket can name an external S3-compatible
|
||||
//! source bucket; GET misses are served from that source and backfilled
|
||||
//! locally. This module owns the bucket-level configuration model
|
||||
//! (`on-demand-migration.json` in the bucket metadata file); the runtime is
|
||||
//! layered on top of it by later tasks (rustfs/backlog#2147).
|
||||
|
||||
pub mod config;
|
||||
pub mod source_client;
|
||||
|
||||
pub use config::{
|
||||
ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION,
|
||||
OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, SourceConfig,
|
||||
SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
|
||||
};
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,789 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Shared builder for outbound `aws_sdk_s3::Client`s.
|
||||
//!
|
||||
//! Replication targets (`bucket_target_sys`) and the on-demand migration
|
||||
//! source client build their remote clients from one neutral
|
||||
//! [`RemoteS3EndpointSpec`]: endpoint assembly, credential handling, path-style
|
||||
//! selection, custom CA / skip-TLS transports and the outbound SSRF gate all
|
||||
//! live here so both callers share exactly one policy. The gate keeps the
|
||||
//! relaxed replication semantics documented in
|
||||
//! `docs/operations/outbound-connection-policy.md`: private addresses are
|
||||
//! always allowed, loopback only behind `RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET`.
|
||||
|
||||
use aws_credential_types::Credentials as SdkCredentials;
|
||||
use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future};
|
||||
use aws_sdk_s3::config::Region as SdkRegion;
|
||||
use aws_sdk_s3::config::RequestChecksumCalculation;
|
||||
use aws_sdk_s3::config::SharedCredentialsProvider;
|
||||
use aws_sdk_s3::config::SharedHttpClient;
|
||||
use aws_sdk_s3::{Client as S3Client, Config as S3Config};
|
||||
use aws_smithy_http_client::{Builder as SmithyHttpClientBuilder, tls as smithy_tls};
|
||||
use aws_smithy_runtime_api::box_error::BoxError;
|
||||
use aws_smithy_runtime_api::client::http::{
|
||||
HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
|
||||
};
|
||||
use aws_smithy_runtime_api::client::interceptors::Intercept;
|
||||
use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
|
||||
use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
|
||||
use aws_smithy_runtime_api::client::result::ConnectorError;
|
||||
use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use aws_smithy_types::config_bag::ConfigBag;
|
||||
use aws_smithy_types::timeout::TimeoutConfig;
|
||||
use http::Uri;
|
||||
use hyper_util::client::legacy::Client as HyperClient;
|
||||
use hyper_util::rt::{TokioExecutor, TokioTimer};
|
||||
use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RUSTFS_CA_CERT, RUSTFS_TLS_CERT};
|
||||
use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url};
|
||||
use rustls_pki_types::pem::PemObject;
|
||||
use std::fmt;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tower::Service;
|
||||
use tracing::warn;
|
||||
use url::Url;
|
||||
|
||||
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||
pub(crate) const EXPIRED_REMOTE_TARGET_CREDENTIALS: &str = "remote target credentials have expired";
|
||||
|
||||
/// Request addressing style for a remote S3-compatible endpoint.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum PathStyle {
|
||||
/// Caller did not choose; the builder defaults to path-style because that
|
||||
/// is what custom S3-compatible endpoints accept most reliably.
|
||||
Auto,
|
||||
/// `https://endpoint/bucket/key`.
|
||||
Path,
|
||||
/// `https://bucket.endpoint/key`.
|
||||
VirtualHost,
|
||||
}
|
||||
|
||||
impl PathStyle {
|
||||
/// Resolves the style to the SDK `force_path_style` flag. `Auto` keeps
|
||||
/// the historical replication default (path-style).
|
||||
pub fn force_path_style(self) -> bool {
|
||||
!matches!(self, PathStyle::VirtualHost)
|
||||
}
|
||||
}
|
||||
|
||||
/// Static or temporary credentials for a remote endpoint. `expiration` without
|
||||
/// a `session_token` is rejected at build time: only STS-style temporary
|
||||
/// credentials expire, so that combination is a corrupted configuration
|
||||
/// rather than a static key.
|
||||
#[derive(Clone)]
|
||||
pub struct RemoteCredentials {
|
||||
pub access_key: String,
|
||||
pub secret_key: String,
|
||||
pub session_token: Option<String>,
|
||||
pub expiration: Option<SystemTime>,
|
||||
/// SDK credential `account_id`; replication targets pass their reset id.
|
||||
pub account_id: String,
|
||||
}
|
||||
|
||||
impl fmt::Debug for RemoteCredentials {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("RemoteCredentials")
|
||||
.field("access_key", &self.access_key)
|
||||
.field("secret_key", &REDACTED_CREDENTIAL)
|
||||
.field("session_token", &self.session_token.as_ref().map(|_| REDACTED_CREDENTIAL))
|
||||
.field("expiration", &self.expiration)
|
||||
.field("account_id", &self.account_id)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// Neutral description of a remote S3 endpoint from which an
|
||||
/// `aws_sdk_s3::Client` is built.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct RemoteS3EndpointSpec {
|
||||
/// `host[:port]` without a scheme; `secure` selects `https` or `http`.
|
||||
pub endpoint: String,
|
||||
pub secure: bool,
|
||||
pub region: String,
|
||||
pub path_style: PathStyle,
|
||||
pub credentials: Option<RemoteCredentials>,
|
||||
/// Accept any server certificate. Takes priority over `ca_cert_pem`.
|
||||
pub skip_tls_verify: bool,
|
||||
/// Extra PEM bundle trusted alongside the platform roots and the
|
||||
/// `RUSTFS_TLS_PATH` bundle. `None` and whitespace-only mean "not set".
|
||||
pub ca_cert_pem: Option<String>,
|
||||
pub connect_timeout: Option<Duration>,
|
||||
pub read_timeout: Option<Duration>,
|
||||
/// Appended to the SDK `User-Agent` (space separated) so the remote side
|
||||
/// can identify the caller; empty means no suffix.
|
||||
pub user_agent_suffix: &'static str,
|
||||
}
|
||||
|
||||
impl RemoteS3EndpointSpec {
|
||||
/// Full endpoint URL (`scheme://host[:port]`) as handed to the SDK.
|
||||
pub fn endpoint_url(&self) -> String {
|
||||
if self.secure {
|
||||
format!("https://{}", self.endpoint)
|
||||
} else {
|
||||
format!("http://{}", self.endpoint)
|
||||
}
|
||||
}
|
||||
|
||||
fn custom_ca_pem(&self) -> Option<&str> {
|
||||
self.ca_cert_pem.as_deref().filter(|pem| !pem.trim().is_empty())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum RemoteS3ClientError {
|
||||
#[error("remote endpoint requires credentials")]
|
||||
MissingCredentials,
|
||||
#[error("{0}")]
|
||||
Credentials(&'static str),
|
||||
#[error("invalid target endpoint: {0}")]
|
||||
InvalidEndpoint(String),
|
||||
#[error("target endpoint is not allowed: {0}")]
|
||||
EndpointNotAllowed(#[source] OutboundUrlError),
|
||||
#[error("invalid target CA PEM: {0}")]
|
||||
InvalidCaPem(String),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct RemoteTargetCredentialsProvider {
|
||||
pub(crate) credentials: SdkCredentials,
|
||||
}
|
||||
|
||||
impl RemoteTargetCredentialsProvider {
|
||||
pub(crate) fn resolve_at(&self, now: SystemTime) -> aws_credential_types::provider::Result {
|
||||
if self.credentials.expiry().is_some_and(|expiration| expiration <= now) {
|
||||
return Err(CredentialsError::provider_error(std::io::Error::other(EXPIRED_REMOTE_TARGET_CREDENTIALS)));
|
||||
}
|
||||
Ok(self.credentials.clone())
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for RemoteTargetCredentialsProvider {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("RemoteTargetCredentialsProvider")
|
||||
.field("temporary", &self.credentials.session_token().is_some())
|
||||
.field("expiration", &self.credentials.expiry())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProvideCredentials for RemoteTargetCredentialsProvider {
|
||||
fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a>
|
||||
where
|
||||
Self: 'a,
|
||||
{
|
||||
future::ProvideCredentials::ready(self.resolve_at(SystemTime::now()))
|
||||
}
|
||||
|
||||
fn fallback_on_interrupt(&self) -> Option<SdkCredentials> {
|
||||
self.resolve_at(SystemTime::now()).ok()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: SystemTime) -> Result<SdkCredentials, &'static str> {
|
||||
if credentials.expiration.is_some() && credentials.session_token.is_none() {
|
||||
return Err("remote target credential expiration requires a session token");
|
||||
}
|
||||
if credentials.expiration.is_some_and(|expiration| expiration <= now) {
|
||||
return Err(EXPIRED_REMOTE_TARGET_CREDENTIALS);
|
||||
}
|
||||
|
||||
let mut builder = SdkCredentials::builder()
|
||||
.access_key_id(credentials.access_key.clone())
|
||||
.secret_access_key(credentials.secret_key.clone())
|
||||
.account_id(credentials.account_id.clone())
|
||||
.provider_name("bucket_target_sys");
|
||||
if let Some(session_token) = &credentials.session_token {
|
||||
builder = builder.session_token(session_token.clone());
|
||||
}
|
||||
if let Some(expiration) = credentials.expiration {
|
||||
builder = builder.expiry(expiration);
|
||||
}
|
||||
Ok(builder.build())
|
||||
}
|
||||
|
||||
/// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
|
||||
/// signing: SigV4 excludes `user-agent` from the canonical request, so the
|
||||
/// signature stays valid.
|
||||
#[derive(Debug)]
|
||||
struct UserAgentSuffixInterceptor {
|
||||
suffix: &'static str,
|
||||
}
|
||||
|
||||
impl Intercept for UserAgentSuffixInterceptor {
|
||||
fn name(&self) -> &'static str {
|
||||
"RustfsUserAgentSuffix"
|
||||
}
|
||||
|
||||
fn modify_before_transmit(
|
||||
&self,
|
||||
context: &mut BeforeTransmitInterceptorContextMut<'_>,
|
||||
_runtime_components: &RuntimeComponents,
|
||||
_cfg: &mut ConfigBag,
|
||||
) -> Result<(), BoxError> {
|
||||
let headers = context.request_mut().headers_mut();
|
||||
let user_agent = match headers.get(http::header::USER_AGENT.as_str()) {
|
||||
Some(existing) => format!("{existing} {}", self.suffix),
|
||||
None => self.suffix.to_string(),
|
||||
};
|
||||
headers.try_insert(http::header::USER_AGENT.as_str(), user_agent)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the SDK config for `spec` without finalizing it, so callers can add
|
||||
/// interceptors or (in tests) swap the HTTP client before `build()`.
|
||||
pub(crate) async fn build_remote_s3_config(
|
||||
spec: &RemoteS3EndpointSpec,
|
||||
) -> Result<aws_sdk_s3::config::Builder, RemoteS3ClientError> {
|
||||
let Some(credentials) = &spec.credentials else {
|
||||
return Err(RemoteS3ClientError::MissingCredentials);
|
||||
};
|
||||
let creds = remote_sdk_credentials(credentials, SystemTime::now()).map_err(RemoteS3ClientError::Credentials)?;
|
||||
|
||||
let endpoint = spec.endpoint_url();
|
||||
let parsed_endpoint = Url::parse(&endpoint).map_err(|err| RemoteS3ClientError::InvalidEndpoint(err.to_string()))?;
|
||||
validate_remote_endpoint(&parsed_endpoint).map_err(RemoteS3ClientError::EndpointNotAllowed)?;
|
||||
|
||||
let mut config_builder = S3Config::builder()
|
||||
.endpoint_url(endpoint)
|
||||
.credentials_provider(SharedCredentialsProvider::new(RemoteTargetCredentialsProvider { credentials: creds }))
|
||||
.region(SdkRegion::new(spec.region.clone()))
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.request_checksum_calculation(replication_request_checksum_calculation());
|
||||
|
||||
if spec.path_style.force_path_style() {
|
||||
config_builder = config_builder.force_path_style(true);
|
||||
}
|
||||
|
||||
if let Some(http_client) = build_aws_s3_http_client_for_spec(spec).await? {
|
||||
config_builder = config_builder.http_client(http_client);
|
||||
}
|
||||
|
||||
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
|
||||
let mut timeouts = TimeoutConfig::builder();
|
||||
if let Some(connect_timeout) = spec.connect_timeout {
|
||||
timeouts = timeouts.connect_timeout(connect_timeout);
|
||||
}
|
||||
if let Some(read_timeout) = spec.read_timeout {
|
||||
timeouts = timeouts.read_timeout(read_timeout);
|
||||
}
|
||||
config_builder = config_builder.timeout_config(timeouts.build());
|
||||
}
|
||||
|
||||
if !spec.user_agent_suffix.is_empty() {
|
||||
config_builder = config_builder.interceptor(UserAgentSuffixInterceptor {
|
||||
suffix: spec.user_agent_suffix,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(config_builder)
|
||||
}
|
||||
|
||||
/// Builds an `aws_sdk_s3::Client` for `spec`, applying the outbound endpoint
|
||||
/// gate, credential validation and the TLS transport selection.
|
||||
pub async fn build_remote_s3_client(spec: &RemoteS3EndpointSpec) -> Result<S3Client, RemoteS3ClientError> {
|
||||
Ok(S3Client::from_conf(build_remote_s3_config(spec).await?.build()))
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct AcceptAnyServerCertVerifier;
|
||||
|
||||
impl rustls::client::danger::ServerCertVerifier for AcceptAnyServerCertVerifier {
|
||||
fn verify_server_cert(
|
||||
&self,
|
||||
_end_entity: &rustls_pki_types::CertificateDer<'_>,
|
||||
_intermediates: &[rustls_pki_types::CertificateDer<'_>],
|
||||
_server_name: &rustls_pki_types::ServerName<'_>,
|
||||
_ocsp_response: &[u8],
|
||||
_now: rustls_pki_types::UnixTime,
|
||||
) -> Result<rustls::client::danger::ServerCertVerified, rustls::Error> {
|
||||
Ok(rustls::client::danger::ServerCertVerified::assertion())
|
||||
}
|
||||
|
||||
fn verify_tls12_signature(
|
||||
&self,
|
||||
_message: &[u8],
|
||||
_cert: &rustls_pki_types::CertificateDer<'_>,
|
||||
_dss: &rustls::DigitallySignedStruct,
|
||||
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
|
||||
Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
|
||||
}
|
||||
|
||||
fn verify_tls13_signature(
|
||||
&self,
|
||||
_message: &[u8],
|
||||
_cert: &rustls_pki_types::CertificateDer<'_>,
|
||||
_dss: &rustls::DigitallySignedStruct,
|
||||
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
|
||||
Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
|
||||
}
|
||||
|
||||
fn supported_verify_schemes(&self) -> Vec<rustls::SignatureScheme> {
|
||||
rustls::crypto::aws_lc_rs::default_provider()
|
||||
.signature_verification_algorithms
|
||||
.supported_schemes()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TargetHyperHttpConnector<C> {
|
||||
client: HyperClient<C, SdkBody>,
|
||||
}
|
||||
|
||||
impl<C> fmt::Debug for TargetHyperHttpConnector<C> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("TargetHyperHttpConnector")
|
||||
.field("client", &"** hyper client **")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl<C> SmithyHttpConnector for TargetHyperHttpConnector<C>
|
||||
where
|
||||
C: Clone + Send + Sync + 'static,
|
||||
C: Service<Uri>,
|
||||
C::Response:
|
||||
hyper::rt::Read + hyper::rt::Write + hyper_util::client::legacy::connect::Connection + Send + Sync + Unpin + 'static,
|
||||
C::Future: Unpin + Send + 'static,
|
||||
C::Error: Into<BoxError>,
|
||||
{
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
let request = match request.try_into_http1x() {
|
||||
Ok(request) => request,
|
||||
Err(err) => return HttpConnectorFuture::ready(Err(ConnectorError::user(err.into()))),
|
||||
};
|
||||
|
||||
let mut client = self.client.clone();
|
||||
let fut = client.call(request);
|
||||
HttpConnectorFuture::new(async move {
|
||||
let response = fut
|
||||
.await
|
||||
.map_err(|err| ConnectorError::io(err.into()))?
|
||||
.map(SdkBody::from_body_1_x);
|
||||
HttpResponse::try_from(response).map_err(|err| ConnectorError::other(err.into(), None))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_rustls_crypto_provider() {
|
||||
if rustls::crypto::CryptoProvider::get_default().is_none() {
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
}
|
||||
}
|
||||
|
||||
/// Env opt-in that re-enables loopback replication targets. Loopback (`127.0.0.1`,
|
||||
/// `::1`, `localhost`) is a classic SSRF vector and stays rejected by default, but
|
||||
/// single-host multi-instance dev setups and the e2e harness legitimately replicate
|
||||
/// over loopback. Never set this in production.
|
||||
const ALLOW_LOOPBACK_REPLICATION_TARGET_ENV: &str = "RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET";
|
||||
|
||||
fn loopback_replication_targets_allowed() -> bool {
|
||||
std::env::var(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV)
|
||||
.map(|v| v.eq_ignore_ascii_case("true") || v == "1")
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
const REPLICATION_STREAMING_CHECKSUMS_ENV: &str = "RUSTFS_REPLICATION_STREAMING_CHECKSUMS";
|
||||
|
||||
/// Streaming trailer checksums make the SDK frame request bodies as
|
||||
/// `aws-chunked`; a target that does not decode that framing stores the frames
|
||||
/// verbatim, silently corrupting every replica while the transfer itself
|
||||
/// succeeds (#6853). Plain signed payloads are the compatible default; the env
|
||||
/// knob restores trailer checksums for fleets whose targets are all known to
|
||||
/// decode them.
|
||||
pub(crate) fn replication_request_checksum_calculation() -> RequestChecksumCalculation {
|
||||
if std::env::var(REPLICATION_STREAMING_CHECKSUMS_ENV)
|
||||
.map(|v| v.eq_ignore_ascii_case("true") || v == "1")
|
||||
.unwrap_or(false)
|
||||
{
|
||||
RequestChecksumCalculation::WhenSupported
|
||||
} else {
|
||||
RequestChecksumCalculation::WhenRequired
|
||||
}
|
||||
}
|
||||
|
||||
/// Outbound gate for operator-configured remote endpoints (replication
|
||||
/// targets, on-demand migration sources). See
|
||||
/// `docs/operations/outbound-connection-policy.md`.
|
||||
pub fn validate_remote_endpoint(url: &Url) -> Result<(), OutboundUrlError> {
|
||||
validate_remote_endpoint_inner(url, loopback_replication_targets_allowed())
|
||||
}
|
||||
|
||||
pub(crate) fn validate_remote_endpoint_inner(url: &Url, allow_loopback: bool) -> Result<(), OutboundUrlError> {
|
||||
match validate_outbound_url(url) {
|
||||
Ok(()) => Ok(()),
|
||||
// Replication targets are trusted infrastructure the operator configures, and
|
||||
// legitimately live on private networks, so private addresses are always allowed.
|
||||
Err(OutboundUrlError::ForbiddenHost {
|
||||
reason: "private address",
|
||||
..
|
||||
}) => Ok(()),
|
||||
// Loopback is far higher SSRF risk, so it is allowed only under the explicit,
|
||||
// off-by-default opt-in above (single-host multi-instance / the e2e harness).
|
||||
Err(OutboundUrlError::ForbiddenHost {
|
||||
reason: "loopback address" | "loopback host",
|
||||
..
|
||||
}) if allow_loopback => Ok(()),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_insecure_aws_s3_http_client() -> SharedHttpClient {
|
||||
ensure_rustls_crypto_provider();
|
||||
|
||||
let tls_config = rustls::ClientConfig::builder()
|
||||
.dangerous()
|
||||
.with_custom_certificate_verifier(Arc::new(AcceptAnyServerCertVerifier))
|
||||
.with_no_client_auth();
|
||||
|
||||
let https = hyper_rustls::HttpsConnectorBuilder::new()
|
||||
.with_tls_config(tls_config)
|
||||
.https_or_http()
|
||||
.enable_http1()
|
||||
.enable_http2()
|
||||
.build();
|
||||
let mut client_builder = HyperClient::builder(TokioExecutor::new());
|
||||
client_builder.pool_timer(TokioTimer::new());
|
||||
let client = client_builder.build(https);
|
||||
let connector = SharedHttpConnector::new(TargetHyperHttpConnector { client });
|
||||
|
||||
http_client_fn(move |_settings, _components| connector.clone())
|
||||
}
|
||||
|
||||
fn validate_ca_pem_bundle(ca_cert_pem: &[u8]) -> Result<(), String> {
|
||||
let certs = rustls_pki_types::CertificateDer::pem_slice_iter(ca_cert_pem)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|err| format!("invalid PEM encoding: {err}"))?;
|
||||
|
||||
if certs.is_empty() {
|
||||
return Err("no certificates found".to_string());
|
||||
}
|
||||
|
||||
// Smithy's rustls adapter defers parsing custom certificates and assumes
|
||||
// they are valid when the HTTPS connector is built. Validate every DER
|
||||
// certificate first so malformed configuration is reported rather than
|
||||
// reaching an `expect` in the dependency.
|
||||
let mut validation_store = rustls::RootCertStore::empty();
|
||||
for cert in certs {
|
||||
validation_store
|
||||
.add(cert)
|
||||
.map_err(|err| format!("invalid X.509 certificate: {err}"))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), RemoteS3ClientError> {
|
||||
validate_ca_pem_bundle(ca_cert_pem.as_bytes()).map_err(RemoteS3ClientError::InvalidCaPem)
|
||||
}
|
||||
|
||||
pub(crate) fn compose_replication_trust_store(
|
||||
certificate_bundles: impl IntoIterator<Item = Vec<u8>>,
|
||||
) -> (smithy_tls::TrustStore, usize) {
|
||||
// `TrustStore::default()` keeps the platform-native roots enabled. Target
|
||||
// and RUSTFS_TLS_PATH certificates extend that baseline instead of
|
||||
// replacing it with a target-specific trust island.
|
||||
let mut trust_store = smithy_tls::TrustStore::default();
|
||||
let mut custom_bundle_count = 0;
|
||||
for pem in certificate_bundles {
|
||||
trust_store.add_pem_certificate(pem);
|
||||
custom_bundle_count += 1;
|
||||
}
|
||||
|
||||
(trust_store, custom_bundle_count)
|
||||
}
|
||||
|
||||
pub(crate) fn build_aws_s3_http_client_with_trust_store(
|
||||
trust_store: smithy_tls::TrustStore,
|
||||
) -> Result<SharedHttpClient, RemoteS3ClientError> {
|
||||
let tls_context = smithy_tls::TlsContext::builder()
|
||||
.with_trust_store(trust_store)
|
||||
.build()
|
||||
.map_err(|err| RemoteS3ClientError::InvalidCaPem(err.to_string()))?;
|
||||
|
||||
Ok(SmithyHttpClientBuilder::new()
|
||||
.tls_provider(smithy_tls::Provider::rustls(smithy_tls::rustls_provider::CryptoMode::AwsLc))
|
||||
.tls_context(tls_context)
|
||||
.build_https())
|
||||
}
|
||||
|
||||
pub(crate) async fn load_tls_path_ca_bundles(tls_dir: &Path, trust_leaf_cert_as_ca: bool) -> Vec<Vec<u8>> {
|
||||
let mut certificate_bundles = Vec::new();
|
||||
|
||||
let ca_path = tls_dir.join(RUSTFS_CA_CERT);
|
||||
match tokio::fs::read(&ca_path).await {
|
||||
Ok(pem) => match validate_ca_pem_bundle(&pem) {
|
||||
Ok(()) => certificate_bundles.push(pem),
|
||||
Err(err) => warn!("ignoring invalid custom CA bundle {:?} for replication client: {}", ca_path, err),
|
||||
},
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(e) => warn!("failed to read custom CA bundle {:?} for replication client: {}", ca_path, e),
|
||||
}
|
||||
|
||||
if trust_leaf_cert_as_ca {
|
||||
let leaf_cert_path = tls_dir.join(RUSTFS_TLS_CERT);
|
||||
match tokio::fs::read(&leaf_cert_path).await {
|
||||
Ok(pem) => match validate_ca_pem_bundle(&pem) {
|
||||
Ok(()) => certificate_bundles.push(pem),
|
||||
Err(err) => warn!(
|
||||
"ignoring invalid leaf certificate {:?} for replication client trust store: {}",
|
||||
leaf_cert_path, err
|
||||
),
|
||||
},
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(e) => warn!("failed to read leaf cert {:?} for replication client trust store: {}", leaf_cert_path, e),
|
||||
}
|
||||
}
|
||||
|
||||
certificate_bundles
|
||||
}
|
||||
|
||||
async fn load_configured_tls_ca_bundles() -> Vec<Vec<u8>> {
|
||||
let tls_path = rustfs_utils::get_env_str(rustfs_config::ENV_RUSTFS_TLS_PATH, rustfs_config::DEFAULT_RUSTFS_TLS_PATH);
|
||||
if tls_path.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
load_tls_path_ca_bundles(
|
||||
Path::new(&tls_path),
|
||||
rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn build_aws_s3_http_client_from_target_ca_pem(
|
||||
ca_cert_pem: &str,
|
||||
) -> Result<SharedHttpClient, RemoteS3ClientError> {
|
||||
validate_target_ca_pem(ca_cert_pem)?;
|
||||
|
||||
let mut certificate_bundles = load_configured_tls_ca_bundles().await;
|
||||
certificate_bundles.push(ca_cert_pem.as_bytes().to_vec());
|
||||
let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
|
||||
|
||||
build_aws_s3_http_client_with_trust_store(trust_store)
|
||||
}
|
||||
|
||||
/// Selects the HTTP client for `spec`: `None` keeps the SDK default (plain
|
||||
/// HTTP, or HTTPS with platform roots when no custom trust is configured).
|
||||
pub(crate) async fn build_aws_s3_http_client_for_spec(
|
||||
spec: &RemoteS3EndpointSpec,
|
||||
) -> Result<Option<SharedHttpClient>, RemoteS3ClientError> {
|
||||
if !spec.secure {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if spec.skip_tls_verify {
|
||||
return Ok(Some(build_insecure_aws_s3_http_client()));
|
||||
}
|
||||
|
||||
if let Some(ca_cert_pem) = spec.custom_ca_pem() {
|
||||
return build_aws_s3_http_client_from_target_ca_pem(ca_cert_pem).await.map(Some);
|
||||
}
|
||||
|
||||
Ok(build_aws_s3_http_client_from_tls_path().await)
|
||||
}
|
||||
|
||||
async fn build_aws_s3_http_client_from_tls_path() -> Option<SharedHttpClient> {
|
||||
let certificate_bundles = load_configured_tls_ca_bundles().await;
|
||||
if certificate_bundles.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let (trust_store, _) = compose_replication_trust_store(certificate_bundles);
|
||||
match build_aws_s3_http_client_with_trust_store(trust_store) {
|
||||
Ok(client) => Some(client),
|
||||
Err(e) => {
|
||||
warn!("failed to build AWS SDK TLS context for replication client: {}", e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aws_smithy_runtime_api::http::StatusCode as SmithyStatusCode;
|
||||
use std::sync::Mutex;
|
||||
|
||||
fn spec(endpoint: &str, secure: bool) -> RemoteS3EndpointSpec {
|
||||
RemoteS3EndpointSpec {
|
||||
endpoint: endpoint.to_string(),
|
||||
secure,
|
||||
region: "us-east-1".to_string(),
|
||||
path_style: PathStyle::Auto,
|
||||
credentials: Some(RemoteCredentials {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
session_token: None,
|
||||
expiration: None,
|
||||
account_id: String::new(),
|
||||
}),
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
connect_timeout: None,
|
||||
read_timeout: None,
|
||||
user_agent_suffix: "",
|
||||
}
|
||||
}
|
||||
|
||||
type RecordedHeaders = Arc<Mutex<Vec<Vec<(String, String)>>>>;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct RecordingHeaderConnector {
|
||||
request_headers: RecordedHeaders,
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for RecordingHeaderConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
self.request_headers
|
||||
.lock()
|
||||
.expect("recorded header lock should not be poisoned")
|
||||
.push(
|
||||
request
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||
.collect(),
|
||||
);
|
||||
HttpConnectorFuture::ready(Ok(HttpResponse::new(
|
||||
SmithyStatusCode::try_from(200_u16).expect("200 should be a valid response status"),
|
||||
SdkBody::empty(),
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_remote_s3_client_rejects_loopback_and_metadata_endpoints() {
|
||||
// Default (no loopback opt-in): loopback in IPv4, IPv6 and hostname
|
||||
// forms plus the metadata endpoint all return the typed gate error.
|
||||
for endpoint in ["127.0.0.1:9000", "[::1]:9000", "localhost:9000", "169.254.169.254"] {
|
||||
let err = build_remote_s3_client(&spec(endpoint, false))
|
||||
.await
|
||||
.err()
|
||||
.unwrap_or_else(|| panic!("{endpoint} must be rejected by the outbound gate"));
|
||||
assert!(
|
||||
matches!(err, RemoteS3ClientError::EndpointNotAllowed(OutboundUrlError::ForbiddenHost { .. })),
|
||||
"{endpoint}: unexpected error {err:?}"
|
||||
);
|
||||
assert!(err.to_string().contains("not allowed"), "{endpoint}: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_remote_s3_client_allows_private_and_public_endpoints() {
|
||||
for endpoint in ["10.0.0.1:9000", "192.168.1.20", "s3.example.com"] {
|
||||
build_remote_s3_client(&spec(endpoint, false))
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("{endpoint} should be allowed: {err}"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_remote_s3_client_requires_credentials() {
|
||||
let mut spec = spec("s3.example.com", true);
|
||||
spec.credentials = None;
|
||||
let err = build_remote_s3_client(&spec)
|
||||
.await
|
||||
.expect_err("missing credentials must be a typed error");
|
||||
assert!(matches!(err, RemoteS3ClientError::MissingCredentials));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_remote_s3_client_rejects_expiration_without_session_token() {
|
||||
let mut spec = spec("s3.example.com", true);
|
||||
spec.credentials
|
||||
.as_mut()
|
||||
.expect("spec fixture carries credentials")
|
||||
.expiration = Some(SystemTime::now() + Duration::from_secs(3_600));
|
||||
let err = build_remote_s3_client(&spec)
|
||||
.await
|
||||
.expect_err("expiration without session token must be rejected");
|
||||
assert_eq!(err.to_string(), "remote target credential expiration requires a session token");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_remote_s3_client_rejects_invalid_custom_ca_pem() {
|
||||
let mut spec = spec("192.168.1.10:9000", true);
|
||||
spec.ca_cert_pem = Some("not a pem".to_string());
|
||||
let err = build_remote_s3_client(&spec)
|
||||
.await
|
||||
.expect_err("invalid custom CA PEM must be rejected");
|
||||
assert!(matches!(err, RemoteS3ClientError::InvalidCaPem(_)));
|
||||
assert!(err.to_string().contains("invalid target CA PEM"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_style_auto_and_path_force_path_style() {
|
||||
assert!(PathStyle::Auto.force_path_style());
|
||||
assert!(PathStyle::Path.force_path_style());
|
||||
assert!(!PathStyle::VirtualHost.force_path_style());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_credentials_debug_redacts_secrets() {
|
||||
let credentials = RemoteCredentials {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "very-secret".to_string(),
|
||||
session_token: Some("session-token".to_string()),
|
||||
expiration: None,
|
||||
account_id: String::new(),
|
||||
};
|
||||
let rendered = format!("{credentials:?}");
|
||||
assert!(rendered.contains("access"));
|
||||
assert!(!rendered.contains("very-secret"));
|
||||
assert!(!rendered.contains("session-token"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn user_agent_suffix_is_appended_after_signing() {
|
||||
let request_headers: RecordedHeaders = Arc::new(Mutex::new(Vec::new()));
|
||||
let connector = SharedHttpConnector::new(RecordingHeaderConnector {
|
||||
request_headers: Arc::clone(&request_headers),
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
|
||||
let mut spec = spec("s3.example.com", true);
|
||||
spec.user_agent_suffix = "RustFS-Test/0.0";
|
||||
spec.connect_timeout = Some(Duration::from_secs(5));
|
||||
spec.read_timeout = Some(Duration::from_secs(5));
|
||||
let config = build_remote_s3_config(&spec)
|
||||
.await
|
||||
.expect("spec should build")
|
||||
.http_client(http_client)
|
||||
.build();
|
||||
S3Client::from_conf(config)
|
||||
.head_bucket()
|
||||
.bucket("bucket")
|
||||
.send()
|
||||
.await
|
||||
.expect("recording connector should accept the request");
|
||||
|
||||
let recorded = request_headers.lock().expect("recorded header lock should not be poisoned");
|
||||
let headers = &recorded[0];
|
||||
let user_agent = headers
|
||||
.iter()
|
||||
.find(|(k, _)| k.eq_ignore_ascii_case("user-agent"))
|
||||
.map(|(_, v)| v.as_str())
|
||||
.expect("SDK request must carry a user-agent");
|
||||
assert!(user_agent.ends_with(" RustFS-Test/0.0"), "user-agent was {user_agent}");
|
||||
assert!(user_agent.starts_with("aws-sdk-rust/"), "SDK identity must be preserved: {user_agent}");
|
||||
assert!(
|
||||
headers.iter().any(|(k, _)| k.eq_ignore_ascii_case("authorization")),
|
||||
"request must still be signed"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,6 @@ pub(crate) use rustfs_replication::{
|
||||
delete_marker_purge_version_id, delete_replication_creates_marker, delete_replication_missing_source_decision,
|
||||
delete_replication_object_opts, heal_uses_delete_replication_path, is_object_lock_denied_delete,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error,
|
||||
resync_existing_delete_replication_info, resync_target_for_object, should_retry_delete_marker_purge,
|
||||
single_part_replica_etag_mismatch, target_delete_version_id,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_existing_delete_replication_info,
|
||||
resync_target_for_object, should_retry_delete_marker_purge, single_part_replica_etag_mismatch, target_delete_version_id,
|
||||
};
|
||||
|
||||
@@ -32,9 +32,8 @@ use super::replication_object_decision_boundary::{
|
||||
MustReplicateOptions, ReplicationMultipartPartInput, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
|
||||
delete_replication_creates_marker, heal_uses_delete_replication_path, is_object_lock_denied_delete,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, replication_single_put_size_error,
|
||||
resync_existing_delete_replication_info, should_retry_delete_marker_purge, single_part_replica_etag_mismatch,
|
||||
target_delete_version_id,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_existing_delete_replication_info,
|
||||
should_retry_delete_marker_purge, single_part_replica_etag_mismatch, target_delete_version_id,
|
||||
};
|
||||
use super::replication_queue_boundary::{DeletedObjectReplicationInfo, ReplicationQueueAdmission};
|
||||
use super::replication_resync_boundary::ResyncStatusType;
|
||||
@@ -89,7 +88,6 @@ use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, LazyLock, Mutex as StdMutex};
|
||||
use std::time::Instant;
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
use tokio::io::AsyncRead;
|
||||
@@ -120,7 +118,6 @@ const EVENT_DELETE_MARKER_PURGE_FAILED: &str = "replication_delete_marker_purge_
|
||||
const EVENT_DELETE_MARKER_PURGE_MRF: &str = "replication_delete_marker_purge_mrf";
|
||||
const METRIC_DELETE_MARKER_PURGE_TOTAL: &str = "rustfs_replication_delete_marker_purge_total";
|
||||
const EVENT_REPLICATION_VERSION_IDENTITY_DRIFT: &str = "replication_version_identity_drift";
|
||||
const EVENT_REPLICATION_OBJECT_FAILED: &str = "replication_object_failed";
|
||||
const EVENT_REPLICATION_PURGE_OBJECT_LOCK_DENIED: &str = "replication_purge_object_lock_denied";
|
||||
|
||||
#[allow(
|
||||
@@ -193,19 +190,11 @@ fn metadata_requires_existing_target(op_type: ReplicationType, object_info: &Obj
|
||||
|
||||
const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_identity_drift_total";
|
||||
|
||||
/// How long a target stays quiet after reporting version-identity drift.
|
||||
///
|
||||
/// This used to be a plain "once per ARN per process": one line ever, which on
|
||||
/// a long-lived server meant the single most important diagnostic for a
|
||||
/// non-converging generic S3 target scrolled away hours before anyone looked
|
||||
/// (rustfs#6822). Re-arming on an interval keeps the log bounded while leaving
|
||||
/// the condition discoverable in any recent window.
|
||||
const VERSION_IDENTITY_DRIFT_LOG_INTERVAL: TokioDuration = TokioDuration::from_secs(600);
|
||||
|
||||
/// When each target last reported version-identity drift, by ARN. Throttling is
|
||||
/// advisory only — the metric still counts every drifting PUT.
|
||||
static VERSION_IDENTITY_WARNED_ARNS: LazyLock<StdMutex<HashMap<String, Instant>>> =
|
||||
LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
/// Targets that already produced a version-identity-drift warning this
|
||||
/// process lifetime, by ARN. Deduping is advisory only (the metric still
|
||||
/// counts every drifting PUT), so a reconfigured target re-warning only
|
||||
/// after a restart is acceptable.
|
||||
static VERSION_IDENTITY_WARNED_ARNS: LazyLock<StdMutex<HashSet<String>>> = LazyLock::new(|| StdMutex::new(HashSet::new()));
|
||||
|
||||
/// Version purges the peer denied under object lock (#6850). A RustFS peer
|
||||
/// with the replicated-purge GOVERNANCE exemption
|
||||
@@ -333,39 +322,20 @@ fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: &
|
||||
return;
|
||||
}
|
||||
counter!(METRIC_VERSION_IDENTITY_DRIFT_TOTAL).increment(1);
|
||||
if !version_identity_drift_log_due(&tgt_client.arn, Instant::now()) {
|
||||
return;
|
||||
}
|
||||
// `error`, not `warn`: the target silently refuses the addressing scheme
|
||||
// every version-addressed delete and heal on it depends on, so replication
|
||||
// to it can never converge. At `warn` this sat below `DEFAULT_LOG_LEVEL`
|
||||
// and no default deployment ever saw the one line that explains why a
|
||||
// purged version is still on the target (rustfs#6822).
|
||||
error!(
|
||||
event = EVENT_REPLICATION_VERSION_IDENTITY_DRIFT,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
arn = %tgt_client.arn,
|
||||
endpoint = %tgt_client.endpoint,
|
||||
sent_version_id = %source_version_id,
|
||||
assigned_version_id = assigned_version_id.unwrap_or("<none>"),
|
||||
"Replication target does not adopt source version ids; version-addressed replication cannot converge (run ?replication-check for details)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Whether this ARN's version-identity drift is due to be logged again at
|
||||
/// `now`, re-arming the throttle when it is. Split out from the audit so the
|
||||
/// interval policy is testable without a target client.
|
||||
fn version_identity_drift_log_due(arn: &str, now: Instant) -> bool {
|
||||
let mut warned = VERSION_IDENTITY_WARNED_ARNS
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
match warned.get(arn) {
|
||||
Some(last) if now.duration_since(*last) < VERSION_IDENTITY_DRIFT_LOG_INTERVAL => false,
|
||||
_ => {
|
||||
warned.insert(arn.to_string(), now);
|
||||
true
|
||||
}
|
||||
if warned.insert(tgt_client.arn.clone()) {
|
||||
warn!(
|
||||
event = EVENT_REPLICATION_VERSION_IDENTITY_DRIFT,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
arn = %tgt_client.arn,
|
||||
endpoint = %tgt_client.endpoint,
|
||||
sent_version_id = %source_version_id,
|
||||
assigned_version_id = assigned_version_id.unwrap_or("<none>"),
|
||||
"Replication target does not adopt source version ids; version-addressed replication cannot converge (run ?replication-check for details)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2080,13 +2050,10 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
}
|
||||
}
|
||||
|
||||
let delete_version_id = dobj.delete_object.version_id.map(|v| v.to_string());
|
||||
note_replication_terminal_failure(&bucket, &dobj.delete_object.object_name, delete_version_id.as_deref(), &rinfos);
|
||||
|
||||
let mut drs = get_replication_state(
|
||||
&rinfos,
|
||||
&dobj.delete_object.replication_state.clone().unwrap_or_default(),
|
||||
delete_version_id,
|
||||
dobj.delete_object.version_id.map(|v| v.to_string()),
|
||||
);
|
||||
if replication_status != prev_status {
|
||||
drs.replication_timestamp = Some(OffsetDateTime::now_utc());
|
||||
@@ -3056,11 +3023,8 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
}
|
||||
}
|
||||
|
||||
let version_id = roi.version_id.map(|v| v.to_string());
|
||||
note_replication_terminal_failure(&bucket, &object, version_id.as_deref(), &rinfos);
|
||||
|
||||
let previous_state = roi.replication_state.clone().unwrap_or_default();
|
||||
let merged_state = get_replication_state(&rinfos, &previous_state, version_id);
|
||||
let merged_state = get_replication_state(&rinfos, &previous_state, roi.version_id.map(|v| v.to_string()));
|
||||
let replication_status = merged_state.composite_replication_status();
|
||||
let new_replication_internal = merged_state.replication_status_internal.clone();
|
||||
let mut object_info = roi.to_object_info();
|
||||
@@ -3137,61 +3101,6 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
(merged_state, state_persisted)
|
||||
}
|
||||
|
||||
/// Emit the operator-visible record of a replication attempt that ended FAILED.
|
||||
///
|
||||
/// Every per-branch failure log in this module is deliberately quieter than
|
||||
/// `error`: most of them sit on the replication hot path and fire once per
|
||||
/// object *per ARN*, so a target that stays unreachable would flood the log
|
||||
/// from inside the transfer loop. That left a hole customers fell into
|
||||
/// (rustfs#6825): `DEFAULT_LOG_LEVEL` is `error`, so on a stock deployment a
|
||||
/// failed object produced no line at all, and an operator staring at a replica
|
||||
/// that never arrived had nothing to correlate — the same trap already
|
||||
/// documented for the GET path in
|
||||
/// `crates/e2e_test/src/get_stream_failure_observability_test.rs`.
|
||||
///
|
||||
/// This is the one place that knows an object reached a *terminal* FAILED state
|
||||
/// for a target, so this is where the guaranteed-visible line belongs. It is
|
||||
/// bounded by the number of objects that actually fail rather than by attempts
|
||||
/// inside a transfer, and it carries the target's own error so a remote
|
||||
/// rejection is diagnosable without the operator first having to lower the
|
||||
/// global log level and reproduce.
|
||||
fn note_replication_terminal_failure(bucket: &str, object: &str, version_id: Option<&str>, rinfos: &ReplicatedInfos) {
|
||||
for target in rinfos.targets.iter() {
|
||||
if target.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let replication_failed = target.replication_status == ReplicationStatusType::Failed;
|
||||
let purge_failed = target.version_purge_status == VersionPurgeStatusType::Failed;
|
||||
if !replication_failed && !purge_failed {
|
||||
continue;
|
||||
}
|
||||
|
||||
error!(
|
||||
event = EVENT_REPLICATION_OBJECT_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
version_id = version_id.unwrap_or("-"),
|
||||
arn = %target.arn,
|
||||
endpoint = %target.endpoint,
|
||||
op_type = %target.op_type,
|
||||
size = target.size,
|
||||
replication_status = %target.replication_status.as_str(),
|
||||
version_purge_status = %target.version_purge_status.as_str(),
|
||||
// The target's error can carry a signed URL or an echoed auth
|
||||
// header, so it goes through the same redaction as the persisted
|
||||
// resync detail rather than straight into the log.
|
||||
error = %target
|
||||
.error
|
||||
.as_deref()
|
||||
.and_then(sanitize_resync_error_detail)
|
||||
.unwrap_or_else(|| "<none>".to_string()),
|
||||
"Replication failed for object"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn unavailable_object_target_info(roi: &ReplicateObjectInfo, arn: &str) -> ReplicatedTargetInfo {
|
||||
ReplicatedTargetInfo {
|
||||
arn: arn.to_string(),
|
||||
@@ -3491,33 +3400,6 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(reason) = replication_single_put_size_error(is_multipart, transfer_size) {
|
||||
drop(gr);
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(reason.clone());
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
target_bucket = %tgt_client.bucket,
|
||||
arn = %tgt_client.arn,
|
||||
object = %object,
|
||||
operation = "put_object",
|
||||
transfer_size = transfer_size,
|
||||
error = %reason,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
if let Some(err) = if is_multipart {
|
||||
drop(gr);
|
||||
let result = replicate_object_with_multipart(MultipartReplicationContext {
|
||||
@@ -4186,14 +4068,6 @@ async fn replicate_all_payload_to_target<S: ReplicationObjectIO>(
|
||||
ctx: ReplicateAllPayloadContext<'_, S>,
|
||||
mut gr: GetObjectReader,
|
||||
) -> Option<std::io::Error> {
|
||||
// Fail before streaming a body the target is required to reject: an S3
|
||||
// PutObject caps at 5 GiB, and this route is chosen by the source object's
|
||||
// storage shape rather than its size (rustfs#6825).
|
||||
if let Some(reason) = replication_single_put_size_error(ctx.is_multipart, ctx.transfer_size) {
|
||||
drop(gr);
|
||||
return Some(std::io::Error::other(reason));
|
||||
}
|
||||
|
||||
if ctx.is_multipart {
|
||||
drop(gr);
|
||||
let result = replicate_object_with_multipart(MultipartReplicationContext {
|
||||
@@ -5842,207 +5716,4 @@ mod tests {
|
||||
assert!(!retry_scheduled.load(Ordering::SeqCst));
|
||||
assert_eq!(result.unwrap_err().to_string(), "transfer failed");
|
||||
}
|
||||
|
||||
/// A replication target's terminal outcome, as the operator sees it.
|
||||
fn failed_target(arn: &str, error: &str) -> ReplicatedTargetInfo {
|
||||
ReplicatedTargetInfo {
|
||||
arn: arn.to_string(),
|
||||
size: 6 * 1024 * 1024 * 1024,
|
||||
op_type: ReplicationType::Object,
|
||||
replication_status: ReplicationStatusType::Failed,
|
||||
endpoint: "s3.wasabisys.com".to_string(),
|
||||
error: Some(error.to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Capture the log this module writes, filtered exactly the way a stock
|
||||
/// deployment filters it.
|
||||
fn logs_at_default_level(emit: impl FnOnce()) -> String {
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
use tracing_subscriber::layer::SubscriberExt;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CapturedLogs {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
struct CapturedLogWriter {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
impl std::io::Write for CapturedLogWriter {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
self.buffer
|
||||
.lock()
|
||||
.expect("captured logs mutex should not be poisoned")
|
||||
.extend_from_slice(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
impl<'a> MakeWriter<'a> for CapturedLogs {
|
||||
type Writer = CapturedLogWriter;
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
CapturedLogWriter {
|
||||
buffer: Arc::clone(&self.buffer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let logs = CapturedLogs::default();
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
// Not a hand-picked level: this is the filter an operator who has
|
||||
// changed nothing is actually running.
|
||||
.with(EnvFilter::new(rustfs_config::DEFAULT_LOG_LEVEL))
|
||||
.with(
|
||||
tracing_subscriber::fmt::layer()
|
||||
.with_writer(logs.clone())
|
||||
.with_ansi(false)
|
||||
.without_time(),
|
||||
);
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
|
||||
emit();
|
||||
|
||||
let buffer = logs
|
||||
.buffer
|
||||
.lock()
|
||||
.expect("captured logs mutex should not be poisoned")
|
||||
.clone();
|
||||
String::from_utf8(buffer).expect("captured logs should be valid UTF-8")
|
||||
}
|
||||
|
||||
/// rustfs#6825: a 6 GiB object never reached the target and the server said
|
||||
/// nothing an operator could act on, because every failure line in this
|
||||
/// module sat below `DEFAULT_LOG_LEVEL`. The object key, the target, and
|
||||
/// the target's own error have to survive the default filter.
|
||||
#[test]
|
||||
fn failed_replication_names_the_object_at_the_default_log_level() {
|
||||
let rinfos = ReplicatedInfos {
|
||||
replication_timestamp: Some(OffsetDateTime::now_utc()),
|
||||
targets: vec![failed_target("arn:replication::wasabi", "put_object failed: EntityTooLarge")],
|
||||
};
|
||||
|
||||
let logs = logs_at_default_level(|| {
|
||||
note_replication_terminal_failure("photos", "backups/vm-image.qcow2", Some("v-9"), &rinfos);
|
||||
});
|
||||
|
||||
assert!(logs.contains("backups/vm-image.qcow2"), "the failed object must be named: {logs}");
|
||||
assert!(logs.contains("arn:replication::wasabi"), "the target must be named: {logs}");
|
||||
assert!(logs.contains("EntityTooLarge"), "the target's own error must survive: {logs}");
|
||||
assert!(logs.contains("v-9"), "the version must be named: {logs}");
|
||||
assert!(logs.contains(EVENT_REPLICATION_OBJECT_FAILED), "the event must be structured: {logs}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn successful_replication_stays_quiet_at_the_default_log_level() {
|
||||
let rinfos = ReplicatedInfos {
|
||||
replication_timestamp: Some(OffsetDateTime::now_utc()),
|
||||
targets: vec![ReplicatedTargetInfo {
|
||||
arn: "arn:replication::wasabi".to_string(),
|
||||
replication_status: ReplicationStatusType::Completed,
|
||||
..Default::default()
|
||||
}],
|
||||
};
|
||||
|
||||
let logs = logs_at_default_level(|| {
|
||||
note_replication_terminal_failure("photos", "backups/ok.bin", None, &rinfos);
|
||||
});
|
||||
|
||||
assert!(logs.is_empty(), "a completed replication must not log an error: {logs}");
|
||||
}
|
||||
|
||||
/// A failed version purge is the 6822 symptom (the version stays on the
|
||||
/// target); it must be as visible as a failed transfer even though the
|
||||
/// replication status itself is not FAILED.
|
||||
#[test]
|
||||
fn failed_version_purge_is_reported_at_the_default_log_level() {
|
||||
let rinfos = ReplicatedInfos {
|
||||
replication_timestamp: Some(OffsetDateTime::now_utc()),
|
||||
targets: vec![ReplicatedTargetInfo {
|
||||
arn: "arn:replication::wasabi".to_string(),
|
||||
op_type: ReplicationType::Delete,
|
||||
replication_status: ReplicationStatusType::Empty,
|
||||
version_purge_status: VersionPurgeStatusType::Failed,
|
||||
error: Some("remove_object failed: NoSuchVersion".to_string()),
|
||||
..Default::default()
|
||||
}],
|
||||
};
|
||||
|
||||
let logs = logs_at_default_level(|| {
|
||||
note_replication_terminal_failure("photos", "backups/purged.bin", Some("v-1"), &rinfos);
|
||||
});
|
||||
|
||||
assert!(logs.contains("backups/purged.bin"), "the purged object must be named: {logs}");
|
||||
assert!(logs.contains("NoSuchVersion"), "the target's own error must survive: {logs}");
|
||||
}
|
||||
|
||||
/// The target's error is echoed remote text and can carry a signed URL or
|
||||
/// an auth header, so it goes through the persisted-detail redaction rather
|
||||
/// than straight into the log.
|
||||
#[test]
|
||||
fn failed_replication_redacts_a_sensitive_target_error() {
|
||||
let rinfos = ReplicatedInfos {
|
||||
replication_timestamp: Some(OffsetDateTime::now_utc()),
|
||||
targets: vec![failed_target(
|
||||
"arn:replication::wasabi",
|
||||
"put_object failed: rejected Authorization: Bearer super-secret",
|
||||
)],
|
||||
};
|
||||
|
||||
let logs = logs_at_default_level(|| {
|
||||
note_replication_terminal_failure("photos", "backups/vm-image.qcow2", None, &rinfos);
|
||||
});
|
||||
|
||||
assert!(logs.contains("backups/vm-image.qcow2"), "the object must still be named: {logs}");
|
||||
assert!(!logs.contains("super-secret"), "the credential must not reach the log: {logs}");
|
||||
}
|
||||
|
||||
/// An empty target slot carries no outcome; reporting it would invent a
|
||||
/// failure for a target that was never attempted.
|
||||
#[test]
|
||||
fn empty_target_slots_are_not_reported_as_failures() {
|
||||
let rinfos = ReplicatedInfos {
|
||||
replication_timestamp: Some(OffsetDateTime::now_utc()),
|
||||
targets: vec![ReplicatedTargetInfo::default()],
|
||||
};
|
||||
|
||||
let logs = logs_at_default_level(|| {
|
||||
note_replication_terminal_failure("photos", "backups/unattempted.bin", None, &rinfos);
|
||||
});
|
||||
|
||||
assert!(logs.is_empty(), "an empty target slot must not be reported: {logs}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_identity_drift_re_arms_after_the_throttle_interval() {
|
||||
let arn = "arn:replication::drift-throttle-test";
|
||||
let start = Instant::now();
|
||||
|
||||
assert!(version_identity_drift_log_due(arn, start), "first drift must be reported");
|
||||
assert!(
|
||||
!version_identity_drift_log_due(arn, start + VERSION_IDENTITY_DRIFT_LOG_INTERVAL / 2),
|
||||
"a second drift inside the interval must stay throttled"
|
||||
);
|
||||
assert!(
|
||||
version_identity_drift_log_due(arn, start + VERSION_IDENTITY_DRIFT_LOG_INTERVAL),
|
||||
"drift must become visible again once the interval elapses, instead of \
|
||||
going silent for the rest of the process lifetime"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_identity_drift_throttles_each_target_independently() {
|
||||
let now = Instant::now();
|
||||
|
||||
assert!(version_identity_drift_log_due("arn:replication::drift-a", now));
|
||||
assert!(
|
||||
version_identity_drift_log_due("arn:replication::drift-b", now),
|
||||
"one target's report must not silence another's"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -260,15 +260,8 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
// the response path relies on). Keep the object's own multipart
|
||||
// flag so encrypted objects stay on the multipart route.
|
||||
} else {
|
||||
let (checksum_meta, checksum_record_is_multipart) = object_info.decrypt_checksums(0, &HeaderMap::new())?;
|
||||
// The checksum record describes how the *checksum* is composed,
|
||||
// not how the object is stored. A full-object checksum carries no
|
||||
// MULTIPART flag even on a multipart upload, so trusting it here
|
||||
// routed a 768-part object through a single PutObject and the
|
||||
// target rejected the 6 GiB body with EntityTooLarge
|
||||
// (rustfs#6825). The object's own shape is the authority: the
|
||||
// record may only add multipart-ness, never take it away.
|
||||
is_multipart = object_info.is_multipart() || checksum_record_is_multipart;
|
||||
let (checksum_meta, is_mp) = object_info.decrypt_checksums(0, &HeaderMap::new())?;
|
||||
is_multipart = is_mp;
|
||||
|
||||
for (key, value) in checksum_meta.iter() {
|
||||
if key != AMZ_CHECKSUM_TYPE {
|
||||
@@ -525,109 +518,6 @@ mod tests {
|
||||
use time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Serialize an object-level checksum record the way
|
||||
/// `complete_multipart_upload` persists it for a **full-object** checksum:
|
||||
/// the record carries the plain algorithm type, without the MULTIPART
|
||||
/// flags that a composite record gets.
|
||||
fn full_object_multipart_checksum_record() -> bytes::Bytes {
|
||||
let checksum_type = rustfs_rio::ChecksumType::from_string_with_obj_type("crc32", "FULL_OBJECT");
|
||||
assert!(checksum_type.is_set(), "crc32 FULL_OBJECT must be a valid checksum type");
|
||||
assert!(checksum_type.full_object_requested());
|
||||
|
||||
let mut combined = Vec::new();
|
||||
let mut checksum = rustfs_rio::Checksum {
|
||||
checksum_type,
|
||||
..Default::default()
|
||||
};
|
||||
for part in [b"part-one".as_slice(), b"part-two".as_slice()] {
|
||||
let part_checksum = rustfs_rio::Checksum::new_from_data(checksum_type, part).expect("part checksum");
|
||||
combined.extend_from_slice(part_checksum.raw.as_slice());
|
||||
checksum.add_part(&part_checksum, part.len() as i64).expect("add part");
|
||||
}
|
||||
|
||||
checksum.to_bytes(&combined)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_object_with_full_object_checksum_keeps_the_multipart_route() {
|
||||
// rustfs#6825: a 768-part upload was replicated with a single
|
||||
// PutObject and rejected by the target with EntityTooLarge. The
|
||||
// object's storage shape says multipart; only the checksum record
|
||||
// looked single-part, and the checksum record must not decide the
|
||||
// transport.
|
||||
let object_info = ObjectInfo {
|
||||
etag: Some("0123456789abcdef0123456789abcdef-768".to_string()),
|
||||
checksum: Some(full_object_multipart_checksum_record()),
|
||||
size: 6 * 1024 * 1024 * 1024,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(object_info.is_multipart(), "the fixture must be a multipart object");
|
||||
let (_, checksum_says_multipart) = object_info
|
||||
.decrypt_checksums(0, &HeaderMap::new())
|
||||
.expect("checksum record must decode");
|
||||
assert!(
|
||||
!checksum_says_multipart,
|
||||
"fixture precondition: a full-object record carries no MULTIPART flag, which is what used to \
|
||||
downgrade the transport"
|
||||
);
|
||||
|
||||
let (_, is_multipart) = replication_put_object_options("STANDARD", &object_info).expect("build put options");
|
||||
|
||||
assert!(
|
||||
is_multipart,
|
||||
"a multipart object must replicate over multipart whatever its checksum record looks like"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checksum_record_never_changes_the_transport_a_single_part_object_needs() {
|
||||
// The mirror of the rustfs#6825 guard: an object stored as one PUT
|
||||
// must keep the single-PUT transport, or its replica's ETag would
|
||||
// change shape and every ETag-based convergence check would re-copy it.
|
||||
let checksum =
|
||||
rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::CRC32, b"whole-object").expect("checksum fixture");
|
||||
let object_info = ObjectInfo {
|
||||
etag: Some("0123456789abcdef0123456789abcdef".to_string()),
|
||||
checksum: Some(checksum.to_bytes(&[])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(!object_info.is_multipart(), "the fixture must be a single-part object");
|
||||
|
||||
let (_, is_multipart) = replication_put_object_options("STANDARD", &object_info).expect("build put options");
|
||||
|
||||
assert!(!is_multipart, "a single-part object must not be promoted onto the multipart transport");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn composite_checksum_multipart_object_keeps_the_multipart_route() {
|
||||
// The checksum shape that already worked before rustfs#6825, pinned so
|
||||
// the fix cannot regress it.
|
||||
let mut checksum_type = rustfs_rio::ChecksumType::from_string("crc32");
|
||||
checksum_type
|
||||
.merge(rustfs_rio::ChecksumType::MULTIPART)
|
||||
.merge(rustfs_rio::ChecksumType::INCLUDES_MULTIPART);
|
||||
|
||||
let mut combined = Vec::new();
|
||||
for part in [b"part-one".as_slice(), b"part-two".as_slice()] {
|
||||
let part_checksum =
|
||||
rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::from_string("crc32"), part).expect("part checksum");
|
||||
combined.extend_from_slice(part_checksum.raw.as_slice());
|
||||
}
|
||||
let checksum = rustfs_rio::Checksum::new_from_data(checksum_type, &combined).expect("composite checksum");
|
||||
|
||||
let object_info = ObjectInfo {
|
||||
etag: Some("0123456789abcdef0123456789abcdef-2".to_string()),
|
||||
checksum: Some(checksum.to_bytes(&combined)),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (_, is_multipart) = replication_put_object_options("STANDARD", &object_info).expect("build put options");
|
||||
|
||||
assert!(is_multipart, "a composite-checksum multipart object must stay on the multipart transport");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_action_for_target_head_existing_object_source_newer_null_version_requires_replication() {
|
||||
let source = ObjectInfo {
|
||||
|
||||
@@ -190,7 +190,7 @@ fn install_heal_bucket_pre_mutation_barrier() -> Arc<DeleteBucketEmptyScanBarrie
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn pause_after_delete_bucket_empty_scan() {
|
||||
async fn pause_after_delete_bucket_empty_scan() {
|
||||
let barrier = DELETE_BUCKET_EMPTY_SCAN_BARRIER
|
||||
.lock()
|
||||
.expect("empty scan barrier lock should not be poisoned")
|
||||
|
||||
@@ -49,7 +49,7 @@ use crate::error::{
|
||||
is_err_version_not_found,
|
||||
};
|
||||
use crate::layout::endpoints::EndpointServerPools;
|
||||
use crate::object_api::{DecommissionCapacityOptions, GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
use crate::object_api::{DecommissionCapacityOptions, GetObjectReader, ObjectOptions};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::services::notification_sys::{
|
||||
acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches, tier_delete_journal_topology_generation,
|
||||
@@ -78,9 +78,7 @@ use rmp_serde::Serializer;
|
||||
use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
|
||||
use rustfs_heal_contracts::heal_channel::HealOpts;
|
||||
use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
|
||||
use rustfs_utils::path::{
|
||||
decode_dir_object, encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path,
|
||||
};
|
||||
use rustfs_utils::path::{encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path};
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ReplicationConfiguration};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -4077,54 +4075,23 @@ pub(crate) struct PoolMetaWriteState {
|
||||
expected_cluster_id: Option<uuid::Uuid>,
|
||||
cluster_epoch: Option<u64>,
|
||||
pool_meta_absent: bool,
|
||||
bootstrap_authority: PoolMetaBootstrapAuthority,
|
||||
fresh_bootstrap_proven: bool,
|
||||
identity_initialized: Option<bool>,
|
||||
identity_fresh_bootstrap_nonce: Option<uuid::Uuid>,
|
||||
identity_needs_repair: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub(crate) enum PoolMetaBootstrapAuthority {
|
||||
#[default]
|
||||
None,
|
||||
Fresh,
|
||||
LegacyAdoption,
|
||||
}
|
||||
|
||||
impl PoolMetaBootstrapAuthority {
|
||||
pub(crate) fn combine_across_pools(self, other: Self) -> Self {
|
||||
if self == other { self } else { Self::None }
|
||||
}
|
||||
|
||||
fn is_proven(self) -> bool {
|
||||
!matches!(self, Self::None)
|
||||
}
|
||||
}
|
||||
|
||||
impl PoolMetaWriteState {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn for_startup(cluster_id: uuid::Uuid, fresh_bootstrap_proven: bool) -> Self {
|
||||
let bootstrap_authority = if fresh_bootstrap_proven {
|
||||
PoolMetaBootstrapAuthority::Fresh
|
||||
} else {
|
||||
PoolMetaBootstrapAuthority::None
|
||||
};
|
||||
Self::for_startup_with_bootstrap_authority(cluster_id, bootstrap_authority)
|
||||
}
|
||||
|
||||
pub(crate) fn for_startup_with_bootstrap_authority(
|
||||
cluster_id: uuid::Uuid,
|
||||
bootstrap_authority: PoolMetaBootstrapAuthority,
|
||||
) -> Self {
|
||||
Self {
|
||||
expected_cluster_id: Some(cluster_id),
|
||||
bootstrap_authority,
|
||||
fresh_bootstrap_proven,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn bootstrap_identity_proven(&self) -> bool {
|
||||
self.bootstrap_authority.is_proven()
|
||||
pub(crate) fn fresh_bootstrap_proven(&self) -> bool {
|
||||
self.fresh_bootstrap_proven
|
||||
}
|
||||
|
||||
pub(crate) fn identity_is_pending(&self) -> bool {
|
||||
@@ -4146,7 +4113,7 @@ impl PoolMetaWriteState {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
fn for_test_bootstrap() -> Self {
|
||||
Self {
|
||||
bootstrap_authority: PoolMetaBootstrapAuthority::Fresh,
|
||||
fresh_bootstrap_proven: true,
|
||||
identity_initialized: Some(false),
|
||||
identity_fresh_bootstrap_nonce: Some(uuid::Uuid::new_v4()),
|
||||
..Default::default()
|
||||
@@ -4207,7 +4174,7 @@ impl PoolMetaWriteState {
|
||||
self.identity_fresh_bootstrap_nonce = selection.identity.and_then(|identity| identity.fresh_bootstrap_nonce);
|
||||
if let Some(identity) = selection.identity {
|
||||
if identity.initialized {
|
||||
self.bootstrap_authority = PoolMetaBootstrapAuthority::None;
|
||||
self.fresh_bootstrap_proven = false;
|
||||
}
|
||||
if let Some(metadata_epoch) = self.cluster_epoch
|
||||
&& metadata_epoch != identity.epoch
|
||||
@@ -4231,11 +4198,11 @@ impl PoolMetaWriteState {
|
||||
return Ok(());
|
||||
}
|
||||
match self.identity_initialized {
|
||||
Some(false) if self.bootstrap_identity_proven() && self.identity_fresh_bootstrap_nonce.is_some() => Ok(()),
|
||||
Some(false) if self.fresh_bootstrap_proven && self.identity_fresh_bootstrap_nonce.is_some() => Ok(()),
|
||||
Some(false) => {
|
||||
self.block_writes();
|
||||
Err(Error::other(
|
||||
"pool metadata recovery required: pending cluster identity exists but this startup has no verified fresh-bootstrap proof or legacy-adoption proof",
|
||||
"pool metadata recovery required: pending cluster identity exists but this startup has no verified fresh-bootstrap proof",
|
||||
))
|
||||
}
|
||||
Some(true) => {
|
||||
@@ -5217,10 +5184,10 @@ where
|
||||
..identity
|
||||
},
|
||||
Some(identity) => identity,
|
||||
None if !initialized && !write_state.bootstrap_identity_proven() => {
|
||||
None if !initialized && !write_state.fresh_bootstrap_proven() => {
|
||||
write_state.block_writes();
|
||||
return Err(Error::other(
|
||||
"pool metadata recovery required: cannot create a pending cluster identity without verified fresh-bootstrap proof or legacy-adoption proof",
|
||||
"pool metadata recovery required: cannot create a pending cluster identity without verified fresh-bootstrap proof",
|
||||
));
|
||||
}
|
||||
None => PersistedPoolMetaIdentity {
|
||||
@@ -7192,170 +7159,6 @@ pub(crate) fn decommission_capacity_mutation_id(
|
||||
uuid::Uuid::from_bytes(bytes)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct ExactDeleteCapacityReconciliation {
|
||||
source_pool_index: usize,
|
||||
target_pool_index: usize,
|
||||
mutation_id: uuid::Uuid,
|
||||
expected_data_bytes: usize,
|
||||
expected_target_physical_bytes: usize,
|
||||
}
|
||||
|
||||
fn plan_exact_delete_capacity_reconciliations(
|
||||
meta: &PoolMeta,
|
||||
object: &str,
|
||||
exact: &ObjectInfo,
|
||||
) -> Result<Vec<ExactDeleteCapacityReconciliation>> {
|
||||
let version_id = exact.version_id.map(|version_id| version_id.to_string());
|
||||
let mut matches = Vec::new();
|
||||
|
||||
for (source_pool_index, pool) in meta.pools.iter().enumerate() {
|
||||
let Some(reservation) = pool
|
||||
.decommission
|
||||
.as_ref()
|
||||
.and_then(|info| info.capacity_reservation.as_ref())
|
||||
.filter(|reservation| reservation.active())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let owner = DecommissionCapacityOwner {
|
||||
source_pool_index,
|
||||
operation_id: reservation.operation_id,
|
||||
generation: reservation.generation,
|
||||
owner_nonce: reservation.owner_nonce,
|
||||
mutation_id: None,
|
||||
};
|
||||
let logical_mutation_id = decommission_capacity_mutation_id(
|
||||
owner,
|
||||
&exact.bucket,
|
||||
&exact.name,
|
||||
version_id.as_deref(),
|
||||
exact.delete_marker,
|
||||
exact.mod_time,
|
||||
);
|
||||
// Existing data-movement producers persist directory-key intents using
|
||||
// either the logical name or its internal `__XLDIR__` representation.
|
||||
// Accept both while retaining the exact persisted identity for CAS.
|
||||
let internal_mutation_id = if object == exact.name {
|
||||
logical_mutation_id
|
||||
} else {
|
||||
decommission_capacity_mutation_id(
|
||||
owner,
|
||||
&exact.bucket,
|
||||
object,
|
||||
version_id.as_deref(),
|
||||
exact.delete_marker,
|
||||
exact.mod_time,
|
||||
)
|
||||
};
|
||||
let mut source_match = None;
|
||||
|
||||
for target in &reservation.targets {
|
||||
if target.pending_physical_bytes == 0 {
|
||||
continue;
|
||||
}
|
||||
let Some(pending_mutation_id) = target.pending_mutation_id else {
|
||||
return Err(decommission_capacity_blocked_error(format!(
|
||||
"source pool {source_pool_index} target pool {} has pending capacity without an object identity",
|
||||
target.pool_index
|
||||
)));
|
||||
};
|
||||
if pending_mutation_id != logical_mutation_id && pending_mutation_id != internal_mutation_id {
|
||||
continue;
|
||||
}
|
||||
if source_match.is_some() {
|
||||
return Err(decommission_capacity_blocked_error(format!(
|
||||
"source pool {source_pool_index} has the same exact-delete capacity intent on multiple targets"
|
||||
)));
|
||||
}
|
||||
source_match = Some((target.pool_index, target.layout, target.pending_physical_bytes, pending_mutation_id));
|
||||
}
|
||||
|
||||
let Some((target_pool_index, target_layout, pending_physical_bytes, mutation_id)) = source_match else {
|
||||
continue;
|
||||
};
|
||||
if exact.version_id.is_none() && exact.mod_time.is_none() {
|
||||
return Err(decommission_capacity_blocked_error(
|
||||
"unversioned exact delete cannot identify pending capacity without a modification time",
|
||||
));
|
||||
}
|
||||
let expected_data_bytes = if exact.delete_marker {
|
||||
0
|
||||
} else {
|
||||
usize::try_from(exact.size).map_err(|_| {
|
||||
decommission_capacity_blocked_error("exact delete cannot reconcile a negative or overflowing object size")
|
||||
})?
|
||||
};
|
||||
let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
|
||||
if pending_physical_bytes != expected_target_physical_bytes {
|
||||
return Err(decommission_capacity_blocked_error(format!(
|
||||
"source pool {source_pool_index} target pool {target_pool_index} pending capacity does not match the exact object size"
|
||||
)));
|
||||
}
|
||||
let remaining_target_physical_bytes = reservation
|
||||
.targets
|
||||
.iter()
|
||||
.find(|target| target.pool_index == target_pool_index)
|
||||
.map(|target| {
|
||||
target.remaining_reserved_physical_bytes(reservation.temporary_copies)
|
||||
/ 1usize.saturating_add(reservation.temporary_copies)
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let remaining_total_physical_bytes = reservation
|
||||
.predicted_physical_bytes
|
||||
.saturating_sub(reservation.consumed_target_physical_bytes);
|
||||
let remaining_data_bytes = reservation
|
||||
.source_data_equivalent_bytes
|
||||
.saturating_sub(reservation.committed_data_bytes);
|
||||
if expected_target_physical_bytes > remaining_target_physical_bytes
|
||||
|| expected_target_physical_bytes > remaining_total_physical_bytes
|
||||
|| expected_data_bytes > remaining_data_bytes
|
||||
{
|
||||
return Err(decommission_capacity_blocked_error(format!(
|
||||
"source pool {source_pool_index} target pool {target_pool_index} lacks reservation capacity for the exact object"
|
||||
)));
|
||||
}
|
||||
matches.push(ExactDeleteCapacityReconciliation {
|
||||
source_pool_index,
|
||||
target_pool_index,
|
||||
mutation_id,
|
||||
expected_data_bytes,
|
||||
expected_target_physical_bytes,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(matches)
|
||||
}
|
||||
|
||||
fn ensure_exact_delete_capacity_namespace_fences(opts: &ObjectOptions, bucket: &str, object: &str) -> Result<()> {
|
||||
let object_fence = opts.namespace_lock_fence.as_ref().ok_or_else(|| {
|
||||
decommission_capacity_blocked_error("exact delete capacity reconciliation requires an object namespace fence")
|
||||
})?;
|
||||
if object_fence.is_lock_lost() {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "exact_delete_capacity_reconciliation",
|
||||
bucket: bucket.to_string(),
|
||||
object: decode_dir_object(object),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
if opts
|
||||
.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(crate::object_api::NamespaceLockFence::is_lock_lost)
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "exact_delete_capacity_bucket_generation",
|
||||
bucket: bucket.to_string(),
|
||||
object: decode_dir_object(object),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_decommission_capacity_mutation_id(bucket: &str, object: &str, opts: &mut ObjectOptions) {
|
||||
if opts
|
||||
.decommission_capacity
|
||||
@@ -8419,117 +8222,6 @@ impl ECStore {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn reconcile_decommission_capacity_before_exact_delete(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
exact: &ObjectInfo,
|
||||
) -> Result<()> {
|
||||
if exact.bucket != bucket || exact.name != decode_dir_object(object) {
|
||||
return Err(decommission_capacity_blocked_error(
|
||||
"exact delete object identity changed before capacity reconciliation",
|
||||
));
|
||||
}
|
||||
|
||||
let reconciliations = {
|
||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (_read_guard, snapshot) = self
|
||||
.acquire_pool_meta_read_guard(&mut save_guard, "exact delete capacity reconciliation failed")
|
||||
.await?;
|
||||
plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)?
|
||||
};
|
||||
if reconciliations.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
|
||||
|
||||
let target_lookup_options = ObjectOptions {
|
||||
versioned: opts.versioned,
|
||||
version_suspended: opts.version_suspended,
|
||||
version_id: opts.version_id.clone(),
|
||||
metadata_chg: opts.version_id.is_some(),
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
for reconciliation in &reconciliations {
|
||||
let target_pool = self.pools.get(reconciliation.target_pool_index).ok_or_else(|| {
|
||||
decommission_capacity_blocked_error(format!(
|
||||
"source pool {} exact-delete capacity target pool {} is out of range",
|
||||
reconciliation.source_pool_index, reconciliation.target_pool_index
|
||||
))
|
||||
})?;
|
||||
let target = target_pool
|
||||
.get_object_info(bucket, object, &target_lookup_options)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
decommission_capacity_blocked_error(format!(
|
||||
"source pool {} target pool {} exact object evidence could not be read: {err}",
|
||||
reconciliation.source_pool_index, reconciliation.target_pool_index
|
||||
))
|
||||
})?;
|
||||
if !Self::is_equivalent_decommission_capacity_target(exact, &target) {
|
||||
return Err(decommission_capacity_blocked_error(format!(
|
||||
"source pool {} target pool {} does not contain an equivalent exact object for its pending capacity intent",
|
||||
reconciliation.source_pool_index, reconciliation.target_pool_index
|
||||
)));
|
||||
}
|
||||
}
|
||||
ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
|
||||
|
||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (write_guard, mut snapshot) = self
|
||||
.acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed")
|
||||
.await?;
|
||||
let current_reconciliations = plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)?;
|
||||
if current_reconciliations != reconciliations {
|
||||
return Err(decommission_capacity_blocked_error(
|
||||
"pending capacity changed while exact target evidence was being verified",
|
||||
));
|
||||
}
|
||||
ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut source_pool_indices = Vec::with_capacity(current_reconciliations.len());
|
||||
for reconciliation in current_reconciliations {
|
||||
resolve_decommission_target_pending(
|
||||
&mut snapshot,
|
||||
reconciliation.source_pool_index,
|
||||
reconciliation.target_pool_index,
|
||||
reconciliation.expected_target_physical_bytes,
|
||||
reconciliation.mutation_id,
|
||||
)?;
|
||||
record_decommission_target_consumption(
|
||||
&mut snapshot,
|
||||
reconciliation.source_pool_index,
|
||||
reconciliation.target_pool_index,
|
||||
DecommissionTargetConsumption {
|
||||
committed_data_bytes: reconciliation.expected_data_bytes,
|
||||
target_physical_bytes: reconciliation.expected_target_physical_bytes,
|
||||
observed_physical_bytes: 0,
|
||||
},
|
||||
reconciliation.mutation_id,
|
||||
now,
|
||||
)?;
|
||||
source_pool_indices.push(reconciliation.source_pool_index);
|
||||
}
|
||||
source_pool_indices.sort_unstable();
|
||||
source_pool_indices.dedup();
|
||||
ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
|
||||
|
||||
let outcome = snapshot
|
||||
.save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &source_pool_indices)
|
||||
.await?;
|
||||
ensure_pool_meta_write_fence(&write_guard, "exact delete capacity reconciliation save failed")?;
|
||||
{
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
publish_pool_meta_updates(&mut pool_meta, &outcome.committed, &source_pool_indices);
|
||||
}
|
||||
ensure_pool_meta_write_fence(&write_guard, "exact delete capacity reconciliation save failed")?;
|
||||
outcome.disarm();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn reconcile_decommission_capacity_after_equivalent_target(
|
||||
&self,
|
||||
owner: DecommissionCapacityOwner,
|
||||
@@ -17457,8 +17149,7 @@ mod pools_tests {
|
||||
use super::{
|
||||
DecommissionCapacityOwner, DecommissionCapacityReservation, DecommissionCapacityTemporaryMutation,
|
||||
decommission_capacity_mutation_id, ensure_decommission_target_owner_admission,
|
||||
ensure_exact_delete_capacity_namespace_fences, ensure_external_decommission_target_admission,
|
||||
is_decommission_capacity_blocked_error, plan_exact_delete_capacity_reconciliations,
|
||||
ensure_external_decommission_target_admission, is_decommission_capacity_blocked_error,
|
||||
record_decommission_target_consumption, reserve_decommission_target_pending, resolve_decommission_target_pending,
|
||||
set_decommission_capacity_info_overrides_for_test,
|
||||
};
|
||||
@@ -17473,7 +17164,7 @@ mod pools_tests {
|
||||
use crate::disk::{STORAGE_FORMAT_FILE, endpoint::Endpoint};
|
||||
use crate::error::{Error, StorageError};
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
|
||||
@@ -21594,135 +21285,6 @@ mod pools_tests {
|
||||
assert_eq!(first, recovered, "a lease nonce rotation must not change the mutation identity");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_delete_capacity_plan_requires_identity_and_exact_size() {
|
||||
let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
|
||||
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
|
||||
let capacity_infos = vec![
|
||||
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
|
||||
DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
|
||||
];
|
||||
let mut meta = PoolMeta {
|
||||
version: POOL_META_VERSION,
|
||||
pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
|
||||
..Default::default()
|
||||
};
|
||||
meta.decommission(0, capacity_infos[0].space).unwrap();
|
||||
reserve_decommission_start_target_capacity(&mut meta, &[0], &capacity_infos, uuid::Uuid::new_v4(), 1, now)
|
||||
.expect("the exact-delete test reservation should fit");
|
||||
let exact = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::from_u128(7)),
|
||||
mod_time: Some(now),
|
||||
size: 10,
|
||||
..Default::default()
|
||||
};
|
||||
let owner = {
|
||||
let reservation = meta.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.and_then(|info| info.capacity_reservation.as_ref())
|
||||
.expect("the exact-delete test reservation should exist");
|
||||
DecommissionCapacityOwner {
|
||||
source_pool_index: 0,
|
||||
operation_id: reservation.operation_id,
|
||||
generation: reservation.generation,
|
||||
owner_nonce: reservation.owner_nonce,
|
||||
mutation_id: None,
|
||||
}
|
||||
};
|
||||
let version_id = exact.version_id.map(|version_id| version_id.to_string());
|
||||
let mutation_id = decommission_capacity_mutation_id(
|
||||
owner,
|
||||
&exact.bucket,
|
||||
&exact.name,
|
||||
version_id.as_deref(),
|
||||
exact.delete_marker,
|
||||
exact.mod_time,
|
||||
);
|
||||
reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(1))
|
||||
.expect("the exact-delete test intent should be reserved");
|
||||
|
||||
let plan = plan_exact_delete_capacity_reconciliations(&meta, &exact.name, &exact)
|
||||
.expect("the exact identity should match the pending intent");
|
||||
assert_eq!(plan.len(), 1);
|
||||
assert_eq!(plan[0].source_pool_index, 0);
|
||||
assert_eq!(plan[0].target_pool_index, 1);
|
||||
assert_eq!(plan[0].expected_data_bytes, 10);
|
||||
assert_eq!(plan[0].expected_target_physical_bytes, 10);
|
||||
|
||||
let mismatched_size = ObjectInfo {
|
||||
size: 9,
|
||||
..exact.clone()
|
||||
};
|
||||
let mismatched_size = plan_exact_delete_capacity_reconciliations(&meta, &mismatched_size.name, &mismatched_size)
|
||||
.expect_err("a different exact size must not consume the pending intent");
|
||||
assert!(mismatched_size.to_string().contains("does not match the exact object size"));
|
||||
|
||||
let directory_exact = ObjectInfo {
|
||||
name: "directory/".to_string(),
|
||||
..exact.clone()
|
||||
};
|
||||
let internal_directory = rustfs_utils::path::encode_dir_object(&directory_exact.name);
|
||||
let internal_directory_mutation_id = decommission_capacity_mutation_id(
|
||||
owner,
|
||||
&directory_exact.bucket,
|
||||
&internal_directory,
|
||||
version_id.as_deref(),
|
||||
directory_exact.delete_marker,
|
||||
directory_exact.mod_time,
|
||||
);
|
||||
meta.pools[0]
|
||||
.decommission
|
||||
.as_mut()
|
||||
.and_then(|info| info.capacity_reservation.as_mut())
|
||||
.expect("the exact-delete test reservation should exist")
|
||||
.targets[0]
|
||||
.pending_mutation_id = Some(internal_directory_mutation_id);
|
||||
let directory_plan = plan_exact_delete_capacity_reconciliations(&meta, &internal_directory, &directory_exact)
|
||||
.expect("an internally encoded directory intent should match its logical exact object");
|
||||
assert_eq!(directory_plan[0].mutation_id, internal_directory_mutation_id);
|
||||
|
||||
let logical_directory_mutation_id = decommission_capacity_mutation_id(
|
||||
owner,
|
||||
&directory_exact.bucket,
|
||||
&directory_exact.name,
|
||||
version_id.as_deref(),
|
||||
directory_exact.delete_marker,
|
||||
directory_exact.mod_time,
|
||||
);
|
||||
meta.pools[0]
|
||||
.decommission
|
||||
.as_mut()
|
||||
.and_then(|info| info.capacity_reservation.as_mut())
|
||||
.expect("the exact-delete test reservation should exist")
|
||||
.targets[0]
|
||||
.pending_mutation_id = Some(logical_directory_mutation_id);
|
||||
let directory_plan = plan_exact_delete_capacity_reconciliations(&meta, &internal_directory, &directory_exact)
|
||||
.expect("a logical directory intent should match its internally encoded delete path");
|
||||
assert_eq!(directory_plan[0].mutation_id, logical_directory_mutation_id);
|
||||
|
||||
meta.pools[0]
|
||||
.decommission
|
||||
.as_mut()
|
||||
.and_then(|info| info.capacity_reservation.as_mut())
|
||||
.expect("the exact-delete test reservation should exist")
|
||||
.targets[0]
|
||||
.pending_mutation_id = None;
|
||||
let unidentified = plan_exact_delete_capacity_reconciliations(&meta, &exact.name, &exact)
|
||||
.expect_err("an unidentified pending intent must fail closed");
|
||||
assert!(unidentified.to_string().contains("without an object identity"));
|
||||
|
||||
let mut opts = ObjectOptions::default();
|
||||
let unfenced = ensure_exact_delete_capacity_namespace_fences(&opts, &exact.bucket, &exact.name)
|
||||
.expect_err("capacity reconciliation must reject a missing object namespace fence");
|
||||
assert!(unfenced.to_string().contains("requires an object namespace fence"));
|
||||
opts.ensure_namespace_lock_fence();
|
||||
ensure_exact_delete_capacity_namespace_fences(&opts, &exact.bucket, &exact.name)
|
||||
.expect("a live object namespace fence should admit capacity reconciliation");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_write_admission_cannot_race_into_a_reserved_target() {
|
||||
let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
|
||||
|
||||
@@ -196,7 +196,7 @@ mod decommission_lock_order_tests {
|
||||
use crate::bucket::lifecycle::lifecycle::TRANSITION_PENDING;
|
||||
use crate::core::pools::{
|
||||
DecommissionCapacityLockOrderBarrier, DecommissionCapacityOwner, DecommissionErasureLayout, DecommissionPoolCapacityInfo,
|
||||
POOL_META_NAME, decommission_capacity_mutation_id, set_decommission_capacity_info_overrides_for_test,
|
||||
POOL_META_NAME, set_decommission_capacity_info_overrides_for_test,
|
||||
};
|
||||
use crate::data_movement;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
@@ -3047,198 +3047,6 @@ mod decommission_lock_order_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn exact_delete_reconciles_pending_capacity_before_removing_replicas() {
|
||||
let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let bucket = test_bucket("exact-delete-capacity");
|
||||
let object = "published-before-exact-delete.bin";
|
||||
let body = vec![0x55; 64 * 1024];
|
||||
let version_id = uuid::Uuid::new_v4().to_string();
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create the exact-delete reconciliation bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id(&bucket)
|
||||
.await
|
||||
.expect("load the exact-delete bucket incarnation");
|
||||
|
||||
let mut source_data = PutObjReader::from_vec(body.clone());
|
||||
let source = store.pools[0]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut source_data,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version_id.clone()),
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed the exact source version");
|
||||
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
|
||||
let target_total = body.len().saturating_mul(4);
|
||||
set_decommission_capacity_info_overrides_for_test(
|
||||
store.id,
|
||||
vec![vec![
|
||||
DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len(), body.len()),
|
||||
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
|
||||
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
|
||||
]],
|
||||
);
|
||||
store
|
||||
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
|
||||
.await
|
||||
.expect("activate the exact-delete capacity reservation");
|
||||
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
|
||||
let target_pool_index = store.pool_meta.read().await.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.and_then(|info| info.capacity_reservation.as_ref())
|
||||
.expect("the exact-delete capacity reservation should exist")
|
||||
.targets[0]
|
||||
.pool_index;
|
||||
assert_eq!(target_pool_index, 2);
|
||||
|
||||
let target_options = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version_id.clone()),
|
||||
mod_time: source.mod_time,
|
||||
preserve_etag: source.etag.clone(),
|
||||
user_defined: (*source.user_defined).clone(),
|
||||
data_movement: true,
|
||||
src_pool_idx: 0,
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
let mut target_data = PutObjReader::from_vec(body.clone());
|
||||
let target = store.pools[target_pool_index]
|
||||
.put_object(&bucket, object, &mut target_data, &target_options)
|
||||
.await
|
||||
.expect("publish the target version before capacity progress");
|
||||
assert_eq!(target.version_id, source.version_id);
|
||||
assert_eq!(target.mod_time, source.mod_time);
|
||||
assert_eq!(target.size, source.size);
|
||||
|
||||
let source_version_id = source.version_id.map(|version_id| version_id.to_string());
|
||||
let mutation_id = decommission_capacity_mutation_id(
|
||||
owner,
|
||||
&source.bucket,
|
||||
&source.name,
|
||||
source_version_id.as_deref(),
|
||||
source.delete_marker,
|
||||
source.mod_time,
|
||||
);
|
||||
{
|
||||
let mut pool_meta = store.pool_meta.write().await;
|
||||
let source_pool = &mut pool_meta.pools[0];
|
||||
let reservation = source_pool
|
||||
.decommission
|
||||
.as_mut()
|
||||
.and_then(|info| info.capacity_reservation.as_mut())
|
||||
.expect("the exact-delete capacity reservation should remain active");
|
||||
let target = reservation
|
||||
.targets
|
||||
.iter_mut()
|
||||
.find(|target| target.pool_index == target_pool_index)
|
||||
.expect("the exact-delete target allocation should exist");
|
||||
target.pending_physical_bytes = body.len();
|
||||
target.pending_mutation_id = Some(mutation_id);
|
||||
reservation.pending_target_physical_bytes = body.len();
|
||||
source_pool.last_update = time::OffsetDateTime::now_utc();
|
||||
}
|
||||
store
|
||||
.save_current_pool_meta_for_test(&[0])
|
||||
.await
|
||||
.expect("persist the simulated post-commit capacity intent");
|
||||
|
||||
let exact_delete_options = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version_id.clone()),
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
store.pools[target_pool_index]
|
||||
.delete_object(&bucket, object, exact_delete_options.clone())
|
||||
.await
|
||||
.expect("remove the target evidence before the fail-closed exact delete");
|
||||
let delete_err = store
|
||||
.delete_object(&bucket, object, exact_delete_options.clone())
|
||||
.await
|
||||
.expect_err("exact delete must fail while its pending target evidence is absent");
|
||||
assert!(matches!(delete_err, crate::error::Error::DecommissionCapacityBlocked { .. }));
|
||||
store.pools[0]
|
||||
.get_object_info(
|
||||
&bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version_id.clone()),
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a failed reconciliation must preserve the source evidence");
|
||||
let mut failed = crate::core::pools::PoolMeta::default();
|
||||
failed
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("the failed exact delete must preserve readable capacity metadata");
|
||||
let failed_reservation = failed.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.and_then(|info| info.capacity_reservation.as_ref())
|
||||
.expect("the failed exact delete reservation should remain present");
|
||||
assert_eq!(failed_reservation.pending_target_physical_bytes, body.len());
|
||||
assert_eq!(failed_reservation.consumed_target_physical_bytes, 0);
|
||||
|
||||
let mut replacement_target_data = PutObjReader::from_vec(body.clone());
|
||||
store.pools[target_pool_index]
|
||||
.put_object(&bucket, object, &mut replacement_target_data, &target_options)
|
||||
.await
|
||||
.expect("restore the equivalent target evidence for the exact-delete retry");
|
||||
|
||||
store
|
||||
.delete_object(&bucket, object, exact_delete_options)
|
||||
.await
|
||||
.expect("the exact delete should reconcile capacity before removing replicas");
|
||||
|
||||
let mut persisted = crate::core::pools::PoolMeta::default();
|
||||
persisted
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
.await
|
||||
.expect("the exact-delete reconciliation should remain durable");
|
||||
let reservation = persisted.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.and_then(|info| info.capacity_reservation.as_ref())
|
||||
.expect("the reconciled reservation should remain present");
|
||||
assert_eq!(reservation.pending_target_physical_bytes, 0);
|
||||
assert_eq!(reservation.consumed_target_physical_bytes, body.len());
|
||||
assert_eq!(reservation.committed_data_bytes, body.len());
|
||||
|
||||
for pool_index in [0, target_pool_index] {
|
||||
let err = store.pools[pool_index]
|
||||
.get_object_info(
|
||||
&bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version_id.clone()),
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("the exact version should be absent after reconciliation and delete");
|
||||
assert!(crate::error::is_err_object_not_found(&err) || crate::error::is_err_version_not_found(&err));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn data_movement_equivalent_target_reconciles_published_capacity_after_restart() {
|
||||
|
||||
@@ -717,8 +717,8 @@ fn is_equivalent_data_movement_part(source: &ObjectPartInfo, target: &ObjectPart
|
||||
== target.checksums.as_ref().filter(|checksums| !checksums.is_empty()))
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_parts_by_number(parts: &[ObjectPartInfo]) -> Option<HashMap<usize, &ObjectPartInfo>> {
|
||||
let mut parts_by_number = HashMap::with_capacity(parts.len());
|
||||
fn data_movement_parts_by_number(parts: &[ObjectPartInfo]) -> Option<BTreeMap<usize, &ObjectPartInfo>> {
|
||||
let mut parts_by_number = BTreeMap::new();
|
||||
for part in parts {
|
||||
if parts_by_number.insert(part.number, part).is_some() {
|
||||
return None;
|
||||
|
||||
@@ -6444,9 +6444,6 @@ impl LocalDisk {
|
||||
.abort_reserved_version_delete(object_dir, rollback_dir, volume, path, "delete_versions_commit_intent", err)
|
||||
.await);
|
||||
}
|
||||
if should_fail_after_delete_commit(self.root.as_path(), path) {
|
||||
return Err(DiskError::Unexpected);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -6517,10 +6514,6 @@ impl LocalDisk {
|
||||
.await);
|
||||
}
|
||||
|
||||
if should_fail_after_delete_commit(self.root.as_path(), path) {
|
||||
return Err(DiskError::Unexpected);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -37,9 +37,7 @@ use rustfs_filemeta::{FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, RestoreS
|
||||
use rustfs_rio::Checksum;
|
||||
use rustfs_utils::CompressionAlgorithm;
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
||||
use rustfs_utils::http::{
|
||||
AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS, SUFFIX_PLAINTEXT_CHECKSUM, get_consistent_str,
|
||||
};
|
||||
use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS};
|
||||
use rustfs_utils::path::decode_dir_object;
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Debug;
|
||||
|
||||
@@ -19,7 +19,6 @@ use crate::storage_api_contracts::{
|
||||
HTTPPreconditions, ObjectLockRetentionOptions, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState,
|
||||
},
|
||||
};
|
||||
use std::io;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
|
||||
use tokio::sync::{Mutex, Notify, OwnedRwLockReadGuard};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -679,196 +678,6 @@ pub struct DecommissionCapacityOptions {
|
||||
pub(crate) mutation_id: Option<Uuid>,
|
||||
}
|
||||
|
||||
/// Opaque storage-owned collection point for post-commit tier free-version
|
||||
/// cleanup receipts. This type is public only because workspace crates build
|
||||
/// [`ObjectOptions`] with struct literals; callers outside `ecstore` must leave
|
||||
/// the corresponding option unset.
|
||||
#[doc(hidden)]
|
||||
#[derive(Clone)]
|
||||
pub struct TierFreeVersionReceiptSink {
|
||||
inner: Arc<parking_lot::Mutex<TierFreeVersionReceiptSinkState>>,
|
||||
}
|
||||
|
||||
struct TierFreeVersionReceiptSinkState {
|
||||
receipts: Option<HashMap<TierFreeVersionReceiptIdentity, TierFreeVersionReceiptPayload>>,
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq, Hash)]
|
||||
struct TierFreeVersionReceiptIdentity {
|
||||
bucket: String,
|
||||
logical_name: String,
|
||||
tier: String,
|
||||
remote_name: String,
|
||||
remote_version_state: TierFreeVersionReceiptVersionState,
|
||||
remote_version: String,
|
||||
backend_identity: crate::services::tier::tier::TierDestinationId,
|
||||
}
|
||||
|
||||
struct TierFreeVersionReceiptPayload {
|
||||
local_free_version_id: Uuid,
|
||||
mod_time: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
|
||||
enum TierFreeVersionReceiptVersionState {
|
||||
KnownDisabled,
|
||||
SuspendedNull,
|
||||
Exact,
|
||||
}
|
||||
|
||||
impl TierFreeVersionReceiptSink {
|
||||
/// Only the delete wrapper may originate a sink. The public type exists so
|
||||
/// workspace struct literals can carry it, but external crates cannot
|
||||
/// create an undrainable collector accidentally.
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
inner: Arc::new(parking_lot::Mutex::new(TierFreeVersionReceiptSinkState {
|
||||
receipts: Some(HashMap::new()),
|
||||
})),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for TierFreeVersionReceiptSink {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let state = self.inner.lock();
|
||||
f.debug_struct("TierFreeVersionReceiptSink")
|
||||
.field("drained", &state.receipts.is_none())
|
||||
.field("receipt_count", &state.receipts.as_ref().map(HashMap::len).unwrap_or_default())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl TierFreeVersionReceiptVersionState {
|
||||
fn persisted(self) -> rustfs_filemeta::TransitionVersionState {
|
||||
match self {
|
||||
Self::KnownDisabled => rustfs_filemeta::TransitionVersionState::KnownDisabled,
|
||||
Self::SuspendedNull => rustfs_filemeta::TransitionVersionState::SuspendedNull,
|
||||
Self::Exact => rustfs_filemeta::TransitionVersionState::Exact,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TierFreeVersionReceiptIdentity {
|
||||
fn into_object_info(self, payload: TierFreeVersionReceiptPayload) -> ObjectInfo {
|
||||
let mut metadata = HashMap::with_capacity(2);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex(self.backend_identity),
|
||||
);
|
||||
ObjectInfo {
|
||||
bucket: self.bucket,
|
||||
name: self.logical_name,
|
||||
mod_time: payload.mod_time,
|
||||
user_defined: Arc::new(metadata),
|
||||
version_id: Some(payload.local_free_version_id),
|
||||
delete_marker: true,
|
||||
transitioned_object: TransitionedObject {
|
||||
name: self.remote_name,
|
||||
version_id: self.remote_version,
|
||||
tier: self.tier,
|
||||
free_version: true,
|
||||
status: String::new(),
|
||||
},
|
||||
transition_version_state: self.remote_version_state.persisted(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn tier_free_version_scheduling_receipt_from_source(
|
||||
source: &ObjectInfo,
|
||||
local_free_version_id: Uuid,
|
||||
) -> io::Result<Option<(TierFreeVersionReceiptIdentity, TierFreeVersionReceiptPayload)>> {
|
||||
if source.transitioned_object.status != rustfs_filemeta::TRANSITION_COMPLETE
|
||||
|| source.transitioned_object.free_version
|
||||
|| source.delete_marker
|
||||
|| source.bucket.is_empty()
|
||||
|| source.name.is_empty()
|
||||
|| source.transitioned_object.tier.is_empty()
|
||||
|| source.transitioned_object.name.is_empty()
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
if local_free_version_id.is_nil() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"tier free-version receipt has a nil local version identity",
|
||||
));
|
||||
}
|
||||
|
||||
let remote_version = source.transitioned_object.version_id.as_str();
|
||||
let remote_version_state = match source.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => return Ok(None),
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled if remote_version.is_empty() => {
|
||||
TierFreeVersionReceiptVersionState::KnownDisabled
|
||||
}
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull if remote_version == "null" => {
|
||||
TierFreeVersionReceiptVersionState::SuspendedNull
|
||||
}
|
||||
rustfs_filemeta::TransitionVersionState::Exact if !remote_version.is_empty() && remote_version != "null" => {
|
||||
TierFreeVersionReceiptVersionState::Exact
|
||||
}
|
||||
_ => return Ok(None),
|
||||
};
|
||||
|
||||
let Some(backend_identity) = crate::services::tier::tier::tier_destination_id_from_metadata(&source.user_defined)
|
||||
.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
Ok(Some((
|
||||
TierFreeVersionReceiptIdentity {
|
||||
bucket: source.bucket.clone(),
|
||||
logical_name: decode_dir_object(&source.name),
|
||||
tier: source.transitioned_object.tier.clone(),
|
||||
remote_name: source.transitioned_object.name.clone(),
|
||||
remote_version_state,
|
||||
remote_version: source.transitioned_object.version_id.clone(),
|
||||
backend_identity,
|
||||
},
|
||||
TierFreeVersionReceiptPayload {
|
||||
local_free_version_id,
|
||||
mod_time: source.mod_time,
|
||||
},
|
||||
)))
|
||||
}
|
||||
|
||||
impl TierFreeVersionReceiptSink {
|
||||
/// Record one committed free-version cleanup target. Cloned options share
|
||||
/// this sink; tuple-equivalent physical copies collapse to one worker task.
|
||||
/// `false` means the source cannot safely identify a destructive cleanup.
|
||||
pub(crate) fn record(&self, source: &ObjectInfo, local_free_version_id: Uuid) -> io::Result<bool> {
|
||||
let Some((identity, payload)) = tier_free_version_scheduling_receipt_from_source(source, local_free_version_id)? else {
|
||||
return Ok(false);
|
||||
};
|
||||
let mut state = self.inner.lock();
|
||||
let receipts = state
|
||||
.receipts
|
||||
.as_mut()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::BrokenPipe, "tier free-version receipt sink was already drained"))?;
|
||||
receipts.entry(identity).or_insert(payload);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Consume every receipt exactly once. A second drain is a caller bug: it
|
||||
/// could otherwise make two outer wrappers believe they own the same tasks.
|
||||
pub(crate) fn drain(&self) -> io::Result<Vec<ObjectInfo>> {
|
||||
let mut state = self.inner.lock();
|
||||
let receipts = state
|
||||
.receipts
|
||||
.take()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::BrokenPipe, "tier free-version receipt sink was already drained"))?;
|
||||
drop(state);
|
||||
Ok(receipts
|
||||
.into_iter()
|
||||
.map(|(identity, payload)| identity.into_object_info(payload))
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct ObjectOptions {
|
||||
// Use the maximum parity (N/2), used when saving server configuration files
|
||||
@@ -907,12 +716,6 @@ pub struct ObjectOptions {
|
||||
pub skip_rebalancing: bool,
|
||||
pub skip_free_version: bool,
|
||||
|
||||
/// Storage-owned, per-request hand-off for committed tier free-version
|
||||
/// cleanup work. The outer delete wrapper installs and drains it; clones
|
||||
/// below that boundary share the same opaque sink.
|
||||
#[doc(hidden)]
|
||||
pub tier_free_version_receipt_sink: Option<TierFreeVersionReceiptSink>,
|
||||
|
||||
/// Cooperative cancellation for an owned PutObject before authoritative
|
||||
/// rename begins. Storage ignores it after entering the durable commit.
|
||||
#[doc(hidden)]
|
||||
@@ -1048,7 +851,6 @@ impl std::fmt::Debug for ObjectOptions {
|
||||
.field("skip_decommissioned", &self.skip_decommissioned)
|
||||
.field("skip_rebalancing", &self.skip_rebalancing)
|
||||
.field("skip_free_version", &self.skip_free_version)
|
||||
.field("tier_free_version_receipt_sink", &self.tier_free_version_receipt_sink)
|
||||
.field("put_object_cancellation", &self.put_object_cancellation.is_some())
|
||||
.field("scanner_publication_commit_scope", &self.scanner_publication_commit_scope)
|
||||
.field("data_movement", &self.data_movement)
|
||||
@@ -1969,10 +1771,9 @@ impl ObjectInfo {
|
||||
}
|
||||
|
||||
if let Some(data) = &self.checksum {
|
||||
if self.is_encrypted() && get_consistent_str(&self.user_defined, SUFFIX_PLAINTEXT_CHECKSUM) != Some("true") {
|
||||
if self.is_encrypted() {
|
||||
// Object-level encrypted checksum bytes require SSE decrypt material,
|
||||
// unless RustFS marked the stored bytes as plaintext. Do not expose
|
||||
// unmarked bytes as checksum headers here. The
|
||||
// so do not expose them as plaintext checksum headers here. The
|
||||
// `false` multipart flag feeds the response-path COMPOSITE
|
||||
// fallback; callers that need accurate multipart routing must
|
||||
// consult `is_multipart()` instead of this value.
|
||||
@@ -2678,31 +2479,6 @@ mod tests {
|
||||
assert!(checksums.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decrypt_checksums_reads_marked_rustfs_encrypted_object_checksum() {
|
||||
let checksum = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::CRC32, b"encrypted-object")
|
||||
.expect("test checksum should be valid");
|
||||
let checksum_key = checksum.checksum_type.to_string();
|
||||
let expected_checksum = checksum.encoded.clone();
|
||||
let mut user_defined =
|
||||
HashMap::from([(rustfs_utils::http::headers::AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "AES256".to_string())]);
|
||||
rustfs_utils::http::insert_str(&mut user_defined, SUFFIX_PLAINTEXT_CHECKSUM, "true".to_string());
|
||||
assert_eq!(user_defined.get("x-rustfs-internal-plaintext-checksum").map(String::as_str), Some("true"));
|
||||
assert_eq!(user_defined.get("x-minio-internal-plaintext-checksum").map(String::as_str), Some("true"));
|
||||
let info = ObjectInfo {
|
||||
checksum: Some(checksum.to_bytes(&[])),
|
||||
user_defined: Arc::new(user_defined),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let (checksums, is_multipart) = info
|
||||
.decrypt_checksums(0, &HeaderMap::new())
|
||||
.expect("marked RustFS checksum should decode");
|
||||
|
||||
assert!(!is_multipart);
|
||||
assert_eq!(checksums.get(&checksum_key), Some(&expected_checksum));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decrypt_checksums_keeps_encrypted_multipart_flag_false_for_response_paths() {
|
||||
let checksum = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::CRC32, b"encrypted-object")
|
||||
@@ -2820,381 +2596,11 @@ mod tests {
|
||||
assert!(default_cloned.parts.is_empty());
|
||||
}
|
||||
|
||||
fn transitioned_receipt_source(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
remote_version: &str,
|
||||
version_state: rustfs_filemeta::TransitionVersionState,
|
||||
identity_hex: Option<&str>,
|
||||
) -> ObjectInfo {
|
||||
let mut metadata = HashMap::new();
|
||||
if let Some(identity_hex) = identity_hex {
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
identity_hex.to_string(),
|
||||
);
|
||||
}
|
||||
ObjectInfo {
|
||||
bucket: bucket.to_string(),
|
||||
name: object.to_string(),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
user_defined: Arc::new(metadata),
|
||||
transitioned_object: TransitionedObject {
|
||||
name: format!("remote/{object}"),
|
||||
version_id: remote_version.to_string(),
|
||||
tier: "WARM".to_string(),
|
||||
status: TRANSITION_COMPLETE.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: version_state,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_free_version_receipt_matches_persisted_free_version_worker_fields() {
|
||||
let bucket = "receipt-bucket";
|
||||
let object = "archive/object.bin";
|
||||
let source_version_id = Uuid::from_u128(1);
|
||||
let local_free_version_id = Uuid::from_u128(2);
|
||||
let remote_version_id = Uuid::from_u128(3);
|
||||
let source_mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::hours(4);
|
||||
let identity_hex = "ab".repeat(32);
|
||||
let mut source_metadata = HashMap::from([
|
||||
("etag".to_string(), "source-etag".to_string()),
|
||||
("x-amz-meta-private".to_string(), "must-not-enter-receipt".to_string()),
|
||||
]);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut source_metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
identity_hex.clone(),
|
||||
);
|
||||
let source_file_info = FileInfo {
|
||||
volume: bucket.to_string(),
|
||||
name: object.to_string(),
|
||||
version_id: Some(source_version_id),
|
||||
transition_status: TRANSITION_COMPLETE.to_string(),
|
||||
transitioned_objname: "remote/receipt-object".to_string(),
|
||||
transition_tier: "WARM".to_string(),
|
||||
transition_version_id: Some(remote_version_id),
|
||||
transition_version: Some(remote_version_id.to_string()),
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
mod_time: Some(source_mod_time),
|
||||
size: 8192,
|
||||
data_dir: Some(Uuid::from_u128(4)),
|
||||
metadata: source_metadata,
|
||||
..Default::default()
|
||||
};
|
||||
let source = ObjectInfo::from_file_info(&source_file_info, bucket, object, true);
|
||||
let mut persisted = FileMeta::new();
|
||||
persisted
|
||||
.add_version(source_file_info)
|
||||
.expect("transitioned receipt source should be persisted");
|
||||
let mut delete_file_info = FileInfo {
|
||||
volume: bucket.to_string(),
|
||||
name: object.to_string(),
|
||||
version_id: Some(source_version_id),
|
||||
mod_time: Some(source_mod_time + time::Duration::minutes(1)),
|
||||
..Default::default()
|
||||
};
|
||||
delete_file_info.set_tier_free_version_id(&local_free_version_id.to_string());
|
||||
persisted
|
||||
.delete_version(&delete_file_info)
|
||||
.expect("transitioned source delete should create a free-version");
|
||||
|
||||
let encoded = persisted.marshal_msg().expect("free-version metadata should encode");
|
||||
let decoded = FileMeta::load(&encoded).expect("free-version metadata should decode");
|
||||
let persisted_free_version = decoded
|
||||
.get_all_file_info_versions(bucket, object, true)
|
||||
.expect("decoded free-version should produce FileInfo")
|
||||
.versions
|
||||
.into_iter()
|
||||
.find(|version| version.tier_free_version())
|
||||
.expect("decoded metadata should contain the persisted free-version");
|
||||
let persisted_object_info = ObjectInfo::from_file_info(&persisted_free_version, bucket, object, true);
|
||||
|
||||
let sink = TierFreeVersionReceiptSink::new();
|
||||
assert!(
|
||||
sink.record(&source, local_free_version_id)
|
||||
.expect("valid transitioned source should produce a receipt")
|
||||
);
|
||||
let mut receipts = sink.drain().expect("receipt owner should drain exactly once");
|
||||
assert_eq!(receipts.len(), 1);
|
||||
let receipt = receipts.pop().expect("one receipt should be present");
|
||||
|
||||
assert_eq!(receipt.bucket, persisted_object_info.bucket);
|
||||
assert_eq!(receipt.name, persisted_object_info.name);
|
||||
assert_eq!(receipt.version_id, persisted_object_info.version_id);
|
||||
assert_eq!(receipt.mod_time, persisted_object_info.mod_time);
|
||||
assert_eq!(receipt.delete_marker, persisted_object_info.delete_marker);
|
||||
assert_eq!(receipt.transitioned_object.name, persisted_object_info.transitioned_object.name);
|
||||
assert_eq!(
|
||||
receipt.transitioned_object.version_id,
|
||||
persisted_object_info.transitioned_object.version_id
|
||||
);
|
||||
assert_eq!(receipt.transitioned_object.tier, persisted_object_info.transitioned_object.tier);
|
||||
assert_eq!(
|
||||
receipt.transitioned_object.free_version,
|
||||
persisted_object_info.transitioned_object.free_version
|
||||
);
|
||||
assert_eq!(receipt.transitioned_object.status, persisted_object_info.transitioned_object.status);
|
||||
assert_eq!(receipt.transition_version_state, persisted_object_info.transition_version_state);
|
||||
assert_eq!(
|
||||
crate::services::tier::tier::tier_destination_id_from_metadata(&receipt.user_defined)
|
||||
.expect("receipt identity should decode"),
|
||||
crate::services::tier::tier::tier_destination_id_from_metadata(&persisted_object_info.user_defined)
|
||||
.expect("persisted identity should decode")
|
||||
);
|
||||
assert_eq!(
|
||||
receipt.user_defined.len(),
|
||||
2,
|
||||
"receipt should carry only the two compatibility identity keys"
|
||||
);
|
||||
assert_eq!(
|
||||
receipt.user_defined.get("x-rustfs-internal-transition-tier-destination-id"),
|
||||
Some(&identity_hex)
|
||||
);
|
||||
assert_eq!(
|
||||
receipt.user_defined.get("x-minio-internal-transition-tier-destination-id"),
|
||||
Some(&identity_hex)
|
||||
);
|
||||
assert!(!receipt.user_defined.contains_key("x-amz-meta-private"));
|
||||
assert_eq!(receipt.size, 0);
|
||||
assert_eq!(receipt.actual_size, 0);
|
||||
assert!(receipt.parts.is_empty());
|
||||
assert!(receipt.etag.is_none());
|
||||
assert!(receipt.checksum.is_none());
|
||||
assert!(receipt.data_dir.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_free_version_receipt_sink_deduplicates_remote_target_and_drains_once() {
|
||||
let identity_hex = "11".repeat(32);
|
||||
let source = transitioned_receipt_source(
|
||||
"bucket",
|
||||
"object",
|
||||
"remote-version",
|
||||
rustfs_filemeta::TransitionVersionState::Exact,
|
||||
Some(&identity_hex),
|
||||
);
|
||||
let other_object = transitioned_receipt_source(
|
||||
"bucket",
|
||||
"other-object",
|
||||
"remote-version",
|
||||
rustfs_filemeta::TransitionVersionState::Exact,
|
||||
Some(&identity_hex),
|
||||
);
|
||||
let sink = TierFreeVersionReceiptSink::new();
|
||||
let clone = sink.clone();
|
||||
|
||||
assert!(
|
||||
sink.record(&source, Uuid::from_u128(10))
|
||||
.expect("first physical receipt should record")
|
||||
);
|
||||
assert!(
|
||||
clone
|
||||
.record(&source, Uuid::from_u128(11))
|
||||
.expect("tuple-equivalent physical receipt should be represented")
|
||||
);
|
||||
assert!(
|
||||
clone
|
||||
.record(&other_object, Uuid::from_u128(12))
|
||||
.expect("a different logical key should retain its own task")
|
||||
);
|
||||
|
||||
let mut receipts = sink.drain().expect("owner should drain shared receipts");
|
||||
receipts.sort_by(|left, right| left.name.cmp(&right.name));
|
||||
assert_eq!(receipts.len(), 2);
|
||||
assert_eq!(receipts[0].name, "object");
|
||||
assert_eq!(receipts[0].version_id, Some(Uuid::from_u128(10)));
|
||||
assert_eq!(receipts[1].name, "other-object");
|
||||
assert_eq!(receipts[1].version_id, Some(Uuid::from_u128(12)));
|
||||
assert_eq!(
|
||||
clone.drain().expect_err("a shared sink must drain only once").kind(),
|
||||
io::ErrorKind::BrokenPipe
|
||||
);
|
||||
assert_eq!(
|
||||
clone
|
||||
.record(&source, Uuid::from_u128(13))
|
||||
.expect_err("recording after drain must fail")
|
||||
.kind(),
|
||||
io::ErrorKind::BrokenPipe
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_free_version_receipt_identity_covers_every_destructive_dimension() {
|
||||
let identity_hex = "44".repeat(32);
|
||||
let baseline = transitioned_receipt_source(
|
||||
"bucket",
|
||||
"directory/",
|
||||
"remote-version",
|
||||
rustfs_filemeta::TransitionVersionState::Exact,
|
||||
Some(&identity_hex),
|
||||
);
|
||||
let mut encoded_duplicate = baseline.clone();
|
||||
encoded_duplicate.name = "directory__XLDIR__".to_string();
|
||||
|
||||
let mut variants = Vec::new();
|
||||
let mut changed = baseline.clone();
|
||||
changed.bucket = "other-bucket".to_string();
|
||||
variants.push(changed);
|
||||
let mut changed = baseline.clone();
|
||||
changed.name = "other-directory/".to_string();
|
||||
variants.push(changed);
|
||||
let mut changed = baseline.clone();
|
||||
changed.transitioned_object.tier = "COLD".to_string();
|
||||
variants.push(changed);
|
||||
let mut changed = baseline.clone();
|
||||
changed.transitioned_object.name = "remote/other-directory/".to_string();
|
||||
variants.push(changed);
|
||||
let mut changed = baseline.clone();
|
||||
changed.transitioned_object.version_id = "other-remote-version".to_string();
|
||||
variants.push(changed);
|
||||
let mut changed = baseline.clone();
|
||||
changed.transition_version_state = rustfs_filemeta::TransitionVersionState::SuspendedNull;
|
||||
changed.transitioned_object.version_id = "null".to_string();
|
||||
variants.push(changed);
|
||||
let mut changed = baseline.clone();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
Arc::make_mut(&mut changed.user_defined),
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
"55".repeat(32),
|
||||
);
|
||||
variants.push(changed);
|
||||
|
||||
let sink = TierFreeVersionReceiptSink::new();
|
||||
assert!(
|
||||
sink.record(&baseline, Uuid::from_u128(20))
|
||||
.expect("baseline receipt should record")
|
||||
);
|
||||
assert!(
|
||||
sink.record(&encoded_duplicate, Uuid::from_u128(21))
|
||||
.expect("the encoded spelling of one logical key should deduplicate")
|
||||
);
|
||||
for (offset, variant) in variants.iter().enumerate() {
|
||||
assert!(
|
||||
sink.record(variant, Uuid::from_u128(30 + offset as u128))
|
||||
.expect("each distinct cleanup identity should record")
|
||||
);
|
||||
}
|
||||
|
||||
let receipts = sink.drain().expect("identity matrix should drain once");
|
||||
assert_eq!(receipts.len(), 8, "every destructive identity dimension must prevent deduplication");
|
||||
let baseline_receipt = receipts
|
||||
.iter()
|
||||
.find(|receipt| {
|
||||
receipt.bucket == "bucket"
|
||||
&& receipt.name == "directory/"
|
||||
&& receipt.transitioned_object.tier == "WARM"
|
||||
&& receipt.transitioned_object.name == "remote/directory/"
|
||||
&& receipt.transitioned_object.version_id == "remote-version"
|
||||
&& receipt.transition_version_state == rustfs_filemeta::TransitionVersionState::Exact
|
||||
&& crate::services::tier::tier::tier_destination_id_from_metadata(&receipt.user_defined)
|
||||
.is_ok_and(|identity| identity == Some([0x44; 32]))
|
||||
})
|
||||
.expect("baseline cleanup identity should remain present");
|
||||
assert_eq!(
|
||||
baseline_receipt.version_id,
|
||||
Some(Uuid::from_u128(20)),
|
||||
"deduplication must retain the first UUID"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_free_version_receipt_source_validation_fails_closed() {
|
||||
let identity_hex = "22".repeat(32);
|
||||
for (state, remote_version) in [
|
||||
(rustfs_filemeta::TransitionVersionState::KnownDisabled, ""),
|
||||
(rustfs_filemeta::TransitionVersionState::SuspendedNull, "null"),
|
||||
(rustfs_filemeta::TransitionVersionState::Exact, "opaque-version"),
|
||||
] {
|
||||
let source = transitioned_receipt_source("bucket", "object", remote_version, state, Some(&identity_hex));
|
||||
assert!(
|
||||
TierFreeVersionReceiptSink::new()
|
||||
.record(&source, Uuid::new_v4())
|
||||
.expect("canonical remote-version state should be eligible"),
|
||||
"state={state:?} remote_version={remote_version:?}"
|
||||
);
|
||||
}
|
||||
|
||||
let unknown = transitioned_receipt_source(
|
||||
"bucket",
|
||||
"object",
|
||||
"opaque-version",
|
||||
rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
Some(&identity_hex),
|
||||
);
|
||||
assert!(
|
||||
!TierFreeVersionReceiptSink::new()
|
||||
.record(&unknown, Uuid::new_v4())
|
||||
.expect("unknown remote version state should defer to recovery")
|
||||
);
|
||||
let missing_identity = transitioned_receipt_source(
|
||||
"bucket",
|
||||
"object",
|
||||
"opaque-version",
|
||||
rustfs_filemeta::TransitionVersionState::Exact,
|
||||
None,
|
||||
);
|
||||
assert!(
|
||||
!TierFreeVersionReceiptSink::new()
|
||||
.record(&missing_identity, Uuid::new_v4())
|
||||
.expect("missing durable identity should defer to recovery")
|
||||
);
|
||||
let invalid_exact = transitioned_receipt_source(
|
||||
"bucket",
|
||||
"object",
|
||||
"",
|
||||
rustfs_filemeta::TransitionVersionState::Exact,
|
||||
Some(&identity_hex),
|
||||
);
|
||||
assert!(
|
||||
!TierFreeVersionReceiptSink::new()
|
||||
.record(&invalid_exact, Uuid::new_v4())
|
||||
.expect("conflicting remote state should defer to recovery")
|
||||
);
|
||||
|
||||
let mut conflicting = transitioned_receipt_source(
|
||||
"bucket",
|
||||
"object",
|
||||
"opaque-version",
|
||||
rustfs_filemeta::TransitionVersionState::Exact,
|
||||
Some(&identity_hex),
|
||||
);
|
||||
Arc::make_mut(&mut conflicting.user_defined)
|
||||
.insert("x-minio-internal-transition-tier-destination-id".to_string(), "33".repeat(32));
|
||||
assert_eq!(
|
||||
TierFreeVersionReceiptSink::new()
|
||||
.record(&conflicting, Uuid::new_v4())
|
||||
.expect_err("conflicting identity aliases must fail closed")
|
||||
.kind(),
|
||||
io::ErrorKind::InvalidData
|
||||
);
|
||||
|
||||
let valid = transitioned_receipt_source(
|
||||
"bucket",
|
||||
"object",
|
||||
"opaque-version",
|
||||
rustfs_filemeta::TransitionVersionState::Exact,
|
||||
Some(&identity_hex),
|
||||
);
|
||||
assert_eq!(
|
||||
TierFreeVersionReceiptSink::new()
|
||||
.record(&valid, Uuid::nil())
|
||||
.expect_err("nil local free-version identity must be rejected")
|
||||
.kind(),
|
||||
io::ErrorKind::InvalidInput
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_options_default_does_not_allocate_lifecycle_delete_all_journal() {
|
||||
let mut opts = ObjectOptions::default();
|
||||
|
||||
assert!(opts.lifecycle_delete_all_journal().is_none());
|
||||
assert!(opts.tier_free_version_receipt_sink.is_none());
|
||||
opts.ensure_lifecycle_delete_all_journal();
|
||||
assert!(opts.lifecycle_delete_all_journal().is_some());
|
||||
}
|
||||
|
||||
@@ -127,7 +127,7 @@ use rustfs_filemeta::{
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::{
|
||||
DriveState, HealAdmissionResult, HealChannelPriority, HealItemType, HealOpts, HealRequestSource, HealScanMode,
|
||||
send_heal_replacement_disk, send_heal_request_with_admission,
|
||||
send_heal_disk, send_heal_request_with_admission,
|
||||
};
|
||||
use rustfs_io_metrics::{
|
||||
record_object_lock_diag_acquire_duration, record_object_lock_diag_enabled, record_object_lock_diag_hold_duration,
|
||||
|
||||
@@ -22,10 +22,12 @@
|
||||
use super::super::{
|
||||
Arc, DiskError, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, Endpoint, Error, FormatV3, HealChannelPriority, LockResult,
|
||||
NamespaceLock, NamespaceLockWrapper, ObjectKey, Result, SetDisks, StorageError, debug, disk, info, load_format_erasure,
|
||||
send_heal_replacement_disk, warn,
|
||||
send_heal_disk, warn,
|
||||
};
|
||||
use crate::disk::DiskAPI;
|
||||
use crate::disk::health_state::DriveMembershipSnapshot;
|
||||
use crate::disk::{DiskAPI, new_disk};
|
||||
#[cfg(test)]
|
||||
use crate::disk::new_disk;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use rand::prelude::SliceRandom;
|
||||
#[cfg(test)]
|
||||
@@ -354,28 +356,11 @@ impl SetDisks {
|
||||
Ok(res) => res,
|
||||
Err(e) => {
|
||||
warn!("renew_disk: connect_endpoint err {:?}", &e);
|
||||
if !matches!(e, DiskError::UnformattedDisk | DiskError::Io(_)) {
|
||||
return;
|
||||
if ep.is_local && e == DiskError::UnformattedDisk {
|
||||
info!("renew_disk unformatteddisk will trigger heal_disk, {:?}", ep);
|
||||
let set_disk_id = format!("pool_{}_set_{}", ep.pool_idx, ep.set_idx);
|
||||
let _ = send_heal_disk(set_disk_id, Some(HealChannelPriority::Normal)).await;
|
||||
}
|
||||
|
||||
let attached = match self.attach_unformatted_replacement_disk(ep).await {
|
||||
Ok(attached) => attached,
|
||||
Err(err) => {
|
||||
warn!(endpoint = %ep, error = ?err, "renew_disk: unformatted replacement probe failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
if !attached {
|
||||
return;
|
||||
}
|
||||
|
||||
info!("renew_disk attached unformatted replacement and will trigger heal_disk, {:?}", ep);
|
||||
let (Ok(pool_index), Ok(set_index)) = (usize::try_from(ep.pool_idx), usize::try_from(ep.set_idx)) else {
|
||||
warn!("renew_disk: replacement target has invalid pool or set index, {:?}", ep);
|
||||
return;
|
||||
};
|
||||
let _ =
|
||||
send_heal_replacement_disk(pool_index, set_index, ep.to_string(), Some(HealChannelPriority::Normal)).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
@@ -427,60 +412,6 @@ impl SetDisks {
|
||||
disk_lock[disk_idx] = Some(new_disk);
|
||||
}
|
||||
|
||||
/// Attach a replacement target only after proving that the exact local slot
|
||||
/// is present and unformatted. A health-checked reconnect may reject a
|
||||
/// blank target before it reaches the format-heal path; that target still
|
||||
/// has to be visible in this set for the formatter to claim it safely.
|
||||
async fn attach_unformatted_replacement_disk(&self, ep: &Endpoint) -> disk::error::Result<bool> {
|
||||
if !ep.is_local
|
||||
|| usize::try_from(ep.pool_idx).ok() != Some(self.pool_index)
|
||||
|| usize::try_from(ep.set_idx).ok() != Some(self.set_index)
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let Some(disk_idx) = self.set_endpoints.iter().position(|candidate| candidate == ep) else {
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
let replacement = new_disk(
|
||||
ep,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
match load_format_erasure(&replacement, false).await {
|
||||
Err(DiskError::UnformattedDisk) => {}
|
||||
Ok(_) => return Ok(false),
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
{
|
||||
let mut disks = self.disks.write().await;
|
||||
if disks[disk_idx].as_ref().is_some_and(|existing| existing.endpoint() != *ep) {
|
||||
warn!(endpoint = %ep, disk_idx, "renew_disk rejected unformatted replacement for an occupied foreign slot");
|
||||
return Ok(false);
|
||||
}
|
||||
disks[disk_idx] = Some(replacement.clone());
|
||||
}
|
||||
|
||||
let local_disk_map = runtime_sources::local_disk_map_handle();
|
||||
local_disk_map
|
||||
.write()
|
||||
.await
|
||||
.insert(replacement.endpoint().to_string(), Some(replacement.clone()));
|
||||
|
||||
if runtime_sources::setup_is_dist_erasure().await {
|
||||
let local_disk_set_drives = runtime_sources::local_disk_set_drives_handle();
|
||||
let mut local_set_drives = local_disk_set_drives.write().await;
|
||||
local_set_drives[self.pool_index][self.set_index][disk_idx] = Some(replacement);
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn find_disk_index(&self, fm: &FormatV3) -> Result<(usize, usize)> {
|
||||
self.format.check_other(fm)?;
|
||||
|
||||
@@ -848,75 +779,6 @@ mod tests {
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renew_disk_attaches_only_a_verified_local_unformatted_replacement() {
|
||||
let disk_count = 4;
|
||||
let format = FormatV3::new(1, disk_count);
|
||||
let mut temp_dirs = Vec::with_capacity(disk_count);
|
||||
let mut endpoints = Vec::with_capacity(disk_count);
|
||||
let mut disks = Vec::with_capacity(disk_count);
|
||||
|
||||
for disk_idx in 0..disk_count - 1 {
|
||||
let (temp_dir, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
|
||||
temp_dirs.push(temp_dir);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
|
||||
let replacement_dir = tempfile::tempdir().expect("replacement tempdir should be created");
|
||||
let mut replacement_endpoint =
|
||||
Endpoint::try_from(replacement_dir.path().to_str().expect("replacement path should be utf8"))
|
||||
.expect("replacement endpoint should parse");
|
||||
replacement_endpoint.set_pool_index(0);
|
||||
replacement_endpoint.set_set_index(0);
|
||||
replacement_endpoint.set_disk_index(disk_count - 1);
|
||||
temp_dirs.push(replacement_dir);
|
||||
endpoints.push(replacement_endpoint.clone());
|
||||
disks.push(None);
|
||||
|
||||
let set_disks = SetDisks::new(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(disks)),
|
||||
disk_count,
|
||||
disk_count / 2,
|
||||
0,
|
||||
0,
|
||||
endpoints,
|
||||
format,
|
||||
Vec::new(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
set_disks
|
||||
.attach_unformatted_replacement_disk(&replacement_endpoint)
|
||||
.await
|
||||
.expect("a blank local replacement should be admitted")
|
||||
);
|
||||
|
||||
let attached = set_disks.get_disks_internal().await;
|
||||
let replacement = attached[disk_count - 1]
|
||||
.as_ref()
|
||||
.expect("the verified replacement must occupy its exact set slot");
|
||||
assert_eq!(replacement.endpoint(), replacement_endpoint);
|
||||
assert!(
|
||||
!replacement.health_check_enabled_for_test(),
|
||||
"the blank replacement must not start health checks before it receives a format"
|
||||
);
|
||||
assert_eq!(
|
||||
load_format_erasure(replacement, false).await.unwrap_err(),
|
||||
DiskError::UnformattedDisk,
|
||||
"only a still-unformatted replacement may be attached by the fallback"
|
||||
);
|
||||
|
||||
runtime_sources::local_disk_map_handle()
|
||||
.write()
|
||||
.await
|
||||
.remove(&replacement_endpoint.to_string());
|
||||
drop(set_disks);
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renew_disk_rejects_a_format_from_another_slot_or_cluster() {
|
||||
let disk_count = 3;
|
||||
|
||||
@@ -66,6 +66,7 @@ use crate::disk::new_disk;
|
||||
use crate::multipart_listing::paginate_multipart_listing;
|
||||
#[cfg(test)]
|
||||
use crate::object_api::ObjectLockConfigSnapshot;
|
||||
use crate::set_disk::core::io_primitives::finish_rename_tail_heal;
|
||||
use crate::set_disk::mem;
|
||||
use crate::set_disk::metadata_sys;
|
||||
use crate::set_disk::runtime_sources;
|
||||
@@ -3123,11 +3124,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let commit_object_lock_guard = object_lock_guard.take();
|
||||
let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
|
||||
let commit_decommission_capacity_guard = decommission_capacity_guard.take();
|
||||
// CompleteMultipartUpload is an S3 publication boundary: after a
|
||||
// successful response, the object must be immediately readable and
|
||||
// usable as a CopyObject source. Do not return on rename quorum while
|
||||
// a tail owner may still hold the object guard and finish shard moves.
|
||||
let commit_allows_early_ack = false;
|
||||
let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation())
|
||||
&& (commit_object_lock_guard.is_some() || commit_decommission_object_lock_guard.is_some());
|
||||
let detach_commit_owner = commit_allows_early_ack || upload_guard.is_some() || quota_mutation_fence;
|
||||
let commit = async move {
|
||||
let mut _object_lock_guard = commit_object_lock_guard;
|
||||
@@ -3258,15 +3256,105 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
commit_allows_early_ack,
|
||||
)
|
||||
.await;
|
||||
let mut rename_guard_release = None;
|
||||
let mut needs_immediate_heal = false;
|
||||
let mut tail_owns_staging_cleanup = false;
|
||||
if let Ok(rename_commit) = rename_result.as_mut() {
|
||||
commit_set.record_capacity_scope_if_needed(commit_capacity_scope_token, &rename_commit.capacity_disks);
|
||||
debug_assert!(
|
||||
rename_commit.tail_drain.is_none(),
|
||||
"multipart completion disables early ACK and must not detach a rename tail"
|
||||
);
|
||||
// Install the tail watcher before any post-commit await. The
|
||||
// latch keeps namespace guards through their prior handoff point.
|
||||
needs_immediate_heal = rename_commit.needs_immediate_heal();
|
||||
if let Some(rename_tail_drain) = rename_commit.tail_drain.take() {
|
||||
tail_owns_staging_cleanup = true;
|
||||
let mut request = rustfs_heal_contracts::heal_channel::create_heal_request_with_options(
|
||||
commit_bucket.clone(),
|
||||
Some(commit_object.clone()),
|
||||
false,
|
||||
Some(HealChannelPriority::Normal),
|
||||
Some(commit_set.pool_index),
|
||||
Some(commit_set.set_index),
|
||||
);
|
||||
request.object_version_id = fi
|
||||
.version_id
|
||||
.or_else(|| commit_version_suspended.then(Uuid::nil))
|
||||
.map(|version_id| version_id.to_string());
|
||||
let object_lock_guard = _object_lock_guard.take();
|
||||
let upload_guard = _upload_guard.take();
|
||||
let decommission_object_lock_guard = _decommission_object_lock_guard.take();
|
||||
let decommission_capacity_guard = _decommission_capacity_guard.take();
|
||||
let cleanup_bucket = commit_bucket.clone();
|
||||
let cleanup_object = commit_object.clone();
|
||||
let heal_set = commit_set.clone();
|
||||
let cleanup_set = commit_set.clone();
|
||||
let committed_data_dir = fi.data_dir;
|
||||
let cleanup_parts = parts.clone();
|
||||
let cleanup_upload_path = commit_upload_id_path.clone();
|
||||
let cleanup_upload_id = commit_upload_id.clone();
|
||||
let fence_disks = commit_disks.clone();
|
||||
let fence_tokens = quota_fence_tokens.clone();
|
||||
let fence_bucket = commit_bucket.clone();
|
||||
let fence_object = commit_object.clone();
|
||||
let (guard_release_tx, guard_release_rx) = tokio::sync::oneshot::channel();
|
||||
rename_guard_release = Some(guard_release_tx);
|
||||
tokio::spawn(finish_rename_tail_heal(
|
||||
rename_tail_drain,
|
||||
guard_release_rx,
|
||||
(
|
||||
object_lock_guard,
|
||||
upload_guard,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
request,
|
||||
move || async move {
|
||||
if quota_mutation_fence {
|
||||
let _ = SetDisks::release_quota_mutation_fences(
|
||||
&fence_disks,
|
||||
&fence_tokens,
|
||||
&fence_bucket,
|
||||
&fence_object,
|
||||
write_quorum,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
},
|
||||
move |(object_lock_guard, upload_guard, decommission_object_lock_guard, decommission_capacity_guard),
|
||||
targets| async move {
|
||||
drop(object_lock_guard);
|
||||
cleanup_set.cleanup_multipart_path(&cleanup_parts).await;
|
||||
cleanup_set
|
||||
.cleanup_rename_tail(
|
||||
targets,
|
||||
&cleanup_bucket,
|
||||
&cleanup_object,
|
||||
committed_data_dir,
|
||||
transaction_epoch,
|
||||
)
|
||||
.await;
|
||||
if let Err(err) = cleanup_set
|
||||
.delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &cleanup_upload_path, write_quorum)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
bucket = %cleanup_bucket,
|
||||
object = %cleanup_object,
|
||||
upload_id = %cleanup_upload_id,
|
||||
error = ?err,
|
||||
"completed multipart upload staging cleanup did not reach write quorum"
|
||||
);
|
||||
}
|
||||
drop(upload_guard);
|
||||
drop(decommission_object_lock_guard);
|
||||
drop(decommission_capacity_guard);
|
||||
},
|
||||
|request| async move { heal_set.submit_rename_tail_heal(request).await },
|
||||
));
|
||||
}
|
||||
}
|
||||
drop(_decommission_capacity_guard.take());
|
||||
if quota_mutation_fence {
|
||||
if !tail_owns_staging_cleanup {
|
||||
drop(_decommission_capacity_guard.take());
|
||||
}
|
||||
if quota_mutation_fence && !tail_owns_staging_cleanup {
|
||||
let _ = SetDisks::release_quota_mutation_fences(
|
||||
&commit_disks,
|
||||
"a_fence_tokens,
|
||||
@@ -3283,7 +3371,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
Ok(result) => result,
|
||||
Err(err) => return Err(err.into()),
|
||||
};
|
||||
let needs_immediate_heal = rename_commit.needs_immediate_heal();
|
||||
let op_old_dir = rename_commit.data_dir;
|
||||
let cleanup_disks = rename_commit.cleanup_disks;
|
||||
let committed_file_info = rename_commit.committed_file_info;
|
||||
@@ -3326,6 +3413,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// parts are swept by a retried completion or upload GC (rustfs/backlog#946).
|
||||
// Compiles to a no-op outside `#[cfg(test)]`.
|
||||
if crash_inject::should_crash_at(CrashPoint::MultipartAfterCommitBeforePartsCleanup, &commit_object) {
|
||||
if let Some(release) = rename_guard_release.take() {
|
||||
let _ = release.send(false);
|
||||
}
|
||||
return Err(StorageError::Unexpected);
|
||||
}
|
||||
|
||||
@@ -3341,7 +3431,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.invalidate_get_object_metadata_cache(&commit_bucket, &commit_object)
|
||||
.await;
|
||||
|
||||
drop(_object_lock_guard.take()); // release the object lock before multipart cleanup IO.
|
||||
if let Some(release) = rename_guard_release.take() {
|
||||
let _ = release.send(true);
|
||||
}
|
||||
drop(_object_lock_guard.take()); // release the object lock before multipart cleanup tail IO.
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::AfterObjectPublication).await;
|
||||
@@ -3354,7 +3447,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// parts; deleting them before the commit would strand the upload
|
||||
// permanently. This mirrors the "clean up only after commit" pattern
|
||||
// already used for the old data-dir GC and the upload-dir delete_all below.
|
||||
commit_set.cleanup_multipart_path(&parts).await;
|
||||
if !tail_owns_staging_cleanup {
|
||||
commit_set.cleanup_multipart_path(&parts).await;
|
||||
}
|
||||
|
||||
if let Some(old_dir) = op_old_dir {
|
||||
// backlog#898: best-effort reclaim of the dereferenced old data dir.
|
||||
@@ -3385,9 +3480,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::AfterRename).await;
|
||||
|
||||
if let Err(err) = commit_set
|
||||
.delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &commit_upload_id_path, write_quorum)
|
||||
.await
|
||||
if !tail_owns_staging_cleanup
|
||||
&& let Err(err) = commit_set
|
||||
.delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &commit_upload_id_path, write_quorum)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
bucket = %commit_bucket,
|
||||
@@ -3914,7 +4010,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(capacity_dirty_scope)]
|
||||
async fn complete_multipart_waits_for_tail_before_releasing_guards_and_marking_capacity() {
|
||||
async fn early_ack_multipart_holds_quota_fences_and_re_marks_capacity_after_tail_drain() {
|
||||
use rustfs_object_capacity::capacity_scope::drain_global_dirty_scopes;
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
@@ -3960,9 +4056,10 @@ mod tests {
|
||||
.collect::<HashSet<_>>();
|
||||
let _ = drain_global_dirty_scopes();
|
||||
|
||||
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let complete_store = Arc::clone(&set_disks);
|
||||
let mut complete = tokio::spawn(async move {
|
||||
let complete = tokio::spawn(async move {
|
||||
let mut opts = ObjectOptions::default();
|
||||
assert!(opts.set_quota_admission(0, u64::MAX));
|
||||
complete_store
|
||||
@@ -3972,15 +4069,20 @@ mod tests {
|
||||
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("multipart completion should pause one tail disk during rename");
|
||||
let cleanup_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_CLEANUP);
|
||||
complete
|
||||
.await
|
||||
.expect("early-ACK multipart task should join before tail release")
|
||||
.expect("multipart completion should return after write quorum");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
|
||||
"multipart completion must not publish success while a tail rename is still paused"
|
||||
rename_tasks.running() >= 1,
|
||||
"the paused multipart tail disk must remain in flight after quorum ACK"
|
||||
);
|
||||
|
||||
let initial = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
|
||||
assert!(
|
||||
initial.is_empty(),
|
||||
"capacity must not be marked as committed before the full multipart rename finishes"
|
||||
expected.is_subset(&initial),
|
||||
"the multipart quorum ACK must mark every candidate disk dirty"
|
||||
);
|
||||
|
||||
let abort_store = Arc::clone(&set_disks);
|
||||
@@ -3990,7 +4092,7 @@ mod tests {
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(2).await;
|
||||
assert!(!abort.is_finished(), "the in-flight completion must retain the multipart upload guard");
|
||||
assert!(!abort.is_finished(), "the detached tail owner must retain the multipart upload guard");
|
||||
|
||||
let retained_staging = futures::future::join_all(
|
||||
disk_stores
|
||||
@@ -4015,20 +4117,25 @@ mod tests {
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(object_attempt).await;
|
||||
assert!(!object_probe.is_finished(), "the in-flight completion must retain the object guard");
|
||||
assert!(!object_probe.is_finished(), "the detached tail owner must retain the object guard");
|
||||
|
||||
rename_barrier.release();
|
||||
complete
|
||||
.await
|
||||
.expect("multipart task should join after tail release")
|
||||
.expect("multipart completion should return after every tail rename finishes");
|
||||
object_probe
|
||||
.await
|
||||
.expect("object guard probe should join after completion releases")
|
||||
.expect("object guard probe should acquire after completion releases");
|
||||
.expect("object guard probe should join after the tail releases")
|
||||
.expect("object guard probe should acquire after the tail releases");
|
||||
tokio::time::timeout(Duration::from_secs(30), cleanup_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("the multipart tail should pause before reclaiming its old body");
|
||||
let after_tail = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
|
||||
assert!(
|
||||
expected.is_subset(&after_tail),
|
||||
"the multipart rename tail must re-mark capacity after the first scope was drained"
|
||||
);
|
||||
cleanup_barrier.release();
|
||||
let abort_err = abort
|
||||
.await
|
||||
.expect("abort task should join after completion releases")
|
||||
.expect("abort task should join after the tail releases")
|
||||
.expect_err("the committed upload should no longer exist");
|
||||
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
|
||||
|
||||
@@ -4041,7 +4148,7 @@ mod tests {
|
||||
let after_cleanup = drain_global_dirty_scopes().into_iter().collect::<HashSet<_>>();
|
||||
assert!(
|
||||
expected.is_subset(&after_cleanup),
|
||||
"the completed multipart commit must mark every candidate disk dirty"
|
||||
"the multipart tail cleanup must re-mark capacity after its preceding scope was drained"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
@@ -4176,26 +4283,23 @@ mod tests {
|
||||
],
|
||||
async {
|
||||
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let complete_store = Arc::clone(&set_disks);
|
||||
let mut complete = tokio::spawn(async move {
|
||||
complete_store
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("fenced multipart completion should commit with a live proof");
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("multipart completion should pause one tail disk during rename");
|
||||
.expect("multipart completion should leave one rename tail in flight after quorum ACK");
|
||||
let disks = disk_stores.clone();
|
||||
let mut epochs = tokio::spawn(async move { object_transaction_epochs(&disks, bucket, object).await });
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
|
||||
"fenced multipart completion must wait for every rename tail before returning"
|
||||
tokio::time::timeout(Duration::from_millis(100), &mut epochs).await.is_err(),
|
||||
"epoch read-back should wait for the lagging rename tail"
|
||||
);
|
||||
rename_barrier.release();
|
||||
complete
|
||||
.await
|
||||
.expect("fenced multipart task should join after tail release")
|
||||
.expect("fenced multipart completion should commit with a live proof");
|
||||
object_transaction_epochs(&disk_stores, bucket, object).await
|
||||
epochs.await.expect("epoch read-back should finish after the rename tail")
|
||||
},
|
||||
)
|
||||
.await;
|
||||
@@ -8288,18 +8392,29 @@ mod tests {
|
||||
let new = payload(0xC3);
|
||||
let (u_new, parts_new) = stage_upload(&set_disks, bucket, object, &new).await;
|
||||
let parts_retry = parts_new.clone();
|
||||
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
crash_inject::arm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
|
||||
let crashed = complete(&set_disks, bucket, object, &u_new, parts_new).await;
|
||||
assert!(
|
||||
matches!(crashed, Err(StorageError::Unexpected)),
|
||||
"the armed post-commit crash point must be the failure that surfaced, got {crashed:?}"
|
||||
);
|
||||
assert!(rename_tasks.running() >= 1, "the crash must interrupt an actual early-ACK tail handoff");
|
||||
crash_inject::disarm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
|
||||
rename_barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while rename_tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("the crash-interrupted rename tail should drain after release");
|
||||
drop(
|
||||
set_disks
|
||||
.acquire_write_lock_diag("post_commit_crash_tail_probe", bucket, object)
|
||||
.await
|
||||
.expect("the failed post-commit completion should release its object guard"),
|
||||
.expect("the crash-interrupted tail should release its object guard"),
|
||||
);
|
||||
|
||||
// The commit landed: the new version reads back whole and correct.
|
||||
@@ -8372,18 +8487,29 @@ mod tests {
|
||||
|
||||
let new = payload(0x52);
|
||||
let (u_new, parts_new) = stage_upload(&set_disks, bucket, object, &new).await;
|
||||
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
crash_inject::arm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
|
||||
let crashed = complete(&set_disks, bucket, object, &u_new, parts_new).await;
|
||||
assert!(
|
||||
matches!(crashed, Err(StorageError::Unexpected)),
|
||||
"the post-commit crash point must surface as unexpected, got {crashed:?}"
|
||||
);
|
||||
assert!(rename_tasks.running() >= 1, "the crash must interrupt an actual early-ACK tail handoff");
|
||||
crash_inject::disarm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
|
||||
rename_barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while rename_tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("the crash-interrupted rename tail should drain after release");
|
||||
drop(
|
||||
set_disks
|
||||
.acquire_write_lock_diag("post_commit_receipt_tail_probe", bucket, object)
|
||||
.await
|
||||
.expect("the failed post-commit completion should release its object guard"),
|
||||
.expect("the crash-interrupted tail should release its object guard"),
|
||||
);
|
||||
|
||||
let (body, _) = read_object(&set_disks, bucket, object).await;
|
||||
@@ -8397,8 +8523,8 @@ mod tests {
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
receipts, 4,
|
||||
"the completed rename must persist old-data cleanup receipts on every disk before surfacing the post-commit crash"
|
||||
receipts, 3,
|
||||
"the committed quorum must persist receipts while the crash-interrupted tail preserves staging"
|
||||
);
|
||||
|
||||
let restarted_endpoints = temp_dirs
|
||||
@@ -8444,12 +8570,12 @@ mod tests {
|
||||
.reconcile_old_data_cleanup_receipts(bucket, object)
|
||||
.await
|
||||
.expect("restart receipt reconciliation should succeed");
|
||||
assert_eq!(removed, 4, "restart receipt reconciliation should delete every committed target");
|
||||
assert_eq!(removed, 3, "restart receipt reconciliation should delete the committed quorum's targets");
|
||||
let reclaimed = restarted_set
|
||||
.reclaim_orphan_data_dirs(bucket, object)
|
||||
.await
|
||||
.expect("restart orphan reconciliation should succeed");
|
||||
assert_eq!(reclaimed, 0, "the post-commit crash should leave no receipt-less late commit orphan");
|
||||
assert_eq!(reclaimed, 1, "the late commit without a receipt must remain reclaimable as an orphan");
|
||||
for disk in &reloaded {
|
||||
assert!(
|
||||
!data_dir_exists(disk, bucket, object, old_dir).await,
|
||||
|
||||
@@ -217,8 +217,7 @@ use crate::bucket::lifecycle::{
|
||||
use crate::bucket::quota::reservation;
|
||||
use crate::bucket::replication::{
|
||||
DeleteReplicationConfigSnapshot, ReplicationLifecycleBridge, ReplicationStatusType, VersionPurgeStatusType,
|
||||
replication_state_to_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta,
|
||||
version_purge_status_to_filemeta,
|
||||
replication_state_to_filemeta, replication_status_from_filemeta, version_purge_status_to_filemeta,
|
||||
};
|
||||
use crate::data_usage::quota_object_size;
|
||||
use crate::diagnostics::get::GetObjectFailureReason;
|
||||
@@ -284,95 +283,12 @@ fn record_transitioned_delete_cleanup_owner(bucket: &str, object: &str, batch: b
|
||||
);
|
||||
}
|
||||
|
||||
/// A causal free-version receipt is only valid when the delete request really
|
||||
/// removes the locked transitioned source. In particular, replication may turn
|
||||
/// an otherwise successful delete into a metadata-only purge-state update, and
|
||||
/// a versioned delete without a version ID writes a new delete marker instead
|
||||
/// of removing the source selected by `goi`.
|
||||
fn transitioned_delete_publishes_free_version(source: &ObjectInfo, delete_request: &FileInfo, skip_free_version: bool) -> bool {
|
||||
if source.delete_marker
|
||||
|| source.transitioned_object.status != TRANSITION_COMPLETE
|
||||
|| skip_free_version
|
||||
|| delete_request.skip_tier_free_version()
|
||||
|| delete_request.expire_restored
|
||||
|| delete_request.transition_status == TRANSITION_COMPLETE
|
||||
|| delete_file_info_version_id(source.version_id) != delete_request.version_id
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Keep this predicate aligned with FileMeta::delete_version's Object
|
||||
// branch: a non-delete-marker request with a nonterminal purge status (or
|
||||
// mark_deleted with no purge status) updates replication metadata in place
|
||||
// and never calls MetaObject::init_free_version.
|
||||
let purge_status = version_purge_status_from_filemeta(delete_request.version_purge_status());
|
||||
let metadata_only = !delete_request.deleted
|
||||
&& ((purge_status.is_empty() && delete_request.mark_deleted)
|
||||
|| (!purge_status.is_empty() && purge_status != VersionPurgeStatusType::Complete));
|
||||
|
||||
!metadata_only
|
||||
}
|
||||
|
||||
fn record_committed_tier_free_version_receipt(
|
||||
opts: &ObjectOptions,
|
||||
async fn acquire_single_tier_delete_lease(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
source: &ObjectInfo,
|
||||
free_version_id: Uuid,
|
||||
batch: bool,
|
||||
) {
|
||||
if let Some(sink) = opts.tier_free_version_receipt_sink.as_ref()
|
||||
&& let Err(err) = sink.record(source, free_version_id)
|
||||
{
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TRANSITIONED_DELETE_CLEANUP_OWNER,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
batch,
|
||||
error = ?err,
|
||||
"Failed to retain the in-memory tier free-version scheduling receipt"
|
||||
);
|
||||
}
|
||||
record_transitioned_delete_cleanup_owner(bucket, object, batch);
|
||||
}
|
||||
|
||||
struct TierFreeVersionReceiptCandidate {
|
||||
source: ObjectInfo,
|
||||
free_version_id: Uuid,
|
||||
}
|
||||
|
||||
fn committed_tier_free_version_receipt_indices(
|
||||
versions: &[FileInfoVersions],
|
||||
delete_errors: &[Option<Error>],
|
||||
candidates: &HashMap<usize, TierFreeVersionReceiptCandidate>,
|
||||
) -> Vec<usize> {
|
||||
if candidates.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut committed = Vec::with_capacity(candidates.len());
|
||||
for group in versions {
|
||||
let should_rollback = group
|
||||
.versions
|
||||
.iter()
|
||||
.any(|version| delete_errors.get(version.idx).is_none_or(|error| error.is_some()));
|
||||
if should_rollback {
|
||||
continue;
|
||||
}
|
||||
committed.extend(
|
||||
group
|
||||
.versions
|
||||
.iter()
|
||||
.map(|version| version.idx)
|
||||
.filter(|idx| candidates.contains_key(idx)),
|
||||
);
|
||||
}
|
||||
committed
|
||||
}
|
||||
|
||||
async fn acquire_single_tier_delete_lease(opts: &ObjectOptions, source: &ObjectInfo) -> Result<Option<TierOperationLease>> {
|
||||
) -> Result<Option<TierOperationLease>> {
|
||||
let Some(api) = opts.tier_delete_journal_api.as_ref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -392,6 +308,7 @@ async fn acquire_single_tier_delete_lease(opts: &ObjectOptions, source: &ObjectI
|
||||
None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &source.transitioned_object.tier).await,
|
||||
}
|
||||
.map_err(Error::other)?;
|
||||
record_transitioned_delete_cleanup_owner(bucket, object, false);
|
||||
Ok(Some(lease))
|
||||
}
|
||||
|
||||
@@ -467,168 +384,6 @@ mod scanner_publication_lease_fence_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tier_free_version_receipt_eligibility_tests {
|
||||
use super::*;
|
||||
use crate::bucket::replication::ReplicationState;
|
||||
|
||||
fn transitioned_source(version_id: Option<Uuid>) -> ObjectInfo {
|
||||
let mut source = ObjectInfo {
|
||||
version_id,
|
||||
..Default::default()
|
||||
};
|
||||
source.transitioned_object.status = TRANSITION_COMPLETE.to_string();
|
||||
source.transitioned_object.tier = "WARM".to_string();
|
||||
source.transitioned_object.name = "remote/object".to_string();
|
||||
source
|
||||
}
|
||||
|
||||
fn delete_request(version_id: Option<Uuid>) -> FileInfo {
|
||||
let mut request = FileInfo {
|
||||
version_id,
|
||||
..Default::default()
|
||||
};
|
||||
request.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
request
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_exact_transitioned_source_removal_and_suspended_null_replacement() {
|
||||
let version_id = Uuid::new_v4();
|
||||
assert!(transitioned_delete_publishes_free_version(
|
||||
&transitioned_source(Some(version_id)),
|
||||
&delete_request(Some(version_id)),
|
||||
false,
|
||||
));
|
||||
|
||||
let mut suspended_null_delete = delete_request(None);
|
||||
suspended_null_delete.deleted = true;
|
||||
suspended_null_delete.mark_deleted = true;
|
||||
assert!(transitioned_delete_publishes_free_version(
|
||||
&transitioned_source(Some(Uuid::nil())),
|
||||
&suspended_null_delete,
|
||||
false,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_new_marker_version_and_non_transitioned_or_delete_marker_sources() {
|
||||
let source_id = Uuid::new_v4();
|
||||
assert!(!transitioned_delete_publishes_free_version(
|
||||
&transitioned_source(Some(source_id)),
|
||||
&delete_request(Some(Uuid::new_v4())),
|
||||
false,
|
||||
));
|
||||
|
||||
let mut ordinary = transitioned_source(Some(source_id));
|
||||
ordinary.transitioned_object.status.clear();
|
||||
assert!(!transitioned_delete_publishes_free_version(
|
||||
&ordinary,
|
||||
&delete_request(Some(source_id)),
|
||||
false,
|
||||
));
|
||||
|
||||
let mut delete_marker = transitioned_source(Some(source_id));
|
||||
delete_marker.delete_marker = true;
|
||||
assert!(!transitioned_delete_publishes_free_version(
|
||||
&delete_marker,
|
||||
&delete_request(Some(source_id)),
|
||||
false,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_skip_restore_and_transition_metadata_updates() {
|
||||
let version_id = Uuid::new_v4();
|
||||
let source = transitioned_source(Some(version_id));
|
||||
|
||||
assert!(!transitioned_delete_publishes_free_version(
|
||||
&source,
|
||||
&delete_request(Some(version_id)),
|
||||
true,
|
||||
));
|
||||
|
||||
let mut skip_request = delete_request(Some(version_id));
|
||||
skip_request.set_skip_tier_free_version();
|
||||
assert!(!transitioned_delete_publishes_free_version(&source, &skip_request, false));
|
||||
|
||||
let mut restore_request = delete_request(Some(version_id));
|
||||
restore_request.expire_restored = true;
|
||||
assert!(!transitioned_delete_publishes_free_version(&source, &restore_request, false));
|
||||
|
||||
let mut transition_update = delete_request(Some(version_id));
|
||||
transition_update.transition_status = TRANSITION_COMPLETE.to_string();
|
||||
assert!(!transitioned_delete_publishes_free_version(&source, &transition_update, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_nonterminal_replication_metadata_only_update_but_accepts_complete_purge() {
|
||||
let version_id = Uuid::new_v4();
|
||||
let source = transitioned_source(Some(version_id));
|
||||
|
||||
let mut pending = delete_request(Some(version_id));
|
||||
pending.replication_state_internal = Some(replication_state_to_filemeta(&ReplicationState {
|
||||
version_purge_status_internal: Some("PENDING".to_string()),
|
||||
..Default::default()
|
||||
}));
|
||||
assert!(!transitioned_delete_publishes_free_version(&source, &pending, false));
|
||||
|
||||
let mut mark_deleted = delete_request(Some(version_id));
|
||||
mark_deleted.mark_deleted = true;
|
||||
assert!(!transitioned_delete_publishes_free_version(&source, &mark_deleted, false));
|
||||
|
||||
let mut complete = delete_request(Some(version_id));
|
||||
complete.replication_state_internal = Some(replication_state_to_filemeta(&ReplicationState {
|
||||
version_purge_status_internal: Some("COMPLETE".to_string()),
|
||||
..Default::default()
|
||||
}));
|
||||
assert!(transitioned_delete_publishes_free_version(&source, &complete, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whole_physical_object_group_must_commit_before_any_receipt_is_retained() {
|
||||
let candidate = || TierFreeVersionReceiptCandidate {
|
||||
source: transitioned_source(Some(Uuid::new_v4())),
|
||||
free_version_id: Uuid::new_v4(),
|
||||
};
|
||||
let candidates = HashMap::from([(0, candidate()), (2, candidate())]);
|
||||
let versions = vec![
|
||||
FileInfoVersions {
|
||||
versions: vec![
|
||||
FileInfo {
|
||||
idx: 0,
|
||||
..Default::default()
|
||||
},
|
||||
FileInfo {
|
||||
idx: 1,
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
..Default::default()
|
||||
},
|
||||
FileInfoVersions {
|
||||
versions: vec![FileInfo {
|
||||
idx: 2,
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let one_sibling_failed = vec![None, Some(Error::other("injected quorum failure")), None];
|
||||
|
||||
assert_eq!(
|
||||
committed_tier_free_version_receipt_indices(&versions, &one_sibling_failed, &candidates),
|
||||
vec![2],
|
||||
"a sibling failure must suppress every receipt from the rolled-back xl.meta group"
|
||||
);
|
||||
assert_eq!(
|
||||
committed_tier_free_version_receipt_indices(&versions, &[None, None, None], &candidates),
|
||||
vec![0, 2],
|
||||
"independent fully committed groups should retain their sparse receipts"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
struct PutObjectCommitCancellation {
|
||||
token: CancellationToken,
|
||||
armed: bool,
|
||||
@@ -7222,7 +6977,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
};
|
||||
let mut vers_map: HashMap<&String, FileInfoVersions> = HashMap::new();
|
||||
let mut tier_reference_leases: Vec<(usize, String, Option<TierDestinationId>)> = Vec::new();
|
||||
let mut tier_free_version_receipt_candidates: HashMap<usize, TierFreeVersionReceiptCandidate> = HashMap::new();
|
||||
let mut transitioned_cleanup_items = vec![false; objects.len()];
|
||||
|
||||
for (i, dobj) in objects.iter().enumerate() {
|
||||
if del_errs[i].is_some() {
|
||||
@@ -7314,6 +7069,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
{
|
||||
match tier_destination_id_from_metadata(&goi.user_defined) {
|
||||
Ok(identity) => {
|
||||
transitioned_cleanup_items[i] = true;
|
||||
tier_reference_leases.push((i, goi.transitioned_object.tier.clone(), identity));
|
||||
}
|
||||
Err(err) => {
|
||||
@@ -7354,8 +7110,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let tier_free_version_id = Uuid::new_v4();
|
||||
vr.set_tier_free_version_id(&tier_free_version_id.to_string());
|
||||
vr.set_tier_free_version_id(&Uuid::new_v4().to_string());
|
||||
|
||||
// Delete
|
||||
// del_objects[i].object_name.clone_from(&vr.name);
|
||||
@@ -7440,19 +7195,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
});
|
||||
}
|
||||
|
||||
if opts.tier_free_version_receipt_sink.is_some()
|
||||
&& !dobj.synthetic_version_id
|
||||
&& transitioned_delete_publishes_free_version(&goi, &vr, opts.skip_free_version)
|
||||
{
|
||||
tier_free_version_receipt_candidates.insert(
|
||||
i,
|
||||
TierFreeVersionReceiptCandidate {
|
||||
source: goi,
|
||||
free_version_id: tier_free_version_id,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// Only add to vers_map if we hold the lock
|
||||
if locked_objects.contains(&dobj.object_name) {
|
||||
vers_map.insert(&dobj.object_name, v);
|
||||
@@ -7529,6 +7271,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
for (idx, transitioned) in transitioned_cleanup_items.into_iter().enumerate() {
|
||||
if transitioned && del_errs[idx].is_none() {
|
||||
record_transitioned_delete_cleanup_owner(bucket, &decode_dir_object(&objects[idx].object_name), true);
|
||||
}
|
||||
}
|
||||
|
||||
// Keep backend generations pinned through the source mutation, its
|
||||
// free-version write quorum, and any local rollback. Ordinary
|
||||
// single/batch deletes never transfer cleanup ownership to a journal.
|
||||
@@ -7640,8 +7388,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||
|
||||
let mut rollback_futures = Vec::new();
|
||||
let committed_receipt_indices =
|
||||
committed_tier_free_version_receipt_indices(&vers, &del_errs, &tier_free_version_receipt_candidates);
|
||||
for fi_vers in &vers {
|
||||
// delete_versions commits one xl.meta per object group, so rollback must use the same boundary.
|
||||
let should_rollback = fi_vers.versions.iter().any(|fi| del_errs[fi.idx].is_some());
|
||||
@@ -7716,20 +7462,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
|
||||
join_all(rollback_futures).await;
|
||||
|
||||
for idx in committed_receipt_indices {
|
||||
let Some(candidate) = tier_free_version_receipt_candidates.remove(&idx) else {
|
||||
continue;
|
||||
};
|
||||
record_committed_tier_free_version_receipt(
|
||||
&opts,
|
||||
bucket,
|
||||
&decode_dir_object(&objects[idx].object_name),
|
||||
&candidate.source,
|
||||
candidate.free_version_id,
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
// TODO(backlog): support partial object deletion for multi-part objects
|
||||
|
||||
if dist_erasure {
|
||||
@@ -8081,7 +7813,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
|
||||
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
|
||||
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
|
||||
let _tier_delete_lease = acquire_single_tier_delete_lease(&opts, &goi).await?;
|
||||
let _tier_delete_lease = acquire_single_tier_delete_lease(bucket, object, &opts, &goi).await?;
|
||||
if opts.skip_free_version {
|
||||
fi.set_skip_tier_free_version();
|
||||
}
|
||||
@@ -8091,12 +7823,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
#[cfg(test)]
|
||||
pause_delete_object_commit_after_publish(bucket, object).await;
|
||||
|
||||
if opts.tier_free_version_receipt_sink.is_some()
|
||||
&& transitioned_delete_publishes_free_version(&goi, &fi, opts.skip_free_version)
|
||||
{
|
||||
record_committed_tier_free_version_receipt(&opts, bucket, object, &goi, find_vid, false);
|
||||
}
|
||||
|
||||
let disks = self.disk_inventory().await;
|
||||
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||
|
||||
@@ -8129,7 +7855,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
|
||||
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
|
||||
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
|
||||
let _tier_delete_lease = acquire_single_tier_delete_lease(&opts, &goi).await?;
|
||||
let _tier_delete_lease = acquire_single_tier_delete_lease(bucket, object, &opts, &goi).await?;
|
||||
if opts.skip_free_version {
|
||||
dfi.set_skip_tier_free_version();
|
||||
}
|
||||
@@ -8139,12 +7865,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
#[cfg(test)]
|
||||
pause_delete_object_commit_after_publish(bucket, object).await;
|
||||
|
||||
if opts.tier_free_version_receipt_sink.is_some()
|
||||
&& transitioned_delete_publishes_free_version(&goi, &dfi, opts.skip_free_version)
|
||||
{
|
||||
record_committed_tier_free_version_receipt(&opts, bucket, object, &goi, find_vid, false);
|
||||
}
|
||||
|
||||
let disks = self.disk_inventory().await;
|
||||
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -14,8 +14,8 @@
|
||||
|
||||
use super::*;
|
||||
use crate::core::pools::{
|
||||
PoolMetaBootstrapAuthority, PoolMetaReplicaState, PoolMetaWriteState, load_pool_meta_identity_observing,
|
||||
local_decommission_queue_prefix, persist_pool_meta_identity_for_startup, pool_meta_has_active_decommission,
|
||||
PoolMetaReplicaState, PoolMetaWriteState, load_pool_meta_identity_observing, local_decommission_queue_prefix,
|
||||
persist_pool_meta_identity_for_startup, pool_meta_has_active_decommission,
|
||||
};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
@@ -173,7 +173,7 @@ async fn establish_pool_meta_bootstrap_identity_if_proven<S>(
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
if elected_writer && write_state.bootstrap_identity_proven() {
|
||||
if elected_writer && write_state.fresh_bootstrap_proven() {
|
||||
persist_pool_meta_identity_for_startup(pools, write_state, false).await?;
|
||||
}
|
||||
Ok(())
|
||||
@@ -215,7 +215,7 @@ where
|
||||
}
|
||||
let mut committed = meta.clone();
|
||||
if should_write {
|
||||
if write_state.bootstrap_identity_proven() || write_state.identity_is_pending() {
|
||||
if write_state.fresh_bootstrap_proven() || write_state.identity_is_pending() {
|
||||
persist_pool_meta_identity_for_startup(pools.clone(), write_state, false)
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("store init failed during prepare_pool_meta_identity: {err}")))?;
|
||||
@@ -402,7 +402,7 @@ impl ECStore {
|
||||
preflight_startup_rpc_secret(&endpoint_pools)?;
|
||||
|
||||
let mut deployment_id = None;
|
||||
let mut pool_meta_bootstrap_authority = None;
|
||||
let mut fresh_bootstrap_proven = true;
|
||||
|
||||
// let (endpoint_pools, _) = EndpointServerPools::create_server_endpoints(address.as_str(), &layouts)?;
|
||||
|
||||
@@ -518,12 +518,7 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
}?;
|
||||
pool_meta_bootstrap_authority = Some(pool_meta_bootstrap_authority.map_or(
|
||||
loaded_format.pool_meta_bootstrap_authority,
|
||||
|authority: PoolMetaBootstrapAuthority| {
|
||||
authority.combine_across_pools(loaded_format.pool_meta_bootstrap_authority)
|
||||
},
|
||||
));
|
||||
fresh_bootstrap_proven &= loaded_format.fresh_bootstrap_proven;
|
||||
let fm = loaded_format.format;
|
||||
|
||||
// Format loading succeeded, enable health monitoring on all disks
|
||||
@@ -564,10 +559,6 @@ impl ECStore {
|
||||
let peer_sys = S3PeerSys::new_with_instance_ctx(&endpoint_pools, instance_ctx.clone());
|
||||
let mut pool_meta = PoolMeta::new(&pools, &PoolMeta::default());
|
||||
pool_meta.dont_save = true;
|
||||
let pool_meta_write_state = PoolMetaWriteState::for_startup_with_bootstrap_authority(
|
||||
deployment_id,
|
||||
pool_meta_bootstrap_authority.unwrap_or_default(),
|
||||
);
|
||||
|
||||
let decommission_cancelers = RwLock::new(vec![None; pools.len()]);
|
||||
let ec = Arc::new(ECStore {
|
||||
@@ -579,7 +570,7 @@ impl ECStore {
|
||||
rebalance_meta: RwLock::new(None),
|
||||
decommission_cancelers,
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::new(pool_meta_write_state),
|
||||
pool_meta_save_gate: Mutex::new(PoolMetaWriteState::for_startup(deployment_id, fresh_bootstrap_proven)),
|
||||
decommission_capacity_entry_gate: Mutex::default(),
|
||||
// Adopt the caller's context (the process bootstrap one on the
|
||||
// legacy path) so startup writes (erasure type recorded before
|
||||
@@ -800,7 +791,6 @@ mod tests {
|
||||
run_local_decommission_watchdog, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init,
|
||||
should_defer_rebalance_auto_start, should_retry_format_load, wait_for_local_decommission_resume_delay,
|
||||
};
|
||||
use crate::core::pools::PoolMetaBootstrapAuthority;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::disk::DiskAPI;
|
||||
#[cfg(feature = "test-util")]
|
||||
@@ -1161,65 +1151,6 @@ mod tests {
|
||||
.expect("successful startup publication must disarm the transaction guard");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_legacy_adoption_pool_meta_bootstrap_reaches_v3_cas() {
|
||||
let deployment_id = Uuid::new_v4();
|
||||
let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
|
||||
let mut write_state =
|
||||
PoolMetaWriteState::for_startup_with_bootstrap_authority(deployment_id, PoolMetaBootstrapAuthority::LegacyAdoption);
|
||||
|
||||
establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut write_state, true)
|
||||
.await
|
||||
.expect("verified legacy adoption should persist a nonce-bound identity before loading pool metadata");
|
||||
let (loaded, replica_state) = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
|
||||
.await
|
||||
.expect("verified legacy adoption should authorize initially missing pool metadata");
|
||||
assert!(loaded.pools.is_empty());
|
||||
|
||||
let committed = persist_pool_meta_for_startup_if_safe(
|
||||
&init_test_pool_meta(None),
|
||||
vec![storage.clone()],
|
||||
replica_state,
|
||||
&mut write_state,
|
||||
true,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("verified legacy adoption should publish initial pool metadata");
|
||||
assert_eq!(committed.pools[0].cmd_line, "pool-0");
|
||||
|
||||
let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
assert!(objects.contains_key(POOL_META_NAME));
|
||||
let identity = objects
|
||||
.get(POOL_META_IDENTITY_NAME)
|
||||
.map(|(payload, _)| payload.clone())
|
||||
.expect("legacy adoption should commit the bootstrap identity");
|
||||
assert!(pool_meta_identity_initialized_for_test(&identity).expect("decode committed identity"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_non_elected_legacy_adoption_cannot_initialize_pool_meta() {
|
||||
let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
|
||||
let mut write_state =
|
||||
PoolMetaWriteState::for_startup_with_bootstrap_authority(Uuid::new_v4(), PoolMetaBootstrapAuthority::LegacyAdoption);
|
||||
|
||||
establish_pool_meta_bootstrap_identity_if_proven(vec![storage.clone()], &mut write_state, false)
|
||||
.await
|
||||
.expect("a non-elected distributed node must not create legacy adoption authority");
|
||||
let err = load_pool_meta_for_startup(vec![storage.clone()], &mut write_state)
|
||||
.await
|
||||
.expect_err("legacy adoption still requires the elected writer to create a durable identity");
|
||||
assert!(err.to_string().contains("no durable bootstrap identity"));
|
||||
assert!(
|
||||
!storage
|
||||
.objects
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.contains_key(POOL_META_NAME),
|
||||
"classification must not create pool metadata on non-elected nodes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_unproven_pending_identity_cannot_authorize_all_missing_pool_meta() {
|
||||
let deployment_id = Uuid::new_v4();
|
||||
@@ -2305,200 +2236,6 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn copy_object_immediately_reads_small_completed_multipart_source() {
|
||||
let temp_dir = tempfile::tempdir().expect("create small multipart copy store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "small-multipart-copy", &[1])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
|
||||
let bucket = format!("small-multipart-copy-{}", Uuid::new_v4());
|
||||
let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
|
||||
let target_object = "docker/registry/v2/blobs/sha256/c0/digest/data";
|
||||
let payload = vec![0xAB; 273];
|
||||
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket for small multipart copy");
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create source multipart upload");
|
||||
let mut part_reader = PutObjReader::from_vec(payload.clone());
|
||||
let part = store
|
||||
.put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("stage small multipart source part");
|
||||
let completed = store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
source_object,
|
||||
&upload.upload_id,
|
||||
vec![crate::storage_api_contracts::multipart::CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("complete the small multipart source");
|
||||
assert_eq!(completed.get_actual_size().expect("completed object logical size"), payload.len() as i64);
|
||||
|
||||
let source_reader = store
|
||||
.get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed multipart source should be immediately readable");
|
||||
let mut copy_info = source_reader.object_info.clone();
|
||||
let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
|
||||
assert_eq!(actual_size, payload.len() as i64);
|
||||
let copy_reader = rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
|
||||
.expect("copy source hash reader should build");
|
||||
copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
|
||||
|
||||
store
|
||||
.copy_object(
|
||||
&bucket,
|
||||
source_object,
|
||||
&bucket,
|
||||
target_object,
|
||||
&mut copy_info,
|
||||
&ObjectOptions::default(),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("CopyObject should accept a freshly completed multipart source");
|
||||
|
||||
let mut target_reader = store
|
||||
.get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("copied target should be readable");
|
||||
let mut target_body = Vec::new();
|
||||
target_reader
|
||||
.stream
|
||||
.read_to_end(&mut target_body)
|
||||
.await
|
||||
.expect("target body should stream");
|
||||
assert_eq!(target_body, payload);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn complete_multipart_waits_for_tail_rename_before_copy_source_visibility() {
|
||||
let temp_dir = tempfile::tempdir().expect("create early-ack multipart copy store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "early-ack-multipart-copy", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
|
||||
let bucket = format!("early-ack-multipart-copy-{}", Uuid::new_v4());
|
||||
let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
|
||||
let target_object = "docker/registry/v2/blobs/sha256/c0/digest/data";
|
||||
let payload = vec![0xCD; 273];
|
||||
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket for early-ack multipart copy");
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, source_object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create source multipart upload");
|
||||
let mut part_reader = PutObjReader::from_vec(payload.clone());
|
||||
let part = store
|
||||
.put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("stage small multipart source part");
|
||||
let completed_parts = vec![crate::storage_api_contracts::multipart::CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}];
|
||||
|
||||
temp_env::async_with_vars([(crate::set_disk::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let rename_tasks = crate::set_disk::rename_fanout_barrier::observe_tasks(source_object);
|
||||
let rename_barrier = crate::set_disk::rename_fanout_barrier::arm(
|
||||
source_object,
|
||||
0,
|
||||
crate::set_disk::rename_fanout_barrier::PHASE_RENAME,
|
||||
);
|
||||
let complete_store = Arc::clone(&store);
|
||||
let complete_bucket = bucket.clone();
|
||||
let complete_upload_id = upload.upload_id.clone();
|
||||
let mut complete = tokio::spawn(async move {
|
||||
complete_store
|
||||
.complete_multipart_upload(
|
||||
&complete_bucket,
|
||||
source_object,
|
||||
&complete_upload_id,
|
||||
completed_parts,
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("multipart completion should pause one tail disk during rename");
|
||||
assert!(
|
||||
rename_tasks.running() >= 1,
|
||||
"the paused multipart tail disk must remain in flight before completion returns"
|
||||
);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), &mut complete).await.is_err(),
|
||||
"CompleteMultipartUpload must not return while a copy source rename tail is still pending"
|
||||
);
|
||||
rename_barrier.release();
|
||||
complete
|
||||
.await
|
||||
.expect("multipart completion task should join")
|
||||
.expect("multipart completion should return after every rename tail finishes");
|
||||
|
||||
let source_reader = store
|
||||
.get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed multipart source should be immediately readable after success");
|
||||
let mut copy_info = source_reader.object_info.clone();
|
||||
let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
|
||||
assert_eq!(actual_size, payload.len() as i64);
|
||||
let copy_reader =
|
||||
rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
|
||||
.expect("copy source hash reader should build");
|
||||
copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
|
||||
|
||||
store
|
||||
.copy_object(
|
||||
&bucket,
|
||||
source_object,
|
||||
&bucket,
|
||||
target_object,
|
||||
&mut copy_info,
|
||||
&ObjectOptions::default(),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("CopyObject should accept a freshly completed multipart source");
|
||||
})
|
||||
.await;
|
||||
|
||||
let mut target_reader = store
|
||||
.get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("copied target should be readable after tail release");
|
||||
let mut target_body = Vec::new();
|
||||
target_reader
|
||||
.stream
|
||||
.read_to_end(&mut target_body)
|
||||
.await
|
||||
.expect("target body should stream");
|
||||
assert_eq!(target_body, payload);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -3805,7 +3542,6 @@ mod tests {
|
||||
assert_eq!(body, multipart_target_body);
|
||||
|
||||
let retry_object = "multipart-retry-object";
|
||||
let retry_object_etag = "0123456789abcdef0123456789abcdef".to_string();
|
||||
let retry_first_part_size = 5 * 1024 * 1024;
|
||||
let retry_object_mod_time =
|
||||
OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed retry timestamp should be valid");
|
||||
@@ -3895,7 +3631,6 @@ mod tests {
|
||||
&ObjectOptions {
|
||||
mod_time: Some(retry_object_mod_time),
|
||||
want_checksum: Some(retry_object_checksum),
|
||||
preserve_etag: Some(retry_object_etag.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -3928,7 +3663,7 @@ mod tests {
|
||||
part.checksums = Some(HashMap::from([(ChecksumType::CRC32C.to_string(), checksum.encoded.clone())]));
|
||||
}
|
||||
retry_source_info.parts = Arc::new(retry_source_parts);
|
||||
assert_eq!(retry_source_info.etag.as_deref(), Some(retry_object_etag.as_str()));
|
||||
retry_source_info.etag = Some("0123456789abcdef0123456789abcdef".to_string());
|
||||
assert!(!retry_source_info.is_multipart());
|
||||
assert!(retry_source_info.parts.iter().all(|part| part.checksums.is_some()));
|
||||
assert_eq!(retry_source_info.checksum.as_deref(), Some(retry_object_checksum_bytes.as_ref()));
|
||||
@@ -10690,8 +10425,6 @@ mod tests {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
restore_before_delete: bool,
|
||||
causal_enqueue: bool,
|
||||
delete_with_journal: bool,
|
||||
) {
|
||||
let temp_dir = tempfile::tempdir().expect("create transitioned delete store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
@@ -10755,51 +10488,11 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
if causal_enqueue {
|
||||
ExpiryState::resize_workers(1, store.clone()).await;
|
||||
}
|
||||
backend.set_remove_failure(!causal_enqueue);
|
||||
if delete_with_journal {
|
||||
store
|
||||
.delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
|
||||
.await
|
||||
.expect("transitioned source journal-wrapper delete should commit");
|
||||
} else {
|
||||
store
|
||||
.delete_object(bucket, object, ObjectOptions::default())
|
||||
.await
|
||||
.expect("transitioned source plain object-layer delete should commit");
|
||||
}
|
||||
|
||||
if causal_enqueue {
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let metadata_absent = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("causal free-version cleanup metadata should remain readable")
|
||||
.is_none();
|
||||
if metadata_absent && backend.remove_versions().await.len() == 1 {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
backend.set_remove_failure(true);
|
||||
store
|
||||
.delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
|
||||
.await
|
||||
.expect("committed free-version should be cleaned without a recovery scan");
|
||||
assert_eq!(backend.object_count().await, 0, "causal cleanup should remove the remote object");
|
||||
assert_eq!(
|
||||
tier_delete_journal_count(store.clone()).await,
|
||||
0,
|
||||
"ordinary causal cleanup must not create a journal"
|
||||
);
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket delete should succeed after causal free-version cleanup");
|
||||
return;
|
||||
}
|
||||
.expect("transitioned source delete should commit");
|
||||
|
||||
let local_versions = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
@@ -10878,8 +10571,6 @@ mod tests {
|
||||
"transitioned-delete-journal-owner-bucket",
|
||||
"transition/archive.bin",
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -10894,40 +10585,6 @@ mod tests {
|
||||
"restored-transitioned-delete-journal-owner-bucket",
|
||||
"transition/archive.bin",
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn transitioned_delete_causally_enqueues_free_version() {
|
||||
run_transitioned_delete_free_version_owner_case(
|
||||
"transitioned-delete-causal-enqueue",
|
||||
"DELETE-CAUSAL",
|
||||
"transitioned-delete-causal-enqueue-bucket",
|
||||
"transition/archive.bin",
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn restored_transitioned_delete_causally_enqueues_free_version() {
|
||||
run_transitioned_delete_free_version_owner_case(
|
||||
"restored-transitioned-delete-causal-enqueue",
|
||||
"RESTORE-DELETE-CAUSAL",
|
||||
"restored-transitioned-delete-causal-enqueue-bucket",
|
||||
"transition/archive.bin",
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -12607,262 +12264,12 @@ mod tests {
|
||||
.is_none(),
|
||||
"free-version recovery must remove the exact cleanup owner"
|
||||
);
|
||||
|
||||
let causal = "causal.bin";
|
||||
let mut causal_reader = PutObjReader::from_vec(vec![b'c'; 1024 * 1024]);
|
||||
let causal_source = store
|
||||
.put_object(bucket, causal, &mut causal_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("causal batch source should be written");
|
||||
store
|
||||
.transition_object(
|
||||
bucket,
|
||||
causal,
|
||||
&ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: causal_source.etag.clone().expect("causal batch source should have an etag"),
|
||||
..Default::default()
|
||||
},
|
||||
mod_time: causal_source.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("causal batch source transition should commit");
|
||||
let (_deleted, errors) = store
|
||||
.delete_objects(
|
||||
bucket,
|
||||
vec![
|
||||
ObjectToDelete {
|
||||
object_name: causal.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: causal.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
ObjectOptions::default(),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
errors.iter().all(Option::is_none),
|
||||
"duplicate causal batch deletes should remain idempotent: {errors:?}"
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let metadata_absent = store.pools[0]
|
||||
.get_disks_by_key(causal)
|
||||
.load_file_info_versions_exact(bucket, causal)
|
||||
.await
|
||||
.expect("causal batch cleanup metadata should remain readable")
|
||||
.is_none();
|
||||
if metadata_absent && backend.remove_versions().await.len() >= 2 {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("batch free-version receipt should converge without another recovery scan");
|
||||
assert_eq!(
|
||||
backend.remove_versions().await.len(),
|
||||
2,
|
||||
"duplicate batch requests must cause only one remote delete for the causal object"
|
||||
);
|
||||
|
||||
crate::bucket::metadata_sys::update_in(
|
||||
&ctx,
|
||||
bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("causal batch bucket versioning should be enabled");
|
||||
let versioned_causal = "versioned-causal.bin";
|
||||
let mut versioned_reader = PutObjReader::from_vec(vec![b'v'; 1024 * 1024]);
|
||||
let versioned_source = store
|
||||
.put_object(
|
||||
bucket,
|
||||
versioned_causal,
|
||||
&mut versioned_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("versioned causal batch source should be written");
|
||||
let versioned_source_id = versioned_source
|
||||
.version_id
|
||||
.expect("versioned causal batch source should have an identity");
|
||||
store
|
||||
.transition_object(
|
||||
bucket,
|
||||
versioned_causal,
|
||||
&ObjectOptions {
|
||||
version_id: Some(versioned_source_id.to_string()),
|
||||
versioned: true,
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: versioned_source
|
||||
.etag
|
||||
.clone()
|
||||
.expect("versioned causal batch source should have an etag"),
|
||||
..Default::default()
|
||||
},
|
||||
mod_time: versioned_source.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("versioned causal batch source transition should commit");
|
||||
let (_deleted, errors) = store
|
||||
.delete_objects(
|
||||
bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: versioned_causal.to_string(),
|
||||
version_id: Some(versioned_source_id),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions::default(),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
errors.iter().all(Option::is_none),
|
||||
"explicit-version causal batch delete should commit: {errors:?}"
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let metadata_absent = store.pools[0]
|
||||
.get_disks_by_key(versioned_causal)
|
||||
.load_file_info_versions_exact(bucket, versioned_causal)
|
||||
.await
|
||||
.expect("versioned causal batch cleanup metadata should remain readable")
|
||||
.is_none();
|
||||
if metadata_absent && backend.remove_versions().await.len() == 3 {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("explicit-version batch receipt should converge without a recovery scan");
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("batch source bucket should be physically empty");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn batch_transitioned_delete_aggregate_error_still_enqueues_committed_free_version() {
|
||||
let temp_dir = tempfile::tempdir().expect("create aggregate-error batch delete store dir");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-transitioned-aggregate-error", &[4, 4]))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let tier_name = "BATCH-AGGREGATE-ERROR";
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let bucket = "batch-transitioned-aggregate-error-bucket";
|
||||
let object = "archive.bin";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("aggregate-error source bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(vec![b'a'; 1024 * 1024]);
|
||||
let source = store.pools[0]
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("aggregate-error source should be written");
|
||||
store.pools[0]
|
||||
.transition_object(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: source.etag.clone().expect("aggregate-error source should have an etag"),
|
||||
..Default::default()
|
||||
},
|
||||
mod_time: source.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("aggregate-error source should transition");
|
||||
|
||||
// Model a data-movement copy: both pools own the same logical source
|
||||
// and exact remote tuple, but each batch delete creates its own local
|
||||
// free-version UUID in the shared request sink.
|
||||
for disk_index in 0..4 {
|
||||
let source_meta = temp_dir
|
||||
.path()
|
||||
.join(format!("pool0/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
|
||||
let target_meta = temp_dir
|
||||
.path()
|
||||
.join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"));
|
||||
tokio::fs::create_dir_all(target_meta.parent().expect("target xl.meta should have a parent"))
|
||||
.await
|
||||
.expect("second-pool object directory should be created");
|
||||
tokio::fs::copy(&source_meta, &target_meta)
|
||||
.await
|
||||
.expect("transitioned xl.meta should copy exactly to the second pool");
|
||||
}
|
||||
|
||||
ExpiryState::resize_workers(1, store.clone()).await;
|
||||
let injection = crate::store::object::BatchDeletePoolErrorInjection::install(
|
||||
bucket,
|
||||
1,
|
||||
vec![(object.to_string(), StorageError::ErasureWriteQuorum)],
|
||||
);
|
||||
let (deleted, errors) = store
|
||||
.delete_objects(
|
||||
bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions::default(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(injection.observed(), 1, "the second pool should inject one post-commit aggregate error");
|
||||
assert_eq!(errors, vec![Some(StorageError::ErasureWriteQuorum)]);
|
||||
assert!(deleted[0].found, "the aggregate error must retain the committed pool result");
|
||||
drop(injection);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let mut metadata_absent = true;
|
||||
for pool in &store.pools {
|
||||
metadata_absent &= pool
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("aggregate-error cleanup metadata should remain readable")
|
||||
.is_none();
|
||||
}
|
||||
if metadata_absent && backend.remove_count().await == 1 {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("aggregate failure must not suppress committed receipt dispatch");
|
||||
assert_eq!(backend.object_count().await, 0, "the shared remote object should be removed exactly once");
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("aggregate-error bucket should be physically empty");
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -13094,217 +12501,6 @@ mod tests {
|
||||
.expect("retry should leave the source bucket empty");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn batch_transitioned_delete_post_commit_failures_roll_back_without_free_version_receipt() {
|
||||
let temp_dir = tempfile::tempdir().expect("create failed batch delete store dir");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-local-failure", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let tier_name = "BATCH-DELETE-LOCAL-FAIL";
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
let bucket = "batch-delete-local-failure-bucket";
|
||||
let object = "archive.bin";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("failed batch source bucket should be created");
|
||||
crate::bucket::metadata_sys::update_in(
|
||||
&ctx,
|
||||
bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("failed batch bucket versioning should be enabled");
|
||||
let mut transitioned_reader = PutObjReader::from_vec(vec![b't'; 1024 * 1024]);
|
||||
let transitioned_source = store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut transitioned_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("failed batch transitioned version should be written");
|
||||
let transitioned_version_id = transitioned_source
|
||||
.version_id
|
||||
.expect("failed batch transitioned source should have a version identity");
|
||||
store
|
||||
.transition_object(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
version_id: Some(transitioned_version_id.to_string()),
|
||||
versioned: true,
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: transitioned_source
|
||||
.etag
|
||||
.clone()
|
||||
.expect("failed batch transitioned source should have an etag"),
|
||||
..Default::default()
|
||||
},
|
||||
mod_time: transitioned_source.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("failed batch source version should transition");
|
||||
let mut ordinary_reader = PutObjReader::from_vec(vec![b'o'; 1024 * 1024]);
|
||||
let ordinary_source = store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut ordinary_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("failed batch ordinary sibling should be written");
|
||||
let ordinary_version_id = ordinary_source
|
||||
.version_id
|
||||
.expect("failed batch ordinary sibling should have a version identity");
|
||||
let delete_requests = || {
|
||||
vec![
|
||||
ObjectToDelete {
|
||||
object_name: object.to_string(),
|
||||
version_id: Some(transitioned_version_id),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectToDelete {
|
||||
object_name: object.to_string(),
|
||||
version_id: Some(ordinary_version_id),
|
||||
..Default::default()
|
||||
},
|
||||
]
|
||||
};
|
||||
|
||||
let set = store.pools[0].get_disks_by_key(object);
|
||||
let disks = set.disks.read().await;
|
||||
assert_eq!(disks.len(), 4, "the rollback fixture must use four disks");
|
||||
// Keep discovery fully online, then make a quorum of disks report an
|
||||
// error only after their batch metadata commit has completed.
|
||||
for disk in disks.iter().take(3) {
|
||||
let disk = disk.as_ref().expect("injected rollback disks should be online");
|
||||
crate::disk::local::set_delete_version_fail_after_commit(disk.path().as_path(), object);
|
||||
}
|
||||
drop(disks);
|
||||
let receipt_sink = crate::object_api::TierFreeVersionReceiptSink::new();
|
||||
let (_deleted, errors) = store
|
||||
.delete_objects(
|
||||
bucket,
|
||||
delete_requests(),
|
||||
ObjectOptions {
|
||||
tier_free_version_receipt_sink: Some(receipt_sink.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
errors,
|
||||
vec![Some(StorageError::Unexpected), Some(StorageError::Unexpected),],
|
||||
"three post-commit disk errors must fail batch delete before receipts publish"
|
||||
);
|
||||
|
||||
assert!(
|
||||
receipt_sink
|
||||
.drain()
|
||||
.expect("the test-owned failed-batch sink should drain exactly once")
|
||||
.is_empty(),
|
||||
"a rolled-back physical group must publish no cleanup receipt"
|
||||
);
|
||||
let retained_transitioned = store
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
version_id: Some(transitioned_version_id.to_string()),
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("failed batch delete must restore the transitioned sibling");
|
||||
assert_eq!(retained_transitioned.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
|
||||
let retained_ordinary = store
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
version_id: Some(ordinary_version_id.to_string()),
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("failed batch delete must restore the ordinary sibling");
|
||||
assert_ne!(retained_ordinary.transitioned_object.status, rustfs_filemeta::TRANSITION_COMPLETE);
|
||||
let retained_versions = set
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("rolled-back batch metadata should decode")
|
||||
.expect("rolled-back batch source should remain on disk");
|
||||
assert_eq!(
|
||||
retained_versions
|
||||
.versions
|
||||
.iter()
|
||||
.chain(retained_versions.free_versions.iter())
|
||||
.filter(|version| version.tier_free_version())
|
||||
.count(),
|
||||
0,
|
||||
"failed batch quorum must not retain a free-version owner"
|
||||
);
|
||||
let retained_version_ids = retained_versions
|
||||
.versions
|
||||
.iter()
|
||||
.filter_map(|version| version.version_id)
|
||||
.collect::<std::collections::HashSet<_>>();
|
||||
assert_eq!(
|
||||
retained_version_ids,
|
||||
std::collections::HashSet::from([transitioned_version_id, ordinary_version_id]),
|
||||
"the physical-group rollback must restore both explicit siblings"
|
||||
);
|
||||
assert_eq!(backend.object_count().await, 1, "failed batch commit must retain the remote object");
|
||||
assert_eq!(backend.remove_count().await, 0, "failed batch commit must not dispatch remote cleanup");
|
||||
|
||||
ExpiryState::resize_workers(1, store.clone()).await;
|
||||
let (_deleted, retry_errors) = store
|
||||
.delete_objects(bucket, delete_requests(), ObjectOptions::default())
|
||||
.await;
|
||||
assert!(
|
||||
retry_errors.iter().all(Option::is_none),
|
||||
"retry after disk recovery should commit: {retry_errors:?}"
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let metadata_absent = set
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("retry cleanup metadata should remain readable")
|
||||
.is_none();
|
||||
if metadata_absent && backend.remove_count().await == 1 {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("successful batch retry should converge without a recovery scan");
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("successful batch retry should leave the bucket empty");
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn run_multi_pool_same_remote_tuple_delete_case(batch: bool) {
|
||||
let temp_dir = tempfile::tempdir().expect("create shared-tuple multi-pool store dir");
|
||||
@@ -13378,10 +12574,8 @@ mod tests {
|
||||
);
|
||||
assert_eq!(backend.object_count().await, 1);
|
||||
|
||||
let receipt_sink = crate::object_api::TierFreeVersionReceiptSink::new();
|
||||
let mut delete_opts = ObjectOptions {
|
||||
tier_delete_journal_api: Some(store.clone()),
|
||||
tier_free_version_receipt_sink: Some(receipt_sink.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
if batch {
|
||||
@@ -13509,20 +12703,7 @@ mod tests {
|
||||
);
|
||||
|
||||
backend.set_remove_failure(false);
|
||||
let receipts = receipt_sink
|
||||
.drain()
|
||||
.expect("the simulated outer multi-pool wrapper should drain exactly once");
|
||||
assert_eq!(
|
||||
receipts.len(),
|
||||
1,
|
||||
"the same physical key and remote tuple must collapse to one causal task"
|
||||
);
|
||||
assert_eq!(
|
||||
crate::bucket::lifecycle::bucket_lifecycle_ops::enqueue_committed_free_versions(&store, receipts).await,
|
||||
1,
|
||||
"the committed shared-tuple task should enter the running worker"
|
||||
);
|
||||
wait_for_expiry_workers_idle(&store).await;
|
||||
wait_for_tier_free_version_recovery(store.clone(), &backend, 1).await;
|
||||
assert_eq!(backend.remove_count().await, 1, "shared remote tuple should be deleted exactly once");
|
||||
for pool_idx in 0..2 {
|
||||
assert!(
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
use crate::cluster::rpc::client::is_network_like_disk_error;
|
||||
use crate::config::storageclass;
|
||||
use crate::core::pools::PoolMetaBootstrapAuthority;
|
||||
use crate::disk::error_reduce::{count_errs, reduce_write_quorum_errs};
|
||||
use crate::disk::{self, DiskAPI};
|
||||
use crate::error::{Error, Result};
|
||||
@@ -85,7 +84,7 @@ pub async fn connect_load_init_formats(
|
||||
|
||||
pub(crate) struct LoadedFormat {
|
||||
pub(crate) format: FormatV3,
|
||||
pub(crate) pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority,
|
||||
pub(crate) fresh_bootstrap_proven: bool,
|
||||
}
|
||||
|
||||
pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
@@ -134,7 +133,7 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, set_drive_count).await?;
|
||||
return Ok(LoadedFormat {
|
||||
format: *format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::LegacyAdoption,
|
||||
fresh_bootstrap_proven: false,
|
||||
});
|
||||
}
|
||||
Ok(LegacyFormatOutcome::Incompatible) => {
|
||||
@@ -154,7 +153,7 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
let fm = init_format_erasure(instance_ctx, disks, set_count, set_drive_count, deployment_id).await?;
|
||||
return Ok(LoadedFormat {
|
||||
format: fm,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::Fresh,
|
||||
fresh_bootstrap_proven: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -183,29 +182,10 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
|
||||
Ok(LoadedFormat {
|
||||
format: fm,
|
||||
pool_meta_bootstrap_authority: verified_legacy_adoption_source(disks, &formats, set_count, set_drive_count).await?,
|
||||
fresh_bootstrap_proven: false,
|
||||
})
|
||||
}
|
||||
|
||||
async fn verified_legacy_adoption_source(
|
||||
disks: &[Option<DiskStore>],
|
||||
rustfs_formats: &[Option<FormatV3>],
|
||||
set_count: usize,
|
||||
set_drive_count: usize,
|
||||
) -> Result<PoolMetaBootstrapAuthority> {
|
||||
match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count).await {
|
||||
Ok(LegacyFormatOutcome::Migrated { .. }) => Ok(PoolMetaBootstrapAuthority::LegacyAdoption),
|
||||
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::Incompatible) => Ok(PoolMetaBootstrapAuthority::None),
|
||||
Err(err) => {
|
||||
debug!(
|
||||
error = %err,
|
||||
"legacy adoption proof skipped because legacy format verification failed"
|
||||
);
|
||||
Ok(PoolMetaBootstrapAuthority::None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn retain_format_quorum_members(
|
||||
instance_ctx: &Arc<InstanceContext>,
|
||||
disks: &mut [Option<DiskStore>],
|
||||
@@ -1331,7 +1311,10 @@ mod tests {
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(¤t_ctx(), true, &mut disks, 1, 3, None)
|
||||
.await
|
||||
.expect("fresh disks should receive a storage format");
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::Fresh);
|
||||
assert!(
|
||||
loaded.fresh_bootstrap_proven,
|
||||
"every configured disk explicitly reporting unformatted should establish fresh topology proof"
|
||||
);
|
||||
let format = loaded.format;
|
||||
|
||||
let (formats, errors) = load_format_erasure_all(&disks, false).await;
|
||||
@@ -1375,11 +1358,12 @@ mod tests {
|
||||
|
||||
let mut expected = legacy;
|
||||
expected.erasure.this = Uuid::nil();
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(¤t_ctx(), true, &mut disks, 1, 3, None)
|
||||
.await
|
||||
.expect("compatible legacy format should migrate");
|
||||
assert_eq!(loaded.format, expected);
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::LegacyAdoption);
|
||||
assert_eq!(
|
||||
connect_load_init_formats(true, &mut disks, 1, 3, None)
|
||||
.await
|
||||
.expect("compatible legacy format should migrate"),
|
||||
expected
|
||||
);
|
||||
let (formats, errors) = load_format_erasure_all(&disks, false).await;
|
||||
assert!(
|
||||
errors.iter().all(Option::is_none),
|
||||
@@ -1394,51 +1378,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compatible_single_drive_legacy_format_marks_adoption_proof() {
|
||||
let (_temp_dir, mut disks) = local_disks(1).await;
|
||||
let legacy = FormatV3::new(1, 1);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(¤t_ctx(), true, &mut disks, 1, 1, None)
|
||||
.await
|
||||
.expect("single-drive MinIO format should migrate");
|
||||
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::LegacyAdoption);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_migrated_format_keeps_legacy_adoption_proof() {
|
||||
let (_temp_dir, mut disks) = local_disks(1).await;
|
||||
let legacy = FormatV3::new(1, 1);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
|
||||
connect_load_init_formats_with_instance_ctx(¤t_ctx(), true, &mut disks, 1, 1, None)
|
||||
.await
|
||||
.expect("first run should migrate the MinIO format");
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(¤t_ctx(), true, &mut disks, 1, 1, None)
|
||||
.await
|
||||
.expect("retry after a partial adoption should reload the migrated RustFS format");
|
||||
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::LegacyAdoption);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_rustfs_format_without_legacy_source_is_not_legacy_adoption() {
|
||||
let (_temp_dir, mut disks) = local_disks(1).await;
|
||||
let mut format = FormatV3::new(1, 1);
|
||||
format.erasure.this = format.erasure.sets[0][0];
|
||||
save_format_file(&disks[0], &Some(format))
|
||||
.await
|
||||
.expect("existing RustFS format should be written");
|
||||
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(¤t_ctx(), true, &mut disks, 1, 1, None)
|
||||
.await
|
||||
.expect("existing RustFS format should load");
|
||||
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compatible_legacy_format_migrates_when_the_file_is_missing() {
|
||||
let (_temp_dir, mut disks) = local_disks(3).await;
|
||||
|
||||
@@ -97,8 +97,6 @@ pub(crate) struct BucketDeleteDiagnosticBudget {
|
||||
deadline: Option<tokio::time::Instant>,
|
||||
max_elapsed: Duration,
|
||||
entries_remaining: usize,
|
||||
#[cfg(test)]
|
||||
first_io_delay: Option<(Duration, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
}
|
||||
|
||||
impl BucketDeleteDiagnosticBudget {
|
||||
@@ -111,17 +109,9 @@ impl BucketDeleteDiagnosticBudget {
|
||||
deadline: None,
|
||||
max_elapsed: elapsed,
|
||||
entries_remaining: entries,
|
||||
#[cfg(test)]
|
||||
first_io_delay: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn with_first_io_delay(mut self, delay: Duration, started: Arc<std::sync::atomic::AtomicBool>) -> Self {
|
||||
self.first_io_delay = Some((delay, started));
|
||||
self
|
||||
}
|
||||
|
||||
fn deadline(&mut self) -> tokio::time::Instant {
|
||||
let max_elapsed = self.max_elapsed;
|
||||
*self.deadline.get_or_insert_with(|| tokio::time::Instant::now() + max_elapsed)
|
||||
@@ -147,20 +137,7 @@ impl BucketDeleteDiagnosticBudget {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Ok(None);
|
||||
}
|
||||
#[cfg(test)]
|
||||
let first_io_delay = self.first_io_delay.take();
|
||||
#[cfg(test)]
|
||||
let timeout_result = tokio::time::timeout_at(deadline, async move {
|
||||
if let Some((delay, started)) = first_io_delay {
|
||||
started.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
future.await
|
||||
})
|
||||
.await;
|
||||
#[cfg(not(test))]
|
||||
let timeout_result = tokio::time::timeout_at(deadline, future).await;
|
||||
match timeout_result {
|
||||
match tokio::time::timeout_at(deadline, future).await {
|
||||
Ok(result) => result.map(Some),
|
||||
Err(_) => Ok(None),
|
||||
}
|
||||
@@ -198,22 +175,6 @@ impl BucketDeleteBlockerKind {
|
||||
Self::DiagnosticBudgetExceeded => "diagnostic_budget_exceeded",
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the blocking residue is something the caller can still see and
|
||||
/// remove through the S3 API.
|
||||
///
|
||||
/// A live version or a tier free-version is ordinary: the bucket really is
|
||||
/// not empty, the client can list and delete what is left, and the 409 it
|
||||
/// receives is a complete answer.
|
||||
///
|
||||
/// The remaining kinds are not. They are on-disk state that no S3 request
|
||||
/// can reach: the caller has drained every version the API will show and
|
||||
/// `DeleteBucket` still refuses, with no way to find out why. That is a
|
||||
/// server-side integrity problem, and it is the reason this classification
|
||||
/// exists — see [`bucket_delete_blocker_level`].
|
||||
pub(crate) const fn is_client_visible(self) -> bool {
|
||||
matches!(self, Self::VisibleVersion | Self::TierFreeVersion)
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketMetadataLessResidue {
|
||||
@@ -425,8 +386,6 @@ pub(crate) mod init_format;
|
||||
pub(crate) mod list_objects;
|
||||
mod multipart;
|
||||
mod object;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use object::DeleteAfterObjectLockSnapshotBarrier;
|
||||
pub(crate) use object::{
|
||||
DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
|
||||
SourceCleanupMutationFence, tiered_data_movement_source_matches,
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::lifecycle::{
|
||||
bucket_lifecycle_ops::{enqueue_committed_free_versions, eval_action_from_lifecycle},
|
||||
bucket_lifecycle_ops::eval_action_from_lifecycle,
|
||||
get_expiry_configs,
|
||||
tier_delete_journal::{
|
||||
ActiveTierDeleteDispatch, EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_LIFECYCLE,
|
||||
@@ -39,7 +39,6 @@ use crate::core::pools::{DecommissionCapacityOwner, ensure_decommission_capacity
|
||||
use crate::disk::OldCurrentSize;
|
||||
use crate::object_api::{
|
||||
NamespaceLockFence, ObjectLockConfigSnapshot, ScannerPublicationCommitScopeGuard, ScannerPublicationCommitState,
|
||||
TierFreeVersionReceiptSink,
|
||||
};
|
||||
use crate::services::notification_sys::acquire_tier_delete_journal_fleet_proof;
|
||||
use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
|
||||
@@ -53,7 +52,6 @@ use crate::storage_api_contracts::{
|
||||
object::{DeleteAccounting, ObjectIO as _, ObjectOperations as _},
|
||||
};
|
||||
use parking_lot::Mutex as ParkingMutex;
|
||||
use rustfs_filemeta::ObjectPartInfo;
|
||||
use rustfs_io_metrics::{
|
||||
record_object_lock_diag_acquire_duration, record_object_lock_diag_hold_duration, record_object_lock_diag_slow_acquire,
|
||||
record_object_lock_diag_slow_hold,
|
||||
@@ -72,26 +70,6 @@ const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 1000;
|
||||
#[cfg(test)]
|
||||
const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 2;
|
||||
|
||||
fn install_tier_free_version_receipt_sink(opts: &mut ObjectOptions) -> Option<TierFreeVersionReceiptSink> {
|
||||
if opts.tier_free_version_receipt_sink.is_some() || opts.skip_free_version || opts.delete_prefix {
|
||||
return None;
|
||||
}
|
||||
|
||||
let sink = TierFreeVersionReceiptSink::new();
|
||||
opts.tier_free_version_receipt_sink = Some(sink.clone());
|
||||
Some(sink)
|
||||
}
|
||||
|
||||
async fn enqueue_recorded_tier_free_versions(store: &ECStore, sink: Option<TierFreeVersionReceiptSink>) -> usize {
|
||||
let Some(sink) = sink else {
|
||||
return 0;
|
||||
};
|
||||
let Ok(receipts) = sink.drain() else {
|
||||
return 0;
|
||||
};
|
||||
enqueue_committed_free_versions(store, receipts).await
|
||||
}
|
||||
|
||||
fn build_tier_delete_journal_entry(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -1314,7 +1292,7 @@ fn should_create_delete_marker_for_missing_object(opts: &ObjectOptions) -> bool
|
||||
(opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.delete_marker && !opts.data_movement
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
struct DeleteAfterObjectLockSnapshotBarrierState {
|
||||
bucket: String,
|
||||
arrived: tokio::sync::Notify,
|
||||
@@ -1323,19 +1301,19 @@ struct DeleteAfterObjectLockSnapshotBarrierState {
|
||||
namespace_acquired: AtomicBool,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub struct DeleteAfterObjectLockSnapshotBarrier {
|
||||
#[cfg(test)]
|
||||
pub(crate) struct DeleteAfterObjectLockSnapshotBarrier {
|
||||
state: Arc<DeleteAfterObjectLockSnapshotBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
static DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<DeleteAfterObjectLockSnapshotBarrierState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
impl DeleteAfterObjectLockSnapshotBarrier {
|
||||
pub fn install(bucket: &str) -> Self {
|
||||
pub(crate) fn install(bucket: &str) -> Self {
|
||||
let state = Arc::new(DeleteAfterObjectLockSnapshotBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
@@ -1352,15 +1330,15 @@ impl DeleteAfterObjectLockSnapshotBarrier {
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub async fn wait_until_paused(&self) {
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
self.state.arrived.notified().await;
|
||||
}
|
||||
|
||||
pub fn release(&self) {
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
|
||||
pub async fn release_and_wait_until_namespace_pending(&self) {
|
||||
pub(crate) async fn release_and_wait_until_namespace_pending(&self) {
|
||||
let namespace_pending = self.state.namespace_pending.notified();
|
||||
self.release();
|
||||
tokio::time::timeout(Duration::from_secs(5), namespace_pending)
|
||||
@@ -1368,12 +1346,12 @@ impl DeleteAfterObjectLockSnapshotBarrier {
|
||||
.expect("delete should proceed to its namespace lock after leaving the snapshot barrier");
|
||||
}
|
||||
|
||||
pub fn namespace_acquired(&self) -> bool {
|
||||
pub(crate) fn namespace_acquired(&self) -> bool {
|
||||
self.state.namespace_acquired.load(Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
impl Drop for DeleteAfterObjectLockSnapshotBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -1386,7 +1364,7 @@ impl Drop for DeleteAfterObjectLockSnapshotBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
async fn pause_delete_after_object_lock_snapshot(bucket: &str) {
|
||||
let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -1398,24 +1376,11 @@ async fn pause_delete_after_object_lock_snapshot(bucket: &str) {
|
||||
if let Some(state) = state {
|
||||
state.arrived.notify_one();
|
||||
state.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
fn notify_delete_namespace_pending(bucket: &str) {
|
||||
let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("delete snapshot barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|state| state.bucket == bucket)
|
||||
.cloned();
|
||||
if let Some(state) = state {
|
||||
state.namespace_pending.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
fn notify_delete_namespace_acquired(bucket: &str) {
|
||||
let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -2149,6 +2114,18 @@ fn remote_tuple_publication_object_source_matches(expected: &ObjectInfo, current
|
||||
let (Ok(expected_actual_size), Ok(current_actual_size)) = (expected.get_actual_size(), current.get_actual_size()) else {
|
||||
return false;
|
||||
};
|
||||
let parts_match = expected.parts.len() == current.parts.len()
|
||||
&& expected.parts.iter().all(|expected_part| {
|
||||
current
|
||||
.parts
|
||||
.iter()
|
||||
.find(|current_part| current_part.number == expected_part.number)
|
||||
.is_some_and(|current_part| {
|
||||
current_part.size == expected_part.size
|
||||
&& current_part.actual_size == expected_part.actual_size
|
||||
&& current_part.etag == expected_part.etag
|
||||
})
|
||||
});
|
||||
|
||||
expected.data_dir.is_some_and(|data_dir| !data_dir.is_nil())
|
||||
&& expected.data_dir == current.data_dir
|
||||
@@ -2156,7 +2133,6 @@ fn remote_tuple_publication_object_source_matches(expected: &ObjectInfo, current
|
||||
&& expected.delete_marker == current.delete_marker
|
||||
&& expected.size == current.size
|
||||
&& expected_actual_size == current_actual_size
|
||||
&& expected.etag == current.etag
|
||||
&& expected.checksum == current.checksum
|
||||
&& expected.mod_time == current.mod_time
|
||||
&& expected.storage_class == current.storage_class
|
||||
@@ -2178,24 +2154,7 @@ fn remote_tuple_publication_object_source_matches(expected: &ObjectInfo, current
|
||||
&& expected.transitioned_object.free_version == current.transitioned_object.free_version
|
||||
&& expected.transitioned_object.status == current.transitioned_object.status
|
||||
&& expected.transition_version_state == current.transition_version_state
|
||||
&& remote_tuple_publication_parts_match(&expected.parts, ¤t.parts)
|
||||
}
|
||||
|
||||
fn remote_tuple_publication_parts_match(expected: &[ObjectPartInfo], current: &[ObjectPartInfo]) -> bool {
|
||||
if expected.len() != current.len() {
|
||||
return false;
|
||||
}
|
||||
let Some(mut current_parts) = crate::data_movement::data_movement_parts_by_number(current) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
expected.iter().all(|expected_part| {
|
||||
current_parts.remove(&expected_part.number).is_some_and(|current_part| {
|
||||
current_part.size == expected_part.size
|
||||
&& current_part.actual_size == expected_part.actual_size
|
||||
&& current_part.etag == expected_part.etag
|
||||
})
|
||||
})
|
||||
&& parts_match
|
||||
}
|
||||
|
||||
impl RemoteTuplePublicationFence {
|
||||
@@ -2394,16 +2353,6 @@ impl ECStore {
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn is_equivalent_decommission_capacity_target(source: &ObjectInfo, target: &ObjectInfo) -> bool {
|
||||
source.bucket == target.bucket
|
||||
&& source.name == decode_dir_object(&target.name)
|
||||
&& if source.delete_marker {
|
||||
is_equivalent_data_movement_delete_marker(source, target)
|
||||
} else {
|
||||
crate::data_movement::is_equivalent_data_movement_object_identity(source, target, true, false)
|
||||
}
|
||||
}
|
||||
|
||||
/// Captures Object Lock state once for a batch of PUTs to the same bucket.
|
||||
/// `handle_put_object` only reuses the token for the same store, bucket,
|
||||
/// bucket incarnation, and Object Lock configuration revision.
|
||||
@@ -2598,10 +2547,6 @@ impl ECStore {
|
||||
let diag_enabled = is_object_lock_diag_enabled();
|
||||
let ns_lock = self.handle_new_ns_lock(bucket, object).await?;
|
||||
let acquire_start = Instant::now();
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if matches!(op, "delete_object" | "delete_objects") {
|
||||
notify_delete_namespace_pending(bucket);
|
||||
}
|
||||
let guard = ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
@@ -4193,16 +4138,7 @@ impl ECStore {
|
||||
opts: ObjectOptions,
|
||||
tier_journal_api: Option<Arc<ECStore>>,
|
||||
) -> Result<ObjectInfo> {
|
||||
Box::pin(async move {
|
||||
let mut opts = opts;
|
||||
let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
|
||||
let result = self
|
||||
.handle_delete_object_with_journal_inner(bucket, object, opts, tier_journal_api)
|
||||
.await;
|
||||
enqueue_recorded_tier_free_versions(self, receipt_sink).await;
|
||||
result
|
||||
})
|
||||
.await
|
||||
Box::pin(self.handle_delete_object_with_journal_inner(bucket, object, opts, tier_journal_api)).await
|
||||
}
|
||||
|
||||
async fn handle_delete_object_with_journal_inner(
|
||||
@@ -4276,7 +4212,7 @@ impl ECStore {
|
||||
if opts.delete_prefix && opts.expected_bucket_incarnation_id.is_none() {
|
||||
opts.expected_bucket_incarnation_id = current_bucket_incarnation_id;
|
||||
}
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
pause_delete_after_object_lock_snapshot(bucket).await;
|
||||
|
||||
if opts.delete_prefix && !opts.delete_prefix_object {
|
||||
@@ -4290,7 +4226,7 @@ impl ECStore {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
if _object_lock_guard.is_some() {
|
||||
notify_delete_namespace_acquired(bucket);
|
||||
}
|
||||
@@ -4504,9 +4440,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
if should_delete_from_all_pools(&opts, errs.len()) {
|
||||
let mut obj = self
|
||||
.delete_object_from_all_pools(bucket, object, &opts, &pinfo.object_info, errs)
|
||||
.await?;
|
||||
let mut obj = self.delete_object_from_all_pools(bucket, object, &opts, errs).await?;
|
||||
obj.name = decode_dir_object(object);
|
||||
return Ok(obj);
|
||||
}
|
||||
@@ -4580,25 +4514,6 @@ impl ECStore {
|
||||
objects: Vec<ObjectToDelete>,
|
||||
opts: ObjectOptions,
|
||||
tier_journal_api: Option<Arc<ECStore>>,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
|
||||
Box::pin(async move {
|
||||
let mut opts = opts;
|
||||
let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
|
||||
let result = self
|
||||
.handle_delete_objects_with_journal_and_accounting_inner(bucket, objects, opts, tier_journal_api)
|
||||
.await;
|
||||
enqueue_recorded_tier_free_versions(self, receipt_sink).await;
|
||||
result
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn handle_delete_objects_with_journal_and_accounting_inner(
|
||||
&self,
|
||||
bucket: &str,
|
||||
objects: Vec<ObjectToDelete>,
|
||||
opts: ObjectOptions,
|
||||
tier_journal_api: Option<Arc<ECStore>>,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
|
||||
// encode object name
|
||||
let objects: Vec<ObjectToDelete> = objects
|
||||
@@ -4683,7 +4598,7 @@ impl ECStore {
|
||||
StorageError::BucketNotFound(bucket.to_string()),
|
||||
);
|
||||
}
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
if current_bucket_incarnation_id.is_some() {
|
||||
pause_delete_after_object_lock_snapshot(bucket).await;
|
||||
}
|
||||
@@ -4691,7 +4606,7 @@ impl ECStore {
|
||||
Ok(guards) => guards,
|
||||
Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
|
||||
};
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
if !_object_lock_guards.is_empty() {
|
||||
notify_delete_namespace_acquired(bucket);
|
||||
}
|
||||
@@ -5908,271 +5823,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_delete_capacity_target_requires_matching_namespace_and_object_identity() {
|
||||
let source = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "directory/".to_string(),
|
||||
version_id: Some(Uuid::from_u128(1)),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
size: 10,
|
||||
etag: Some("source-etag".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
name: rustfs_utils::path::encode_dir_object(&source.name),
|
||||
..source.clone()
|
||||
};
|
||||
assert!(ECStore::is_equivalent_decommission_capacity_target(&source, &target));
|
||||
|
||||
let mismatched_identity = ObjectInfo {
|
||||
etag: Some("different-etag".to_string()),
|
||||
..target.clone()
|
||||
};
|
||||
assert!(!ECStore::is_equivalent_decommission_capacity_target(&source, &mismatched_identity));
|
||||
|
||||
let wrong_bucket = ObjectInfo {
|
||||
bucket: "other-bucket".to_string(),
|
||||
..target
|
||||
};
|
||||
assert!(!ECStore::is_equivalent_decommission_capacity_target(&source, &wrong_bucket));
|
||||
}
|
||||
|
||||
fn publication_part(number: usize) -> ObjectPartInfo {
|
||||
ObjectPartInfo {
|
||||
number,
|
||||
size: 100 + number,
|
||||
actual_size: i64::try_from(200 + number).expect("test part size should fit in i64"),
|
||||
etag: format!("part-etag-{number}"),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn publication_source(parts: Vec<ObjectPartInfo>) -> ObjectInfo {
|
||||
ObjectInfo {
|
||||
data_dir: Some(Uuid::new_v4()),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
size: 4096,
|
||||
actual_size: 4096,
|
||||
etag: Some("object-etag".to_string()),
|
||||
checksum: Some(Bytes::from_static(b"object-checksum")),
|
||||
mod_time: Some(time::OffsetDateTime::UNIX_EPOCH),
|
||||
parts: Arc::new(parts),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publication_parts_match_is_order_independent_and_bijective() {
|
||||
let ordered = vec![publication_part(1), publication_part(2), publication_part(3)];
|
||||
let mut reversed = ordered.clone();
|
||||
reversed.reverse();
|
||||
|
||||
assert!(remote_tuple_publication_parts_match(&[], &[]));
|
||||
assert!(remote_tuple_publication_parts_match(&ordered, &ordered));
|
||||
assert!(remote_tuple_publication_parts_match(&ordered, &reversed));
|
||||
assert!(!remote_tuple_publication_parts_match(&ordered[..2], &ordered));
|
||||
|
||||
let expected_duplicate = vec![publication_part(1), publication_part(1)];
|
||||
let current_unique = vec![publication_part(1), publication_part(2)];
|
||||
assert!(
|
||||
!remote_tuple_publication_parts_match(&expected_duplicate, ¤t_unique),
|
||||
"two expected entries must not reuse the same current part"
|
||||
);
|
||||
assert!(!remote_tuple_publication_parts_match(¤t_unique, &expected_duplicate));
|
||||
assert!(!remote_tuple_publication_parts_match(&expected_duplicate, &expected_duplicate));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publication_parts_match_preserves_exact_part_identity_contract() {
|
||||
let expected = vec![publication_part(1)];
|
||||
|
||||
let mut different_number = expected.clone();
|
||||
different_number[0].number = 2;
|
||||
assert!(!remote_tuple_publication_parts_match(&expected, &different_number));
|
||||
|
||||
let mut different_size = expected.clone();
|
||||
different_size[0].size += 1;
|
||||
assert!(!remote_tuple_publication_parts_match(&expected, &different_size));
|
||||
|
||||
let mut different_actual_size = expected.clone();
|
||||
different_actual_size[0].actual_size += 1;
|
||||
assert!(!remote_tuple_publication_parts_match(&expected, &different_actual_size));
|
||||
|
||||
let mut different_etag = expected.clone();
|
||||
different_etag[0].etag.push_str("-changed");
|
||||
assert!(!remote_tuple_publication_parts_match(&expected, &different_etag));
|
||||
|
||||
let mut ignored_fields = expected.clone();
|
||||
ignored_fields[0].index = Some(Bytes::from_static(b"different-index"));
|
||||
ignored_fields[0].checksums = Some(HashMap::from([("CRC32C".to_string(), "different".to_string())]));
|
||||
ignored_fields[0].mod_time = Some(time::OffsetDateTime::UNIX_EPOCH);
|
||||
assert!(
|
||||
remote_tuple_publication_parts_match(&expected, &ignored_fields),
|
||||
"the publication fence must retain its existing checksum/index/mod-time compatibility contract"
|
||||
);
|
||||
|
||||
let zero_actual_size = vec![ObjectPartInfo {
|
||||
actual_size: 0,
|
||||
..publication_part(1)
|
||||
}];
|
||||
assert!(
|
||||
!remote_tuple_publication_parts_match(&zero_actual_size, &expected),
|
||||
"the publication fence compares raw part actual sizes without the broader comparator's fallback"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publication_source_match_rejects_object_etag_and_identity_mutations() {
|
||||
let expected = publication_source(vec![publication_part(1)]);
|
||||
assert!(remote_tuple_publication_object_source_matches(&expected, &expected));
|
||||
|
||||
let mut current = expected.clone();
|
||||
current.etag = Some("changed-object-etag".to_string());
|
||||
assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
|
||||
|
||||
let mut current = expected.clone();
|
||||
current.checksum = Some(Bytes::from_static(b"changed-checksum"));
|
||||
assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
|
||||
|
||||
let mut current = expected.clone();
|
||||
current.actual_size += 1;
|
||||
assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
|
||||
|
||||
let mut current = expected.clone();
|
||||
current.data_dir = Some(Uuid::new_v4());
|
||||
assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
|
||||
|
||||
let mut current = expected.clone();
|
||||
current.version_id = Some(Uuid::new_v4());
|
||||
assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
|
||||
|
||||
let mut current = expected.clone();
|
||||
current.mod_time = current.mod_time.map(|mod_time| mod_time + time::Duration::SECOND);
|
||||
assert!(!remote_tuple_publication_object_source_matches(&expected, ¤t));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publication_source_match_preserves_effective_actual_size_compatibility() {
|
||||
let expected = publication_source(vec![publication_part(1)]);
|
||||
let current = ObjectInfo {
|
||||
actual_size: 0,
|
||||
..expected.clone()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
expected.get_actual_size().expect("expected size should be valid"),
|
||||
current.get_actual_size().expect("current size should be valid")
|
||||
);
|
||||
assert!(remote_tuple_publication_object_source_matches(&expected, ¤t));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publication_source_match_rejects_duplicate_parts_through_the_full_fence() {
|
||||
let expected = publication_source(vec![publication_part(1), publication_part(1)]);
|
||||
let current = ObjectInfo {
|
||||
parts: Arc::new(vec![publication_part(1), publication_part(2)]),
|
||||
..expected.clone()
|
||||
};
|
||||
|
||||
assert!(
|
||||
!remote_tuple_publication_object_source_matches(&expected, ¤t),
|
||||
"the full source fence must reject the original non-bijective false-positive"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publication_source_match_handles_10_000_reversed_parts_without_payload_cloning() {
|
||||
let parts = Arc::new((1..=10_000).map(publication_part).collect::<Vec<_>>());
|
||||
let mut reversed = parts.as_ref().clone();
|
||||
reversed.reverse();
|
||||
let expected = publication_source(Vec::new());
|
||||
let expected = ObjectInfo {
|
||||
parts: Arc::clone(&parts),
|
||||
..expected
|
||||
};
|
||||
let current = ObjectInfo {
|
||||
parts: Arc::new(reversed),
|
||||
..expected.clone()
|
||||
};
|
||||
let expected_parts = Arc::clone(&expected.parts);
|
||||
|
||||
assert!(remote_tuple_publication_object_source_matches(&expected, ¤t));
|
||||
assert!(Arc::ptr_eq(&expected.parts, &expected_parts));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_commit_guard_rejects_an_etag_only_source_change() {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store =
|
||||
Arc::new(new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await);
|
||||
let bucket = "publication-etag-source-change";
|
||||
let object = "source.bin";
|
||||
for set in [&first_set, &second_set] {
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("publication ETag test bucket should be created");
|
||||
}
|
||||
|
||||
let mut source_body = PutObjReader::from_vec(b"source body".to_vec());
|
||||
first_set
|
||||
.put_object(bucket, object, &mut source_body, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("publication source should be written");
|
||||
let source = first_set
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
include_part_checksums: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("publication source should be readable");
|
||||
let publication = store
|
||||
.acquire_remote_tuple_publication_fence(bucket, 0, &source, false)
|
||||
.await
|
||||
.expect("the source snapshot should produce a publication capability");
|
||||
|
||||
let changed = first_set
|
||||
.put_object_metadata(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
eval_metadata: Some(HashMap::from([("etag".to_string(), "changed-etag".to_string())])),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the source ETag should be updated in place");
|
||||
assert_ne!(source.etag, changed.etag);
|
||||
let unchanged_except_etag = ObjectInfo {
|
||||
etag: source.etag.clone(),
|
||||
..changed.clone()
|
||||
};
|
||||
assert!(
|
||||
remote_tuple_publication_object_source_matches(&source, &unchanged_except_etag),
|
||||
"the persisted source update fixture must differ only by object ETag"
|
||||
);
|
||||
|
||||
let encoded = encode_dir_object(object);
|
||||
let err = match publication.into_commit_guard(1, bucket, &encoded).await {
|
||||
Ok(_) => panic!("the publication guard must reject an ETag-only source change"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(matches!(err, Error::DataMovementOverwriteErr(_, _, _)));
|
||||
assert!(
|
||||
second_set
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"a rejected publication must not create a target object"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generic_data_movement_put_rejects_transition_ownership_without_capability() {
|
||||
let (_dirs, set) = make_local_set_disks(4, 2).await;
|
||||
@@ -7594,15 +7244,6 @@ mod tests {
|
||||
);
|
||||
drop(unified_future);
|
||||
|
||||
let batch_future =
|
||||
store.handle_delete_objects_with_journal_and_accounting("bucket", Vec::new(), ObjectOptions::default(), None);
|
||||
let batch_future_size = std::mem::size_of_val(&batch_future);
|
||||
assert!(
|
||||
batch_future_size <= 4 * 1024,
|
||||
"batch delete handler future must remain stack-bounded; measured {batch_future_size} bytes"
|
||||
);
|
||||
drop(batch_future);
|
||||
|
||||
let outer_future = store.handle_delete_object("bucket", "object", ObjectOptions::default());
|
||||
let outer_future_size = std::mem::size_of_val(&outer_future);
|
||||
assert!(
|
||||
|
||||
@@ -939,12 +939,8 @@ impl ECStore {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
exact: &ObjectInfo,
|
||||
errs: Vec<PoolErr>,
|
||||
) -> Result<ObjectInfo> {
|
||||
self.reconcile_decommission_capacity_before_exact_delete(bucket, object, opts, exact)
|
||||
.await?;
|
||||
|
||||
let mut results = Vec::with_capacity(errs.len());
|
||||
|
||||
for pe in errs.iter() {
|
||||
|
||||
@@ -347,9 +347,6 @@ pub struct HealChannelRequest {
|
||||
pub id: String,
|
||||
/// Disk ID for heal disk/erasure set task
|
||||
pub disk: Option<String>,
|
||||
/// Exact endpoints of replacement disks for an automatic erasure-set
|
||||
/// rebuild. An empty list retains the generic erasure-set heal behavior.
|
||||
pub heal_endpoints: Vec<String>,
|
||||
/// Bucket name
|
||||
pub bucket: String,
|
||||
/// Object prefix (optional)
|
||||
@@ -597,7 +594,6 @@ pub fn create_heal_request(
|
||||
timeout_seconds: None,
|
||||
source: HealRequestSource::Internal,
|
||||
disk: None,
|
||||
heal_endpoints: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -638,13 +634,12 @@ pub fn create_heal_response(
|
||||
}
|
||||
}
|
||||
|
||||
fn create_auto_heal_disk_request(set_disk_id: String, priority: Option<HealChannelPriority>) -> HealChannelRequest {
|
||||
HealChannelRequest {
|
||||
pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPriority>) -> Result<(), String> {
|
||||
let req = HealChannelRequest {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
bucket: "".to_string(),
|
||||
object_prefix: None,
|
||||
disk: Some(set_disk_id),
|
||||
heal_endpoints: Vec::new(),
|
||||
object_version_id: None,
|
||||
force_start: false,
|
||||
priority: priority.unwrap_or(HealChannelPriority::Low),
|
||||
@@ -659,71 +654,8 @@ fn create_auto_heal_disk_request(set_disk_id: String, priority: Option<HealChann
|
||||
no_lock: None,
|
||||
timeout_seconds: None,
|
||||
source: HealRequestSource::AutoHeal,
|
||||
}
|
||||
}
|
||||
|
||||
fn create_auto_replacement_disk_request(
|
||||
pool_index: usize,
|
||||
set_index: usize,
|
||||
replacement_endpoint: String,
|
||||
priority: Option<HealChannelPriority>,
|
||||
) -> HealChannelRequest {
|
||||
let mut request = create_auto_heal_disk_request(format!("pool_{pool_index}_set_{set_index}"), priority);
|
||||
request.heal_endpoints = vec![replacement_endpoint];
|
||||
request.pool_index = Some(pool_index);
|
||||
request.set_index = Some(set_index);
|
||||
request
|
||||
}
|
||||
|
||||
/// Submit the legacy generic erasure-set auto-heal request.
|
||||
pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPriority>) -> Result<(), String> {
|
||||
send_heal_request(create_auto_heal_disk_request(set_disk_id, priority)).await
|
||||
}
|
||||
|
||||
/// Submit an automatic replacement heal for one known disk endpoint.
|
||||
///
|
||||
/// The endpoint makes the request eligible for the durable replacement intent
|
||||
/// and completion-proof path in the heal task.
|
||||
pub async fn send_heal_replacement_disk(
|
||||
pool_index: usize,
|
||||
set_index: usize,
|
||||
replacement_endpoint: String,
|
||||
priority: Option<HealChannelPriority>,
|
||||
) -> Result<(), String> {
|
||||
send_heal_request(create_auto_replacement_disk_request(
|
||||
pool_index,
|
||||
set_index,
|
||||
replacement_endpoint,
|
||||
priority,
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod auto_heal_disk_request_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn replacement_disk_request_carries_its_exact_endpoint() {
|
||||
let request =
|
||||
create_auto_replacement_disk_request(2, 3, "http://node2:9000/drive3".to_string(), Some(HealChannelPriority::Normal));
|
||||
|
||||
assert_eq!(request.disk.as_deref(), Some("pool_2_set_3"));
|
||||
assert_eq!(request.heal_endpoints, ["http://node2:9000/drive3"]);
|
||||
assert_eq!(request.pool_index, Some(2));
|
||||
assert_eq!(request.set_index, Some(3));
|
||||
assert_eq!(request.source, HealRequestSource::AutoHeal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_auto_heal_disk_request_has_no_replacement_endpoint() {
|
||||
let request = create_auto_heal_disk_request("pool_2_set_3".to_string(), None);
|
||||
|
||||
assert!(request.heal_endpoints.is_empty());
|
||||
assert_eq!(request.pool_index, None);
|
||||
assert_eq!(request.set_index, None);
|
||||
assert_eq!(request.source, HealRequestSource::AutoHeal);
|
||||
}
|
||||
};
|
||||
send_heal_request(req).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -646,12 +646,10 @@ impl HealChannelProcessor {
|
||||
/// Convert channel request to heal request
|
||||
fn convert_to_heal_request(&self, request: HealChannelRequest) -> Result<HealRequest> {
|
||||
let recursive = request.recursive.unwrap_or(false);
|
||||
let mut inferred_set_scope = None;
|
||||
let heal_type = if let Some(disk_id) = &request.disk {
|
||||
let set_disk_id = utils::normalize_set_disk_id(disk_id).ok_or_else(|| Error::InvalidHealType {
|
||||
heal_type: format!("erasure-set({disk_id})"),
|
||||
})?;
|
||||
inferred_set_scope = utils::parse_set_disk_id(&set_disk_id).ok();
|
||||
HealType::ErasureSet {
|
||||
buckets: vec![],
|
||||
set_disk_id,
|
||||
@@ -714,14 +712,13 @@ impl HealChannelProcessor {
|
||||
dry_run: request.dry_run.unwrap_or(false),
|
||||
no_lock,
|
||||
timeout: request.timeout_seconds.map(std::time::Duration::from_secs),
|
||||
pool_index: request.pool_index.or_else(|| inferred_set_scope.map(|(pool, _)| pool)),
|
||||
set_index: request.set_index.or_else(|| inferred_set_scope.map(|(_, set)| set)),
|
||||
pool_index: request.pool_index,
|
||||
set_index: request.set_index,
|
||||
};
|
||||
|
||||
let mut heal_request = HealRequest::new(heal_type, options, priority);
|
||||
heal_request.id = request.id;
|
||||
heal_request.source = request.source;
|
||||
heal_request.heal_endpoints = request.heal_endpoints;
|
||||
// force_start controls admission/queue semantics only. Do not reinterpret it as
|
||||
// destructive heal options: admin clients commonly pass forceStart=true together
|
||||
// with remove=false, and turning that into remove_corrupted=true can delete the
|
||||
@@ -909,7 +906,6 @@ mod tests {
|
||||
object_prefix: None,
|
||||
object_version_id: None,
|
||||
disk: None,
|
||||
heal_endpoints: Vec::new(),
|
||||
priority: HealChannelPriority::Normal,
|
||||
scan_mode: None,
|
||||
remove_corrupted: None,
|
||||
@@ -942,7 +938,6 @@ mod tests {
|
||||
object_prefix: None,
|
||||
object_version_id: None,
|
||||
disk: None,
|
||||
heal_endpoints: Vec::new(),
|
||||
priority: HealChannelPriority::High,
|
||||
scan_mode: Some(HealScanMode::Normal),
|
||||
remove_corrupted: Some(false),
|
||||
@@ -975,7 +970,6 @@ mod tests {
|
||||
object_prefix: Some("test-object".to_string()),
|
||||
object_version_id: None,
|
||||
disk: None,
|
||||
heal_endpoints: Vec::new(),
|
||||
priority: HealChannelPriority::High,
|
||||
scan_mode: Some(HealScanMode::Deep),
|
||||
remove_corrupted: Some(true),
|
||||
@@ -1029,7 +1023,6 @@ mod tests {
|
||||
object_prefix: Some("test-object".to_string()),
|
||||
object_version_id: None,
|
||||
disk: None,
|
||||
heal_endpoints: Vec::new(),
|
||||
priority: HealChannelPriority::Low,
|
||||
scan_mode: None,
|
||||
remove_corrupted: None,
|
||||
@@ -1067,7 +1060,6 @@ mod tests {
|
||||
object_prefix: Some("test-object".to_string()),
|
||||
object_version_id: None,
|
||||
disk: None,
|
||||
heal_endpoints: Vec::new(),
|
||||
priority: HealChannelPriority::Normal,
|
||||
scan_mode: None,
|
||||
remove_corrupted: None,
|
||||
@@ -1107,7 +1099,6 @@ mod tests {
|
||||
object_prefix: Some("test-object".to_string()),
|
||||
object_version_id: None,
|
||||
disk: None,
|
||||
heal_endpoints: Vec::new(),
|
||||
priority: HealChannelPriority::Normal,
|
||||
scan_mode: None,
|
||||
remove_corrupted: None,
|
||||
@@ -1140,7 +1131,6 @@ mod tests {
|
||||
object_prefix: Some("logs/".to_string()),
|
||||
object_version_id: None,
|
||||
disk: None,
|
||||
heal_endpoints: Vec::new(),
|
||||
priority: HealChannelPriority::High,
|
||||
scan_mode: Some(HealScanMode::Normal),
|
||||
remove_corrupted: Some(false),
|
||||
@@ -1176,10 +1166,7 @@ mod tests {
|
||||
object_prefix: None,
|
||||
object_version_id: None,
|
||||
disk: Some("pool_0_set_1".to_string()),
|
||||
heal_endpoints: vec!["http://node0:9000/drive1".to_string()],
|
||||
priority: HealChannelPriority::Critical,
|
||||
pool_index: Some(0),
|
||||
set_index: Some(1),
|
||||
scan_mode: None,
|
||||
remove_corrupted: None,
|
||||
recreate_missing: None,
|
||||
@@ -1188,16 +1175,15 @@ mod tests {
|
||||
dry_run: None,
|
||||
no_lock: None,
|
||||
timeout_seconds: None,
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
force_start: false,
|
||||
source: HealRequestSource::AutoHeal,
|
||||
source: HealRequestSource::Internal,
|
||||
};
|
||||
|
||||
let heal_request = processor.convert_to_heal_request(channel_request).unwrap();
|
||||
assert!(matches!(heal_request.heal_type, HealType::ErasureSet { .. }));
|
||||
assert_eq!(heal_request.priority, HealPriority::Urgent);
|
||||
assert_eq!(heal_request.heal_endpoints, ["http://node0:9000/drive1"]);
|
||||
assert_eq!(heal_request.options.pool_index, Some(0));
|
||||
assert_eq!(heal_request.options.set_index, Some(1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1211,7 +1197,6 @@ mod tests {
|
||||
object_prefix: None,
|
||||
object_version_id: None,
|
||||
disk: Some("invalid-disk-id".to_string()),
|
||||
heal_endpoints: Vec::new(),
|
||||
priority: HealChannelPriority::Normal,
|
||||
scan_mode: None,
|
||||
remove_corrupted: None,
|
||||
@@ -1250,7 +1235,6 @@ mod tests {
|
||||
object_prefix: None,
|
||||
object_version_id: None,
|
||||
disk: None,
|
||||
heal_endpoints: Vec::new(),
|
||||
priority: channel_priority,
|
||||
scan_mode: None,
|
||||
remove_corrupted: None,
|
||||
@@ -1282,7 +1266,6 @@ mod tests {
|
||||
object_prefix: None,
|
||||
object_version_id: None,
|
||||
disk: None,
|
||||
heal_endpoints: Vec::new(),
|
||||
priority: HealChannelPriority::Normal,
|
||||
scan_mode: None,
|
||||
remove_corrupted: Some(false),
|
||||
@@ -1316,7 +1299,6 @@ mod tests {
|
||||
object_prefix: Some("".to_string()), // Empty prefix should be treated as bucket heal
|
||||
object_version_id: None,
|
||||
disk: None,
|
||||
heal_endpoints: Vec::new(),
|
||||
priority: HealChannelPriority::Normal,
|
||||
scan_mode: None,
|
||||
remove_corrupted: None,
|
||||
@@ -1354,7 +1336,6 @@ mod tests {
|
||||
object_prefix: Some("object".to_string()),
|
||||
object_version_id: None,
|
||||
disk: None,
|
||||
heal_endpoints: Vec::new(),
|
||||
priority: HealChannelPriority::Low,
|
||||
scan_mode: Some(HealScanMode::Normal),
|
||||
remove_corrupted: None,
|
||||
@@ -1633,7 +1614,6 @@ mod tests {
|
||||
object_prefix: None,
|
||||
object_version_id: None,
|
||||
disk: Some("invalid".to_string()),
|
||||
heal_endpoints: Vec::new(),
|
||||
priority: HealChannelPriority::Normal,
|
||||
scan_mode: None,
|
||||
remove_corrupted: None,
|
||||
|
||||
@@ -26,10 +26,12 @@ pub mod utils;
|
||||
|
||||
use storage_api::owner::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET,
|
||||
EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskOption,
|
||||
EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO,
|
||||
ObjectOperations, ecstore_local_disk_map_read, ecstore_new_disk,
|
||||
EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult,
|
||||
EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations,
|
||||
ecstore_local_disk_map_read,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use storage_api::owner::{EcstoreDiskOption, ecstore_new_disk};
|
||||
|
||||
pub use erasure_healer::ErasureSetHealer;
|
||||
pub use manager::{HealManager, HealOperationsSnapshot, HealPriorityCounts, HealSourceCounts};
|
||||
@@ -245,8 +247,10 @@ pub(crate) async fn local_disk_map_read() -> tokio::sync::OwnedRwLockReadGuard<L
|
||||
ecstore_local_disk_map_read().await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) type DiskOption = EcstoreDiskOption;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> DiskResult<DiskStore> {
|
||||
ecstore_new_disk(ep, opt).await
|
||||
}
|
||||
|
||||
@@ -14,9 +14,9 @@
|
||||
|
||||
use std::{fs, path::Path};
|
||||
|
||||
use super::{
|
||||
DiskOption, DiskStore, Endpoint, HealDiskExt as _, local_disk_map_read, new_disk, resume::ReplacementTargetIdentity,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use super::Endpoint;
|
||||
use super::{DiskStore, HealDiskExt as _, local_disk_map_read, resume::ReplacementTargetIdentity};
|
||||
|
||||
pub(crate) async fn auto_replacement_target_ready(disk: &DiskStore, local_disks: &[DiskStore]) -> bool {
|
||||
auto_replacement_target_identity(disk, local_disks).await.is_some()
|
||||
@@ -72,38 +72,8 @@ pub(crate) async fn auto_replacement_target_identity(
|
||||
.flatten()
|
||||
}
|
||||
|
||||
fn local_replacement_endpoint(target: &str, local_grid_hosts: &[String]) -> Option<Endpoint> {
|
||||
let mut endpoint = Endpoint::try_from(target).ok()?;
|
||||
if endpoint.is_local {
|
||||
return Some(endpoint);
|
||||
}
|
||||
|
||||
let grid_host = endpoint.grid_host();
|
||||
if grid_host.is_empty() || !local_grid_hosts.iter().any(|local_host| local_host == &grid_host) {
|
||||
return None;
|
||||
}
|
||||
|
||||
endpoint.is_local = true;
|
||||
Some(endpoint)
|
||||
}
|
||||
|
||||
async fn replacement_target_disk(target: &str, local_disks: &[DiskStore]) -> Option<DiskStore> {
|
||||
if let Some(disk) = local_disks.iter().find(|disk| disk.endpoint().to_string() == target) {
|
||||
return Some(disk.clone());
|
||||
}
|
||||
|
||||
let local_grid_hosts = local_disks.iter().map(|disk| disk.endpoint().grid_host()).collect::<Vec<_>>();
|
||||
let endpoint = local_replacement_endpoint(target, &local_grid_hosts)?;
|
||||
|
||||
new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.ok()
|
||||
pub(crate) async fn auto_replacement_targets_ready(targets: &[String]) -> bool {
|
||||
auto_replacement_target_identities(targets).await.is_some()
|
||||
}
|
||||
|
||||
pub(crate) async fn auto_replacement_target_identities(targets: &[String]) -> Option<Vec<ReplacementTargetIdentity>> {
|
||||
@@ -118,8 +88,8 @@ pub(crate) async fn auto_replacement_target_identities(targets: &[String]) -> Op
|
||||
|
||||
let mut identities = Vec::with_capacity(targets.len());
|
||||
for target in targets {
|
||||
let disk = replacement_target_disk(target, &local_disks).await?;
|
||||
identities.push(auto_replacement_target_identity(&disk, &local_disks).await?);
|
||||
let disk = local_disks.iter().find(|disk| disk.endpoint().to_string() == *target)?;
|
||||
identities.push(auto_replacement_target_identity(disk, &local_disks).await?);
|
||||
}
|
||||
identities.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
identities.dedup_by(|left, right| left.endpoint == right.endpoint);
|
||||
@@ -161,29 +131,6 @@ mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn local_replacement_endpoint_accepts_a_url_on_a_registered_local_grid_host() {
|
||||
let local_grid_hosts = vec!["http://127.0.0.1:9000".to_owned()];
|
||||
let endpoint = local_replacement_endpoint("http://127.0.0.1:9000/replacement", &local_grid_hosts)
|
||||
.expect("matching local grid host should be accepted");
|
||||
|
||||
assert!(endpoint.is_local);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_replacement_endpoint_rejects_a_url_on_an_unregistered_grid_host() {
|
||||
let local_grid_hosts = vec!["http://127.0.0.1:9000".to_owned()];
|
||||
|
||||
assert!(local_replacement_endpoint("http://127.0.0.1:9001/replacement", &local_grid_hosts).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_replacement_endpoint_keeps_a_local_path_local() {
|
||||
let endpoint = local_replacement_endpoint("/replacement", &[]).expect("local path should be accepted");
|
||||
|
||||
assert!(endpoint.is_local);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_environment_cannot_bypass_mount_admission() {
|
||||
temp_env::async_with_vars(
|
||||
|
||||
@@ -394,6 +394,11 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Err(Error::other("target-scoped replacement format is unsupported"))
|
||||
}
|
||||
|
||||
/// Recheck admitted replacement targets immediately before destructive work.
|
||||
async fn replacement_targets_ready(&self, _targets: &[String]) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Read target-specific physical evidence for one replacement version.
|
||||
///
|
||||
/// This is only used by automatic replacement healing after the normal
|
||||
@@ -1166,6 +1171,10 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn replacement_targets_ready(&self, targets: &[String]) -> Result<bool> {
|
||||
Ok(super::replacement_readiness::auto_replacement_targets_ready(targets).await)
|
||||
}
|
||||
|
||||
async fn replacement_targets_have_version(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -24,6 +24,7 @@ pub(crate) use rustfs_ecstore::api::disk::{
|
||||
DiskStore as EcstoreDiskStore, HEALING_MARKER_PATH as ECSTORE_HEALING_MARKER_PATH,
|
||||
RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
|
||||
pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read;
|
||||
@@ -42,6 +43,7 @@ pub(crate) mod owner {
|
||||
EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::{EcstoreDiskOption, ecstore_new_disk};
|
||||
}
|
||||
|
||||
|
||||
@@ -93,6 +93,16 @@ impl HealTask {
|
||||
None
|
||||
};
|
||||
|
||||
if is_auto_replacement
|
||||
&& !self
|
||||
.await_with_control(self.storage.replacement_targets_ready(&self.heal_endpoints))
|
||||
.await?
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement target is no longer ready for automatic heal {set_disk_id}"),
|
||||
});
|
||||
}
|
||||
|
||||
let replacement_resume_disk = if is_auto_replacement {
|
||||
Some(match replacement_resume_disk {
|
||||
Some(disk) => disk,
|
||||
|
||||
@@ -84,7 +84,7 @@ async fn automatic_replacement_uses_target_scoped_format() {
|
||||
let temp = TempDir::new().expect("temporary resume disk directory should be created");
|
||||
let disk = make_resume_disk(&temp).await;
|
||||
let storage = Arc::new(MockStorage {
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_targets_ready: Mutex::new(true),
|
||||
resume_disk: Mutex::new(Some(disk)),
|
||||
..Default::default()
|
||||
});
|
||||
@@ -123,7 +123,7 @@ async fn automatic_replacement_uses_target_scoped_format() {
|
||||
#[tokio::test]
|
||||
async fn automatic_replacement_persists_intent_before_format() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_targets_ready: Mutex::new(true),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::new(
|
||||
@@ -155,7 +155,7 @@ async fn automatic_replacement_persists_intent_before_format() {
|
||||
#[tokio::test]
|
||||
async fn recovered_replacement_never_uses_a_fresh_resume_disk() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_targets_ready: Mutex::new(true),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::new(
|
||||
@@ -193,7 +193,7 @@ async fn automatic_replacement_rejects_a_new_identity_after_format() {
|
||||
let first_identity = replacement_identity("replacement-a", "device-a", "filesystem-a");
|
||||
let second_identity = replacement_identity("replacement-a", "device-b", "filesystem-b");
|
||||
let storage = Arc::new(MockStorage {
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_targets_ready: Mutex::new(true),
|
||||
replacement_target_identity_sequences: Mutex::new(VecDeque::from([
|
||||
vec![first_identity.clone()],
|
||||
vec![first_identity.clone()],
|
||||
@@ -259,7 +259,7 @@ async fn automatic_replacement_reuses_an_existing_non_target_resume_anchor() {
|
||||
.await
|
||||
.expect("existing intent should be stored on the non-target anchor");
|
||||
let storage = Arc::new(MockStorage {
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_targets_ready: Mutex::new(true),
|
||||
replacement_resume_disk: Mutex::new(Some(anchor.clone())),
|
||||
..Default::default()
|
||||
});
|
||||
@@ -493,7 +493,7 @@ async fn verified_recovery_keeps_state_when_marker_clear_fails() {
|
||||
|
||||
let storage = Arc::new(MockStorage {
|
||||
replacement_resume_disk: Mutex::new(Some(anchor.clone())),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_targets_ready: Mutex::new(true),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::new(
|
||||
@@ -551,7 +551,7 @@ struct MockStorage {
|
||||
format_error: Mutex<Option<Error>>,
|
||||
global_format_calls: Mutex<u32>,
|
||||
replacement_format_calls: Mutex<Vec<(usize, usize, Vec<String>)>>,
|
||||
replacement_target_identities_ready: Mutex<bool>,
|
||||
replacement_targets_ready: Mutex<bool>,
|
||||
replacement_target_identity_sequences: Mutex<VecDeque<Vec<crate::heal::resume::ReplacementTargetIdentity>>>,
|
||||
listed_prefixes: Mutex<Vec<String>>,
|
||||
truncate_without_token: Mutex<bool>,
|
||||
@@ -943,6 +943,10 @@ impl HealStorageAPI for MockStorage {
|
||||
))
|
||||
}
|
||||
|
||||
async fn replacement_targets_ready(&self, _targets: &[String]) -> Result<bool> {
|
||||
Ok(*self.replacement_targets_ready.lock().unwrap())
|
||||
}
|
||||
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1024,7 +1028,7 @@ impl HealStorageAPI for MockStorage {
|
||||
&self,
|
||||
targets: &[String],
|
||||
) -> Result<Vec<crate::heal::resume::ReplacementTargetIdentity>> {
|
||||
if !*self.replacement_target_identities_ready.lock().unwrap() {
|
||||
if !*self.replacement_targets_ready.lock().unwrap() {
|
||||
return Err(Error::other("replacement target is not ready"));
|
||||
}
|
||||
if let Some(identities) = self.replacement_target_identity_sequences.lock().unwrap().pop_front() {
|
||||
|
||||
@@ -60,6 +60,7 @@ rustfs-storage-api.workspace = true
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
tracing.workspace = true
|
||||
url.workspace = true
|
||||
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rustfs_credentials::Credentials;
|
||||
use s3s::dto::*;
|
||||
|
||||
#[cfg(feature = "webdav")]
|
||||
@@ -28,33 +27,49 @@ pub trait StorageBackend: Send + Sync {
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
credentials: &Credentials,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
start_pos: Option<u64>,
|
||||
) -> Result<GetObjectOutput, Self::Error>;
|
||||
async fn get_object_range(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
credentials: &Credentials,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
start_pos: u64,
|
||||
length: u64,
|
||||
) -> Result<GetObjectOutput, Self::Error>;
|
||||
/// Put object content with metadata
|
||||
async fn put_object(&self, input: PutObjectInput, credentials: &Credentials) -> Result<PutObjectOutput, Self::Error>;
|
||||
async fn put_object(&self, input: PutObjectInput, access_key: &str, secret_key: &str)
|
||||
-> Result<PutObjectOutput, Self::Error>;
|
||||
/// Delete an object
|
||||
async fn delete_object(&self, bucket: &str, key: &str, credentials: &Credentials) -> Result<DeleteObjectOutput, Self::Error>;
|
||||
async fn delete_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<DeleteObjectOutput, Self::Error>;
|
||||
/// Get object metadata without content
|
||||
async fn head_object(&self, bucket: &str, key: &str, credentials: &Credentials) -> Result<HeadObjectOutput, Self::Error>;
|
||||
async fn head_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<HeadObjectOutput, Self::Error>;
|
||||
/// Check if bucket exists and get metadata
|
||||
async fn head_bucket(&self, bucket: &str, credentials: &Credentials) -> Result<HeadBucketOutput, Self::Error>;
|
||||
async fn head_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<HeadBucketOutput, Self::Error>;
|
||||
/// List objects in a bucket with pagination
|
||||
async fn list_objects_v2(
|
||||
&self,
|
||||
input: ListObjectsV2Input,
|
||||
credentials: &Credentials,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<ListObjectsV2Output, Self::Error>;
|
||||
/// List all buckets (requires authentication).
|
||||
async fn list_buckets(&self, credentials: &Credentials) -> Result<ListBucketsOutput, Self::Error>;
|
||||
async fn list_buckets(&self, access_key: &str, secret_key: &str) -> Result<ListBucketsOutput, Self::Error>;
|
||||
/// List buckets visible to the authenticated session.
|
||||
///
|
||||
/// Backends that implement this must apply per-bucket authorization. The default denies the
|
||||
@@ -72,15 +87,20 @@ pub trait StorageBackend: Send + Sync {
|
||||
))
|
||||
}
|
||||
/// Create a new bucket
|
||||
async fn create_bucket(&self, bucket: &str, credentials: &Credentials) -> Result<CreateBucketOutput, Self::Error>;
|
||||
async fn create_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<CreateBucketOutput, Self::Error>;
|
||||
/// Delete a bucket (must be empty)
|
||||
async fn delete_bucket(&self, bucket: &str, credentials: &Credentials) -> Result<DeleteBucketOutput, Self::Error>;
|
||||
async fn delete_bucket(&self, bucket: &str, access_key: &str, secret_key: &str) -> Result<DeleteBucketOutput, Self::Error>;
|
||||
/// Server-side copy of an object from one bucket+key to another.
|
||||
/// The input carries the full S3 surface (content type, metadata map,
|
||||
/// metadata directive, storage class, SSE config, conditional-copy
|
||||
/// headers) so protocol drivers can map client-supplied metadata
|
||||
/// onto the destination object.
|
||||
async fn copy_object(&self, input: CopyObjectInput, credentials: &Credentials) -> Result<CopyObjectOutput, Self::Error>;
|
||||
async fn copy_object(
|
||||
&self,
|
||||
input: CopyObjectInput,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<CopyObjectOutput, Self::Error>;
|
||||
/// Initiate a multipart upload. Returns an upload_id that identifies
|
||||
/// the in-progress upload for subsequent UploadPart, CompleteMultipartUpload,
|
||||
/// and AbortMultipartUpload calls. The input carries the full S3 surface
|
||||
@@ -90,18 +110,25 @@ pub trait StorageBackend: Send + Sync {
|
||||
async fn create_multipart_upload(
|
||||
&self,
|
||||
input: CreateMultipartUploadInput,
|
||||
credentials: &Credentials,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<CreateMultipartUploadOutput, Self::Error>;
|
||||
/// Upload one part of a multipart upload. The part_number must be in
|
||||
/// the range 1 to the 10 000-part S3 limit. The returned ETag
|
||||
/// identifies the part in the subsequent CompleteMultipartUpload call.
|
||||
async fn upload_part(&self, input: UploadPartInput, credentials: &Credentials) -> Result<UploadPartOutput, Self::Error>;
|
||||
async fn upload_part(
|
||||
&self,
|
||||
input: UploadPartInput,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<UploadPartOutput, Self::Error>;
|
||||
/// Assemble the parts listed in the input into the final object.
|
||||
/// The parts list must be sorted by part_number with no duplicates.
|
||||
async fn complete_multipart_upload(
|
||||
&self,
|
||||
input: CompleteMultipartUploadInput,
|
||||
credentials: &Credentials,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<CompleteMultipartUploadOutput, Self::Error>;
|
||||
/// Abort an in-progress multipart upload. Releases any storage
|
||||
/// associated with the upload_id. Idempotent: calling abort on an
|
||||
@@ -111,7 +138,8 @@ pub trait StorageBackend: Send + Sync {
|
||||
async fn abort_multipart_upload(
|
||||
&self,
|
||||
input: AbortMultipartUploadInput,
|
||||
credentials: &Credentials,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<AbortMultipartUploadOutput, Self::Error>;
|
||||
/// Copy a byte range from an existing object into a part of an
|
||||
/// in-progress multipart upload. Used by rename for objects larger
|
||||
@@ -119,6 +147,7 @@ pub trait StorageBackend: Send + Sync {
|
||||
async fn upload_part_copy(
|
||||
&self,
|
||||
input: UploadPartCopyInput,
|
||||
credentials: &Credentials,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<UploadPartCopyOutput, Self::Error>;
|
||||
}
|
||||
|
||||
@@ -35,7 +35,6 @@ use crate::common::session::SessionContext;
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures_util::stream::{self, StreamExt};
|
||||
use rustfs_credentials::Credentials;
|
||||
use s3s::dto::{
|
||||
AbortMultipartUploadInput, AbortMultipartUploadOutput, CompleteMultipartUploadInput, CompleteMultipartUploadOutput,
|
||||
CopyObjectInput, CopyObjectOutput, CopyPartResult, CreateBucketOutput, CreateMultipartUploadInput,
|
||||
@@ -606,7 +605,8 @@ impl StorageBackend for DummyBackend {
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
_credentials: &Credentials,
|
||||
_ak: &str,
|
||||
_sk: &str,
|
||||
_start_pos: Option<u64>,
|
||||
) -> Result<GetObjectOutput, Self::Error> {
|
||||
match self.inner.lock().expect("lock").get_object.pop_front() {
|
||||
@@ -619,7 +619,8 @@ impl StorageBackend for DummyBackend {
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
_credentials: &Credentials,
|
||||
_ak: &str,
|
||||
_sk: &str,
|
||||
_start_pos: u64,
|
||||
_length: u64,
|
||||
) -> Result<GetObjectOutput, Self::Error> {
|
||||
@@ -629,7 +630,7 @@ impl StorageBackend for DummyBackend {
|
||||
}
|
||||
}
|
||||
|
||||
async fn put_object(&self, input: PutObjectInput, _credentials: &Credentials) -> Result<PutObjectOutput, Self::Error> {
|
||||
async fn put_object(&self, input: PutObjectInput, _ak: &str, _sk: &str) -> Result<PutObjectOutput, Self::Error> {
|
||||
// Decide control flow while holding the lock. Release before
|
||||
// awaiting so the stall path does not hold the Mutex across
|
||||
// an await point.
|
||||
@@ -658,12 +659,7 @@ impl StorageBackend for DummyBackend {
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
_credentials: &Credentials,
|
||||
) -> Result<DeleteObjectOutput, Self::Error> {
|
||||
async fn delete_object(&self, bucket: &str, key: &str, _ak: &str, _sk: &str) -> Result<DeleteObjectOutput, Self::Error> {
|
||||
let mut inner = self.inner.lock().expect("lock");
|
||||
inner.delete_object_calls.push(DeleteObjectCall {
|
||||
bucket: bucket.to_string(),
|
||||
@@ -675,7 +671,7 @@ impl StorageBackend for DummyBackend {
|
||||
}
|
||||
}
|
||||
|
||||
async fn head_object(&self, bucket: &str, key: &str, _credentials: &Credentials) -> Result<HeadObjectOutput, Self::Error> {
|
||||
async fn head_object(&self, bucket: &str, key: &str, _ak: &str, _sk: &str) -> Result<HeadObjectOutput, Self::Error> {
|
||||
{
|
||||
let mut inner = self.inner.lock().expect("lock");
|
||||
inner.head_object_calls.push(HeadObjectCall {
|
||||
@@ -689,7 +685,7 @@ impl StorageBackend for DummyBackend {
|
||||
}
|
||||
}
|
||||
|
||||
async fn head_bucket(&self, bucket: &str, _credentials: &Credentials) -> Result<HeadBucketOutput, Self::Error> {
|
||||
async fn head_bucket(&self, bucket: &str, _ak: &str, _sk: &str) -> Result<HeadBucketOutput, Self::Error> {
|
||||
match self.inner.lock().expect("lock").head_bucket.pop_front() {
|
||||
Some(r) => r,
|
||||
None => Err(DummyError::NoSuchBucket(bucket.to_string())),
|
||||
@@ -699,7 +695,8 @@ impl StorageBackend for DummyBackend {
|
||||
async fn list_objects_v2(
|
||||
&self,
|
||||
_input: ListObjectsV2Input,
|
||||
_credentials: &Credentials,
|
||||
_ak: &str,
|
||||
_sk: &str,
|
||||
) -> Result<ListObjectsV2Output, Self::Error> {
|
||||
// Decide control flow while holding the lock. Release before
|
||||
// awaiting so the stall path does not hold the Mutex across
|
||||
@@ -724,7 +721,7 @@ impl StorageBackend for DummyBackend {
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_buckets(&self, _credentials: &Credentials) -> Result<ListBucketsOutput, Self::Error> {
|
||||
async fn list_buckets(&self, _access_key: &str, _secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
|
||||
match self.inner.lock().expect("lock").list_buckets.pop_front() {
|
||||
Some(r) => r,
|
||||
None => Ok(ListBucketsOutput::default()),
|
||||
@@ -747,14 +744,14 @@ impl StorageBackend for DummyBackend {
|
||||
.unwrap_or_else(|| Ok(ListBucketsOutput::default()))
|
||||
}
|
||||
|
||||
async fn create_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<CreateBucketOutput, Self::Error> {
|
||||
async fn create_bucket(&self, _bucket: &str, _ak: &str, _sk: &str) -> Result<CreateBucketOutput, Self::Error> {
|
||||
match self.inner.lock().expect("lock").create_bucket.pop_front() {
|
||||
Some(r) => r,
|
||||
None => Err(DummyError::Unconfigured("create_bucket")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_bucket(&self, bucket: &str, _credentials: &Credentials) -> Result<DeleteBucketOutput, Self::Error> {
|
||||
async fn delete_bucket(&self, bucket: &str, _ak: &str, _sk: &str) -> Result<DeleteBucketOutput, Self::Error> {
|
||||
let mut inner = self.inner.lock().expect("lock");
|
||||
inner.delete_bucket_calls.push(bucket.to_string());
|
||||
match inner.delete_bucket.pop_front() {
|
||||
@@ -763,7 +760,7 @@ impl StorageBackend for DummyBackend {
|
||||
}
|
||||
}
|
||||
|
||||
async fn copy_object(&self, _input: CopyObjectInput, _credentials: &Credentials) -> Result<CopyObjectOutput, Self::Error> {
|
||||
async fn copy_object(&self, _input: CopyObjectInput, _ak: &str, _sk: &str) -> Result<CopyObjectOutput, Self::Error> {
|
||||
match self.inner.lock().expect("lock").copy_object.pop_front() {
|
||||
Some(r) => r,
|
||||
None => Err(DummyError::Unconfigured("copy_object")),
|
||||
@@ -773,7 +770,8 @@ impl StorageBackend for DummyBackend {
|
||||
async fn create_multipart_upload(
|
||||
&self,
|
||||
input: CreateMultipartUploadInput,
|
||||
_credentials: &Credentials,
|
||||
_ak: &str,
|
||||
_sk: &str,
|
||||
) -> Result<CreateMultipartUploadOutput, Self::Error> {
|
||||
{
|
||||
let mut inner = self.inner.lock().expect("lock");
|
||||
@@ -789,7 +787,7 @@ impl StorageBackend for DummyBackend {
|
||||
}
|
||||
}
|
||||
|
||||
async fn upload_part(&self, input: UploadPartInput, _credentials: &Credentials) -> Result<UploadPartOutput, Self::Error> {
|
||||
async fn upload_part(&self, input: UploadPartInput, _ak: &str, _sk: &str) -> Result<UploadPartOutput, Self::Error> {
|
||||
// Record the call and decide the control flow while holding the
|
||||
// lock. Release the lock before awaiting so the stall path does
|
||||
// not hold the Mutex across an await point.
|
||||
@@ -823,7 +821,8 @@ impl StorageBackend for DummyBackend {
|
||||
async fn complete_multipart_upload(
|
||||
&self,
|
||||
input: CompleteMultipartUploadInput,
|
||||
_credentials: &Credentials,
|
||||
_ak: &str,
|
||||
_sk: &str,
|
||||
) -> Result<CompleteMultipartUploadOutput, Self::Error> {
|
||||
let part_count = input
|
||||
.multipart_upload
|
||||
@@ -848,7 +847,8 @@ impl StorageBackend for DummyBackend {
|
||||
async fn abort_multipart_upload(
|
||||
&self,
|
||||
input: AbortMultipartUploadInput,
|
||||
_credentials: &Credentials,
|
||||
_ak: &str,
|
||||
_sk: &str,
|
||||
) -> Result<AbortMultipartUploadOutput, Self::Error> {
|
||||
{
|
||||
let mut inner = self.inner.lock().expect("lock");
|
||||
@@ -867,7 +867,8 @@ impl StorageBackend for DummyBackend {
|
||||
async fn upload_part_copy(
|
||||
&self,
|
||||
_input: UploadPartCopyInput,
|
||||
_credentials: &Credentials,
|
||||
_ak: &str,
|
||||
_sk: &str,
|
||||
) -> Result<UploadPartCopyOutput, Self::Error> {
|
||||
match self.inner.lock().expect("lock").upload_part_copy.pop_front() {
|
||||
Some(r) => r,
|
||||
@@ -883,8 +884,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn dummy_backend_reports_not_found_by_default() {
|
||||
let backend = DummyBackend::new();
|
||||
let credentials = Credentials::default();
|
||||
let result = backend.head_object("b", "k", &credentials).await;
|
||||
let result = backend.head_object("b", "k", "ak", "sk").await;
|
||||
let Err(err) = result else {
|
||||
panic!("default head_object must return an error");
|
||||
};
|
||||
@@ -897,23 +897,21 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn dummy_backend_returns_queued_head_object_response() {
|
||||
let backend = DummyBackend::new();
|
||||
let credentials = Credentials::default();
|
||||
backend.queue_head_object_ok(42, None);
|
||||
let out = backend.head_object("b", "k", &credentials).await.expect("queued Ok");
|
||||
let out = backend.head_object("b", "k", "ak", "sk").await.expect("queued Ok");
|
||||
assert_eq!(out.content_length, Some(42));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dummy_backend_logs_abort_multipart_calls() {
|
||||
let backend = Arc::new(DummyBackend::new());
|
||||
let credentials = Credentials::default();
|
||||
let input = AbortMultipartUploadInput::builder()
|
||||
.bucket("b".to_string())
|
||||
.key("k".to_string())
|
||||
.upload_id("UP-1".to_string())
|
||||
.build()
|
||||
.expect("build");
|
||||
backend.abort_multipart_upload(input, &credentials).await.expect("Ok");
|
||||
backend.abort_multipart_upload(input, "ak", "sk").await.expect("Ok");
|
||||
let calls = backend.abort_multipart_calls();
|
||||
assert_eq!(calls.len(), 1);
|
||||
assert_eq!(calls[0].upload_id, "UP-1");
|
||||
@@ -922,7 +920,6 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn dummy_backend_unconfigured_errors_loudly() {
|
||||
let backend = DummyBackend::new();
|
||||
let credentials = Credentials::default();
|
||||
let err = backend
|
||||
.create_multipart_upload(
|
||||
CreateMultipartUploadInput::builder()
|
||||
@@ -930,7 +927,8 @@ mod tests {
|
||||
.key("k".to_string())
|
||||
.build()
|
||||
.expect("build"),
|
||||
&credentials,
|
||||
"ak",
|
||||
"sk",
|
||||
)
|
||||
.await
|
||||
.expect_err("default create_multipart_upload must error");
|
||||
|
||||
@@ -288,7 +288,12 @@ pub async fn is_authorized(
|
||||
}
|
||||
};
|
||||
|
||||
let claims = policy_claims_for_session(session_context);
|
||||
// Create policy arguments
|
||||
let mut claims = HashMap::new();
|
||||
claims.insert(
|
||||
"principal".to_string(),
|
||||
serde_json::Value::String(session_context.principal.access_key().to_string()),
|
||||
);
|
||||
|
||||
let policy_action: rustfs_policy::policy::action::Action = action.clone().into();
|
||||
|
||||
@@ -310,21 +315,6 @@ pub async fn is_authorized(
|
||||
Ok(iam_sys.is_allowed(&args).await)
|
||||
}
|
||||
|
||||
fn policy_claims_for_session(session_context: &SessionContext) -> HashMap<String, serde_json::Value> {
|
||||
let mut claims = session_context
|
||||
.principal
|
||||
.user_identity
|
||||
.credentials
|
||||
.claims
|
||||
.clone()
|
||||
.unwrap_or_default();
|
||||
claims.insert(
|
||||
"principal".to_string(),
|
||||
serde_json::Value::String(session_context.principal.access_key().to_string()),
|
||||
);
|
||||
claims
|
||||
}
|
||||
|
||||
/// Authorize an operation and return an error if not authorized.
|
||||
/// AccessDenied covers both the protocol-not-supported case and the
|
||||
/// policy-denies case. IamUnavailable propagates from is_authorized
|
||||
@@ -467,9 +457,7 @@ pub use test_auth_override::{with_test_auth_override, with_test_iam_unavailable}
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::common::session::{Protocol, ProtocolPrincipal, SessionContext};
|
||||
use rustfs_credentials::{IAM_POLICY_CLAIM_NAME_SA, INHERITED_POLICY_TYPE};
|
||||
use rustfs_policy::auth::UserIdentity;
|
||||
use serde_json::Value;
|
||||
use std::net::{IpAddr, Ipv4Addr};
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -478,44 +466,6 @@ mod tests {
|
||||
SessionContext::new(principal, Protocol::Sftp, IpAddr::V4(Ipv4Addr::LOCALHOST))
|
||||
}
|
||||
|
||||
fn session_with_claims(access_key: &str, claims: HashMap<String, Value>) -> SessionContext {
|
||||
let identity = UserIdentity::new(rustfs_credentials::Credentials {
|
||||
access_key: access_key.to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
claims: Some(claims),
|
||||
..Default::default()
|
||||
});
|
||||
let principal = ProtocolPrincipal::new(Arc::new(identity));
|
||||
SessionContext::new(principal, Protocol::WebDav, IpAddr::V4(Ipv4Addr::LOCALHOST))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_claims_preserve_authenticated_service_account_claims() {
|
||||
let parent = "parent-user";
|
||||
let mut stored_claims = HashMap::new();
|
||||
stored_claims.insert("parent".to_string(), Value::String(parent.to_string()));
|
||||
stored_claims.insert(IAM_POLICY_CLAIM_NAME_SA.to_string(), Value::String(INHERITED_POLICY_TYPE.to_string()));
|
||||
let session = session_with_claims("service-account", stored_claims);
|
||||
|
||||
let claims = policy_claims_for_session(&session);
|
||||
|
||||
assert_eq!(claims.get("parent").and_then(Value::as_str), Some(parent));
|
||||
assert_eq!(claims.get(IAM_POLICY_CLAIM_NAME_SA).and_then(Value::as_str), Some(INHERITED_POLICY_TYPE));
|
||||
assert_eq!(claims.get("principal").and_then(Value::as_str), Some("service-account"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_claims_overwrite_untrusted_principal_claim() {
|
||||
let session = session_with_claims(
|
||||
"authenticated-service-account",
|
||||
HashMap::from([("principal".to_string(), Value::String("forged-principal".to_string()))]),
|
||||
);
|
||||
|
||||
let claims = policy_claims_for_session(&session);
|
||||
|
||||
assert_eq!(claims.get("principal").and_then(Value::as_str), Some("authenticated-service-account"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn with_test_auth_override_allow_returns_ok() {
|
||||
let session = test_session();
|
||||
|
||||
@@ -84,11 +84,6 @@ impl SessionContext {
|
||||
pub fn access_key(&self) -> &str {
|
||||
self.principal.access_key()
|
||||
}
|
||||
|
||||
/// Get the authenticated credentials for this session.
|
||||
pub fn credentials(&self) -> &Credentials {
|
||||
&self.principal.user_identity.credentials
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a SessionContext suitable for driver-level unit tests. The
|
||||
|
||||
@@ -129,7 +129,14 @@ where
|
||||
}
|
||||
|
||||
let mut list_result = Vec::new();
|
||||
match self.storage.list_buckets(session_context.credentials()).await {
|
||||
match self
|
||||
.storage
|
||||
.list_buckets(
|
||||
&session_context.principal.user_identity.credentials.access_key,
|
||||
&session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(output) => {
|
||||
if let Some(buckets) = output.buckets {
|
||||
for bucket in buckets {
|
||||
@@ -183,7 +190,15 @@ where
|
||||
Error::new(ErrorKind::PermanentFileNotAvailable, format!("Failed to build ListObjectsV2Input: {}", e))
|
||||
})?;
|
||||
|
||||
if let Ok(output) = self.storage.list_objects_v2(list_input, session_context.credentials()).await {
|
||||
if let Ok(output) = self
|
||||
.storage
|
||||
.list_objects_v2(
|
||||
list_input,
|
||||
&session_context.principal.user_identity.credentials.access_key,
|
||||
&session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
// Delete all objects in this page
|
||||
if let Some(objects) = output.contents {
|
||||
for obj in objects {
|
||||
@@ -194,7 +209,12 @@ where
|
||||
|
||||
let _ = self
|
||||
.storage
|
||||
.delete_object(bucket, &obj_key, session_context.credentials())
|
||||
.delete_object(
|
||||
bucket,
|
||||
&obj_key,
|
||||
&session_context.principal.user_identity.credentials.access_key,
|
||||
&session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
@@ -211,7 +231,15 @@ where
|
||||
}
|
||||
|
||||
// Then delete the bucket
|
||||
match self.storage.delete_bucket(bucket, session_context.credentials()).await {
|
||||
match self
|
||||
.storage
|
||||
.delete_bucket(
|
||||
bucket,
|
||||
&session_context.principal.user_identity.credentials.access_key,
|
||||
&session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) if e.to_string().contains("NoSuchBucket") => Ok(()),
|
||||
Err(e) => {
|
||||
@@ -249,7 +277,16 @@ where
|
||||
.await
|
||||
.map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
|
||||
|
||||
match self.storage.head_object(&bucket, &key, session_context.credentials()).await {
|
||||
match self
|
||||
.storage
|
||||
.head_object(
|
||||
&bucket,
|
||||
&key,
|
||||
&session_context.principal.user_identity.credentials.access_key,
|
||||
&session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(output) => {
|
||||
let size = output.content_length.unwrap_or(0) as u64;
|
||||
let modified = output.last_modified.map(|dt| {
|
||||
@@ -286,7 +323,15 @@ where
|
||||
.map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
|
||||
|
||||
let bucket_clone = bucket.clone();
|
||||
match self.storage.head_bucket(&bucket, session_context.credentials()).await {
|
||||
match self
|
||||
.storage
|
||||
.head_bucket(
|
||||
&bucket,
|
||||
&session_context.principal.user_identity.credentials.access_key,
|
||||
&session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(FtpsMetadata {
|
||||
size: 0,
|
||||
modified: Some(std::time::SystemTime::now()),
|
||||
@@ -345,7 +390,15 @@ where
|
||||
Error::new(ErrorKind::PermanentFileNotAvailable, format!("Failed to build ListObjectsV2Input: {}", e))
|
||||
})?;
|
||||
|
||||
match self.storage.list_objects_v2(list_input, session_context.credentials()).await {
|
||||
match self
|
||||
.storage
|
||||
.list_objects_v2(
|
||||
list_input,
|
||||
&session_context.principal.user_identity.credentials.access_key,
|
||||
&session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(output) => {
|
||||
let mut fileinfos = Vec::new();
|
||||
|
||||
@@ -462,7 +515,8 @@ where
|
||||
.get_object(
|
||||
&bucket,
|
||||
&key,
|
||||
session_context.credentials(),
|
||||
&session_context.principal.user_identity.credentials.access_key,
|
||||
&session_context.principal.user_identity.credentials.secret_key,
|
||||
Some(start_pos), // Pass start_pos for range request
|
||||
)
|
||||
.await
|
||||
@@ -570,7 +624,15 @@ where
|
||||
.build()
|
||||
.map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Failed to build PutObjectInput"))?;
|
||||
|
||||
match self.storage.put_object(put_input, session_context.credentials()).await {
|
||||
match self
|
||||
.storage
|
||||
.put_object(
|
||||
put_input,
|
||||
&session_context.principal.user_identity.credentials.access_key,
|
||||
&session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_output) => {
|
||||
Ok(file_size as u64) // Return the size of the uploaded object
|
||||
}
|
||||
@@ -619,7 +681,16 @@ where
|
||||
.map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
|
||||
|
||||
// Delete file
|
||||
match self.storage.delete_object(&bucket, &key, session_context.credentials()).await {
|
||||
match self
|
||||
.storage
|
||||
.delete_object(
|
||||
&bucket,
|
||||
&key,
|
||||
&session_context.principal.user_identity.credentials.access_key,
|
||||
&session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => {
|
||||
error!(
|
||||
@@ -677,7 +748,15 @@ where
|
||||
.map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
|
||||
|
||||
// Create bucket for directory
|
||||
match self.storage.create_bucket(&bucket, session_context.credentials()).await {
|
||||
match self
|
||||
.storage
|
||||
.create_bucket(
|
||||
&bucket,
|
||||
&session_context.principal.user_identity.credentials.access_key,
|
||||
&session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
event = EVENT_FTPS_DIRECTORY_STATE,
|
||||
@@ -777,7 +856,15 @@ where
|
||||
.map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
|
||||
|
||||
// Check if bucket exists
|
||||
match self.storage.head_bucket(&bucket, session_context.credentials()).await {
|
||||
match self
|
||||
.storage
|
||||
.head_bucket(
|
||||
&bucket,
|
||||
&session_context.principal.user_identity.credentials.access_key,
|
||||
&session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => {
|
||||
error!(
|
||||
|
||||
@@ -137,7 +137,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
// on success. Size and mtime are not returned by HeadBucket.
|
||||
None => {
|
||||
self.authorize(&S3Action::HeadBucket, &bucket, None).await?;
|
||||
self.run_backend("head_bucket", self.storage.head_bucket(&bucket, self.credentials()))
|
||||
self.run_backend("head_bucket", self.storage.head_bucket(&bucket, self.access_key(), self.secret_key()))
|
||||
.await?;
|
||||
Ok(s3_attrs_to_sftp(0, None, true))
|
||||
}
|
||||
@@ -154,7 +154,11 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
Some(object_key) => {
|
||||
self.authorize(&S3Action::HeadObject, &bucket, Some(&object_key)).await?;
|
||||
match self
|
||||
.run_backend_with_err("head_object", self.storage.head_object(&bucket, &object_key, self.credentials()))
|
||||
.run_backend_with_err(
|
||||
"head_object",
|
||||
self.storage
|
||||
.head_object(&bucket, &object_key, self.access_key(), self.secret_key()),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
Ok(out) => {
|
||||
@@ -179,7 +183,10 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
.build()
|
||||
.map_err(|e| s3_error_to_sftp("build_list_objects", e))?;
|
||||
let out = self
|
||||
.run_backend("list_objects_v2", self.storage.list_objects_v2(input, self.credentials()))
|
||||
.run_backend(
|
||||
"list_objects_v2",
|
||||
self.storage.list_objects_v2(input, self.access_key(), self.secret_key()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let has_contents = out.contents.map(|c| !c.is_empty()).unwrap_or(false);
|
||||
|
||||
@@ -102,7 +102,10 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
let input = builder.build().map_err(|e| s3_error_to_sftp("build_list_objects", e))?;
|
||||
|
||||
let out = self
|
||||
.run_backend("list_objects_v2", self.storage.list_objects_v2(input, self.credentials()))
|
||||
.run_backend(
|
||||
"list_objects_v2",
|
||||
self.storage.list_objects_v2(input, self.access_key(), self.secret_key()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut entries = Vec::new();
|
||||
@@ -193,7 +196,10 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
// Issue list_objects_v2. On Err the destructive caller never
|
||||
// runs because validate_directory_empty returns the Err.
|
||||
let out = self
|
||||
.run_backend("list_objects_v2", self.storage.list_objects_v2(input, self.credentials()))
|
||||
.run_backend(
|
||||
"list_objects_v2",
|
||||
self.storage.list_objects_v2(input, self.access_key(), self.secret_key()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Count content entries that are not the directory's own marker.
|
||||
@@ -228,7 +234,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
self.authorize(&S3Action::ListBuckets, "", None).await?;
|
||||
|
||||
let out = self
|
||||
.run_backend("list_buckets", self.storage.list_buckets(self.credentials()))
|
||||
.run_backend("list_buckets", self.storage.list_buckets(self.access_key(), self.secret_key()))
|
||||
.await?;
|
||||
|
||||
let mut entries = Vec::new();
|
||||
@@ -274,7 +280,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
/// MKDIR for a bucket-level path: authorise and issue CreateBucket.
|
||||
pub(super) async fn mkdir_bucket(&self, bucket: &str) -> Result<(), SftpError> {
|
||||
self.authorize(&S3Action::CreateBucket, bucket, None).await?;
|
||||
self.run_backend("create_bucket", self.storage.create_bucket(bucket, self.credentials()))
|
||||
self.run_backend("create_bucket", self.storage.create_bucket(bucket, self.access_key(), self.secret_key()))
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -296,7 +302,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
.body(Some(streaming))
|
||||
.build()
|
||||
.map_err(|e| s3_error_to_sftp("build_put_object", e))?;
|
||||
self.run_backend("put_object", self.storage.put_object(input, self.credentials()))
|
||||
self.run_backend("put_object", self.storage.put_object(input, self.access_key(), self.secret_key()))
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -306,7 +312,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
pub(super) async fn rmdir_bucket(&self, bucket: &str) -> Result<(), SftpError> {
|
||||
self.validate_directory_empty(bucket, "").await?;
|
||||
self.authorize(&S3Action::DeleteBucket, bucket, None).await?;
|
||||
self.run_backend("delete_bucket", self.storage.delete_bucket(bucket, self.credentials()))
|
||||
self.run_backend("delete_bucket", self.storage.delete_bucket(bucket, self.access_key(), self.secret_key()))
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -320,8 +326,12 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
|
||||
let marker_key = path::encode_dir_object(&prefix);
|
||||
self.authorize(&S3Action::DeleteObject, bucket, Some(&marker_key)).await?;
|
||||
self.run_backend("delete_object", self.storage.delete_object(bucket, &marker_key, self.credentials()))
|
||||
.await?;
|
||||
self.run_backend(
|
||||
"delete_object",
|
||||
self.storage
|
||||
.delete_object(bucket, &marker_key, self.access_key(), self.secret_key()),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -388,7 +398,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
if prefix.is_empty() { None } else { Some(prefix.as_str()) },
|
||||
)
|
||||
.await?;
|
||||
self.run_backend("head_bucket", self.storage.head_bucket(&bucket, self.credentials()))
|
||||
self.run_backend("head_bucket", self.storage.head_bucket(&bucket, self.access_key(), self.secret_key()))
|
||||
.await?;
|
||||
DirCursor::Listing {
|
||||
bucket,
|
||||
|
||||
@@ -34,7 +34,6 @@ use crate::common::client::s3::StorageBackend;
|
||||
use crate::common::gateway::{AuthorizationError, S3Action, authorize_operation};
|
||||
use crate::common::session::SessionContext;
|
||||
use russh_sftp::protocol::{Attrs, Data, File, FileAttributes, Handle, Name, OpenFlags, Packet, Status, StatusCode, Version};
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_utils::MaskedAccessKey;
|
||||
use s3s::dto::{AbortMultipartUploadInput, CopyObjectInput, CopySource};
|
||||
use std::collections::HashMap;
|
||||
@@ -166,14 +165,16 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
super::read_cache::ReadCache::new(Arc::clone(&self.read_cache_in_use))
|
||||
}
|
||||
|
||||
/// Borrow the authenticated principal's S3 access key for diagnostics.
|
||||
/// Borrow the authenticated principal's S3 access key. Each StorageBackend
|
||||
/// call needs this alongside the secret key for signing.
|
||||
pub(super) fn access_key(&self) -> &str {
|
||||
&self.credentials().access_key
|
||||
&self.session_context.principal.user_identity.credentials.access_key
|
||||
}
|
||||
|
||||
/// Borrow the authenticated principal credentials for backend calls.
|
||||
pub(super) fn credentials(&self) -> &Credentials {
|
||||
self.session_context.credentials()
|
||||
/// Borrow the authenticated principal's S3 secret key. Used together with
|
||||
/// access_key for signing every backend call.
|
||||
pub(super) fn secret_key(&self) -> &str {
|
||||
&self.session_context.principal.user_identity.credentials.secret_key
|
||||
}
|
||||
|
||||
/// Returns Err(PermissionDenied) when the driver is read-only,
|
||||
@@ -786,8 +787,12 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
|
||||
self.authorize(&S3Action::DeleteObject, &bucket, Some(&object_key)).await?;
|
||||
|
||||
self.run_backend("delete_object", self.storage.delete_object(&bucket, &object_key, self.credentials()))
|
||||
.await?;
|
||||
self.run_backend(
|
||||
"delete_object",
|
||||
self.storage
|
||||
.delete_object(&bucket, &object_key, self.access_key(), self.secret_key()),
|
||||
)
|
||||
.await?;
|
||||
Ok(ok_status(id))
|
||||
}
|
||||
|
||||
@@ -893,7 +898,11 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
// single-shot vs multipart-copy branch below.
|
||||
self.authorize(&S3Action::HeadObject, &src_bucket, Some(&src_object)).await?;
|
||||
let head = self
|
||||
.run_backend("head_object", self.storage.head_object(&src_bucket, &src_object, self.credentials()))
|
||||
.run_backend(
|
||||
"head_object",
|
||||
self.storage
|
||||
.head_object(&src_bucket, &src_object, self.access_key(), self.secret_key()),
|
||||
)
|
||||
.await?;
|
||||
let content_length = head.content_length.unwrap_or(0).max(0) as u64;
|
||||
|
||||
@@ -911,7 +920,7 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
.key(dst_object.clone())
|
||||
.build()
|
||||
.map_err(|e| s3_error_to_sftp("build_copy_object", e))?;
|
||||
self.run_backend("copy_object", self.storage.copy_object(input, self.credentials()))
|
||||
self.run_backend("copy_object", self.storage.copy_object(input, self.access_key(), self.secret_key()))
|
||||
.await?;
|
||||
} else {
|
||||
self.multipart_copy(&src_bucket, &src_object, &dst_bucket, &dst_object, content_length)
|
||||
@@ -923,8 +932,12 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
// delete separately.
|
||||
self.authorize(&S3Action::DeleteObject, &src_bucket, Some(&src_object))
|
||||
.await?;
|
||||
self.run_backend("delete_object", self.storage.delete_object(&src_bucket, &src_object, self.credentials()))
|
||||
.await?;
|
||||
self.run_backend(
|
||||
"delete_object",
|
||||
self.storage
|
||||
.delete_object(&src_bucket, &src_object, self.access_key(), self.secret_key()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(ok_status(id))
|
||||
}
|
||||
@@ -1016,12 +1029,14 @@ impl<S: StorageBackend + Send + Sync + 'static> russh_sftp::server::Handler for
|
||||
impl<S: StorageBackend + Send + Sync + 'static> Drop for SftpDriver<S> {
|
||||
fn drop(&mut self) {
|
||||
// Snapshot credentials, peer IP, and the per-call backend
|
||||
// timeout before draining the handle table. Borrowing
|
||||
// self.session_context inside the loop would conflict with the
|
||||
// mutable borrow of self.handles. The timeout is copied into each
|
||||
// spawned abort task so the deadline applies uniformly to inline
|
||||
// calls and Drop-time aborts.
|
||||
let credentials = self.session_context.credentials().clone();
|
||||
// timeout before draining the handle table. self.access_key()
|
||||
// and self.secret_key() borrow self.session_context immutably,
|
||||
// which conflicts with the mutable borrow of self.handles
|
||||
// inside the loop. The timeout is copied into each spawned
|
||||
// abort task so the deadline applies uniformly to inline calls
|
||||
// and Drop-time aborts.
|
||||
let access_key = self.session_context.principal.user_identity.credentials.access_key.clone();
|
||||
let secret_key = self.session_context.principal.user_identity.credentials.secret_key.clone();
|
||||
let peer = self.session_context.source_ip;
|
||||
let backend_op_timeout_secs = self.backend_op_timeout_secs;
|
||||
|
||||
@@ -1041,7 +1056,7 @@ impl<S: StorageBackend + Send + Sync + 'static> Drop for SftpDriver<S> {
|
||||
key = %key,
|
||||
upload_id = %upload_id,
|
||||
peer = %peer,
|
||||
access_key = %MaskedAccessKey(&credentials.access_key),
|
||||
access_key = %access_key,
|
||||
"skipped abort of orphaned multipart upload on session drop, principal lacks s3:AbortMultipartUpload, bucket lifecycle rules must reclaim parts",
|
||||
);
|
||||
}
|
||||
@@ -1050,7 +1065,8 @@ impl<S: StorageBackend + Send + Sync + 'static> Drop for SftpDriver<S> {
|
||||
};
|
||||
|
||||
let storage = Arc::clone(&self.storage);
|
||||
let credentials = credentials.clone();
|
||||
let access_key = access_key.clone();
|
||||
let secret_key = secret_key.clone();
|
||||
let upload_id = upload_id_owned;
|
||||
|
||||
// Cap the global abort fan-out so a burst of session
|
||||
@@ -1106,7 +1122,7 @@ impl<S: StorageBackend + Send + Sync + 'static> Drop for SftpDriver<S> {
|
||||
};
|
||||
match tokio::time::timeout(
|
||||
std::time::Duration::from_secs(backend_op_timeout_secs),
|
||||
storage.abort_multipart_upload(input, &credentials),
|
||||
storage.abort_multipart_upload(input, &access_key, &secret_key),
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -46,7 +46,11 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
// the body. These are cached on the handle so READ can detect EOF
|
||||
// and FSTAT can answer without another backend call.
|
||||
let head = self
|
||||
.run_backend("head_object", self.storage.head_object(&bucket, &object_key, self.credentials()))
|
||||
.run_backend(
|
||||
"head_object",
|
||||
self.storage
|
||||
.head_object(&bucket, &object_key, self.access_key(), self.secret_key()),
|
||||
)
|
||||
.await?;
|
||||
let size = head.content_length.unwrap_or(0).max(0) as u64;
|
||||
let mtime = timestamp_to_mtime(head.last_modified);
|
||||
@@ -162,7 +166,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
.run_backend(
|
||||
"get_object_range",
|
||||
self.storage
|
||||
.get_object_range(bucket, key, self.credentials(), offset, fetch_len),
|
||||
.get_object_range(bucket, key, self.access_key(), self.secret_key(), offset, fetch_len),
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -293,7 +293,11 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
// not-found error means the key is free. Any other error is
|
||||
// propagated rather than misinterpreted as "does not exist".
|
||||
match self
|
||||
.run_backend_with_err("head_object", self.storage.head_object(&bucket, &object_key, self.credentials()))
|
||||
.run_backend_with_err(
|
||||
"head_object",
|
||||
self.storage
|
||||
.head_object(&bucket, &object_key, self.access_key(), self.secret_key()),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
Ok(_) => return Err(SftpError::code(StatusCode::Failure)),
|
||||
@@ -381,7 +385,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
.map_err(|e| s3_error_to_sftp("build_put_object", e))?;
|
||||
|
||||
let outcome = self
|
||||
.run_backend_with_err("put_object", self.storage.put_object(input, self.credentials()))
|
||||
.run_backend_with_err("put_object", self.storage.put_object(input, self.access_key(), self.secret_key()))
|
||||
.await?;
|
||||
|
||||
let backend_err = match outcome {
|
||||
@@ -444,7 +448,7 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
.map_err(|e| s3_error_to_sftp("build_upload_part", e))?;
|
||||
|
||||
let out = self
|
||||
.run_backend("upload_part", self.storage.upload_part(input, self.credentials()))
|
||||
.run_backend("upload_part", self.storage.upload_part(input, self.access_key(), self.secret_key()))
|
||||
.await?;
|
||||
|
||||
let e_tag = out.e_tag.ok_or_else(|| {
|
||||
@@ -524,7 +528,11 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
.map_err(|e| s3_error_to_sftp("build_create_multipart_upload", e))?;
|
||||
|
||||
let out = self
|
||||
.run_backend("create_multipart_upload", self.storage.create_multipart_upload(input, self.credentials()))
|
||||
.run_backend(
|
||||
"create_multipart_upload",
|
||||
self.storage
|
||||
.create_multipart_upload(input, self.access_key(), self.secret_key()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let upload_id = out.upload_id.ok_or_else(|| {
|
||||
@@ -577,7 +585,8 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
let result = self
|
||||
.run_backend(
|
||||
"complete_multipart_upload",
|
||||
self.storage.complete_multipart_upload(input, self.credentials()),
|
||||
self.storage
|
||||
.complete_multipart_upload(input, self.access_key(), self.secret_key()),
|
||||
)
|
||||
.await;
|
||||
result?;
|
||||
@@ -843,8 +852,12 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
.build()
|
||||
.map_err(|e| s3_error_to_sftp("build_abort_multipart_upload", e))?;
|
||||
|
||||
self.run_backend("abort_multipart_upload", self.storage.abort_multipart_upload(input, self.credentials()))
|
||||
.await?;
|
||||
self.run_backend(
|
||||
"abort_multipart_upload",
|
||||
self.storage
|
||||
.abort_multipart_upload(input, self.access_key(), self.secret_key()),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1053,7 +1066,10 @@ impl<S: StorageBackend + Send + Sync + 'static> SftpDriver<S> {
|
||||
.map_err(|e| s3_error_to_sftp("build_upload_part_copy", e))?;
|
||||
|
||||
let out = self
|
||||
.run_backend("upload_part_copy", self.storage.upload_part_copy(input, self.credentials()))
|
||||
.run_backend(
|
||||
"upload_part_copy",
|
||||
self.storage.upload_part_copy(input, self.access_key(), self.secret_key()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let e_tag = out.copy_part_result.and_then(|r| r.e_tag).ok_or_else(|| {
|
||||
@@ -1109,7 +1125,6 @@ mod tests {
|
||||
use crate::common::dummy_storage::{AbortCall, DummyBackend, DummyError};
|
||||
use crate::common::gateway::with_test_auth_override;
|
||||
use russh_sftp::protocol::{FileAttributes, OpenFlags, StatusCode};
|
||||
use rustfs_credentials::Credentials;
|
||||
use s3s::dto::ETag;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -2309,10 +2324,9 @@ mod tests {
|
||||
let backend = Arc::new(DummyBackend::new());
|
||||
backend.queue_head_object_err(DummyError::AccessDenied("pinned".to_string()));
|
||||
let driver = build_driver(backend, TEST_PART_SIZE);
|
||||
let credentials = Credentials::default();
|
||||
|
||||
let result = driver
|
||||
.run_backend_with_err("head_object", driver.storage.head_object("b", "k", &credentials))
|
||||
.run_backend_with_err("head_object", driver.storage.head_object("b", "k", "ak", "sk"))
|
||||
.await;
|
||||
|
||||
match result {
|
||||
|
||||
@@ -4,78 +4,23 @@ Swift-compatible object storage API implementation for RustFS.
|
||||
|
||||
## Features
|
||||
|
||||
The lists below are bounded to what `router.rs` / `handler.rs` dispatch and
|
||||
what the test suite exercises. A module existing under `src/swift/` does not
|
||||
by itself mean the feature is reachable over HTTP.
|
||||
This implementation provides **Phase 1 Swift API support** (~25% of full Swift API):
|
||||
|
||||
### Wired through the router and handler
|
||||
- ✅ Container CRUD operations (create, list, delete, metadata)
|
||||
- ✅ Object CRUD with streaming downloads (upload, get, head, delete)
|
||||
- ✅ Keystone token authentication
|
||||
- ✅ Multi-tenant isolation with secure SHA256-based bucket prefixing
|
||||
- ✅ Server-side object copy (COPY method)
|
||||
- ✅ HTTP Range requests for partial downloads (206, 416 responses)
|
||||
- ✅ Custom metadata support (X-Object-Meta-*, X-Container-Meta-*)
|
||||
|
||||
- ✅ Account listing (`GET /v1/AUTH_{project}`, JSON) and additive account
|
||||
metadata updates (`POST`, `X-Account-Meta-*` / `X-Remove-Account-Meta-*`)
|
||||
- ✅ Container CRUD (create, list, head, update metadata, delete)
|
||||
- ✅ Object CRUD with streaming downloads, HTTP Range requests (206 / 416),
|
||||
and server-side copy via the `COPY` method
|
||||
- ✅ Keystone token authentication and multi-tenant isolation with
|
||||
SHA256-based bucket prefixing
|
||||
- ✅ Custom metadata (`X-Object-Meta-*`, `X-Container-Meta-*`); container and
|
||||
account POSTs are additive, object POSTs replace the set
|
||||
- ✅ Container ACLs (`X-Container-Read` / `X-Container-Write`, set and remove on
|
||||
container POST, reported on HEAD). Enforcement is account-level plus
|
||||
referrer checks; per-user grants are not evaluated because credentials
|
||||
carry no user id
|
||||
- ✅ CORS: `OPTIONS` preflight on container and object routes, and response
|
||||
header injection driven by `X-Container-Meta-Access-Control-*`
|
||||
- ✅ TempURL (`temp_url_sig` / `temp_url_expires` on object GET, HEAD, PUT;
|
||||
key stored as account metadata; optional client-IP restriction)
|
||||
- ✅ FormPost (container POST with `multipart/form-data`, signed with the
|
||||
account TempURL key)
|
||||
- ✅ Large objects: Static Large Objects (`?multipart-manifest=put|get|delete`)
|
||||
and Dynamic Large Objects (`X-Object-Manifest`)
|
||||
- ✅ Bulk operations: `DELETE /v1/AUTH_{project}?bulk-delete` and
|
||||
`PUT /v1/AUTH_{project}/{container}?extract-archive=tar|tar.gz|tar.bz2`
|
||||
- ✅ Object versioning in the Swift `X-Versions-Location` style: the previous
|
||||
copy is archived on PUT / DELETE and restored on DELETE
|
||||
- ✅ Symlinks (`X-Symlink-Target` on PUT, resolved on GET / HEAD with loop and
|
||||
depth checks)
|
||||
- ✅ Container quotas (`X-Container-Meta-Quota-Bytes` / `-Quota-Count`),
|
||||
enforced on object PUT
|
||||
- ✅ Static website serving on object GET when `web-index` / `web-listings`
|
||||
container metadata is set
|
||||
- ✅ Object expiration headers: `X-Delete-At` / `X-Delete-After` are validated,
|
||||
stored, and returned on GET / HEAD
|
||||
|
||||
### Not yet wired, or partially wired
|
||||
|
||||
- ⏳ Account `HEAD` returns `501 Not Implemented`; no account-level usage
|
||||
statistics are exposed
|
||||
- ⏳ Automatic deletion of expired objects: `expiration_worker.rs` exists but
|
||||
the server never starts it, so objects past `X-Delete-At` are not removed
|
||||
- ⏳ Container sync (`sync.rs`): no `X-Container-Sync-*` header handling and no
|
||||
background worker; the module is unit-tested only
|
||||
- ⏳ `X-Copy-From` on object PUT (only the `COPY` method is supported)
|
||||
- ⏳ `X-History-Location` versioning mode
|
||||
- ⏳ Static website index / listing pages at the container root (only the
|
||||
object GET route consults static-web settings)
|
||||
- ⏳ XML / plain-text listing formats; the `format=` query parameter is
|
||||
ignored and listings are always JSON
|
||||
|
||||
### Test coverage
|
||||
|
||||
- Unit tests live next to each module (`acl.rs`, `bulk.rs`, `cors.rs`,
|
||||
`dlo.rs`, `slo.rs`, `tempurl.rs`, `formpost.rs`, `staticweb.rs`,
|
||||
`symlink.rs`, `quota.rs`, `expiration.rs`, `versioning.rs`, `router.rs`,
|
||||
`handler.rs`, and others) and run in the CI `swift` feature lane
|
||||
- `crates/protocols/tests/swift_metadata_persistence.rs` runs account,
|
||||
container, ACL, TempURL-key, and versioning metadata writes against a real
|
||||
ECStore and reloads them from disk
|
||||
- `crates/protocols/tests/swift_versioning_integration.rs`,
|
||||
`swift_listing_symlink_tests.rs`, `swift_simple_integration.rs`, and
|
||||
`swift_phase4_integration.rs` cover version naming, listing parameters,
|
||||
symlink parsing, and module-level helpers without a server
|
||||
- `rustfs/tests/swift_container_integration_test.rs` and
|
||||
`swift_object_integration_test.rs` exercise the HTTP surface end to end but
|
||||
are `#[ignore]` and need a running server (`TEST_RUSTFS_SERVER`); they are
|
||||
not part of CI
|
||||
**Not yet implemented:**
|
||||
- ⏳ Account-level operations (statistics, metadata)
|
||||
- ⏳ Large object support (multi-part uploads >5GB)
|
||||
- ⏳ Object versioning
|
||||
- ⏳ Container ACLs and CORS
|
||||
- ⏳ Temporary URLs (TempURL)
|
||||
- ⏳ XML/plain-text response formats (JSON only)
|
||||
|
||||
## Enable Feature
|
||||
|
||||
@@ -97,52 +42,38 @@ cargo build --features full
|
||||
|
||||
## Configuration
|
||||
|
||||
Swift API uses Keystone for authentication. The variables below are read by
|
||||
`crates/keystone/src/config.rs`; that file is the authoritative list.
|
||||
Swift API uses Keystone for authentication. Configure the following environment variables:
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `RUSTFS_KEYSTONE_ENABLE` | Set to `true` to enable Keystone authentication (default `false`; nothing else is read while disabled) |
|
||||
| `RUSTFS_KEYSTONE_AUTH_URL` | Keystone authentication endpoint URL (required once enabled) |
|
||||
| `RUSTFS_KEYSTONE_VERSION` | Keystone API version, `v3` or `v2.0` (default `v3`) |
|
||||
| `RUSTFS_KEYSTONE_ADMIN_USER` | Admin username (optional) |
|
||||
| `RUSTFS_KEYSTONE_ADMIN_PASSWORD` | Admin password (optional) |
|
||||
| `RUSTFS_KEYSTONE_ADMIN_PROJECT` | Admin project name (optional) |
|
||||
| `RUSTFS_KEYSTONE_ADMIN_DOMAIN` | Admin domain name (optional) |
|
||||
| `RUSTFS_KEYSTONE_VERIFY_SSL` | Verify the Keystone TLS certificate (default `true`) |
|
||||
| `RUSTFS_KEYSTONE_ENABLE_CACHE` / `RUSTFS_KEYSTONE_CACHE_SIZE` / `RUSTFS_KEYSTONE_CACHE_TTL` | Token cache toggle, entry count, and TTL in seconds (defaults `true`, `10000`, `300`) |
|
||||
| `RUSTFS_KEYSTONE_TENANT_PREFIX` | Prefix bucket names with the tenant hash (default `true`) |
|
||||
| `RUSTFS_KEYSTONE_IMPLICIT_TENANTS` | Allow implicit tenant creation (default `true`) |
|
||||
| `RUSTFS_KEYSTONE_TIMEOUT` | Keystone request timeout in seconds (default `30`) |
|
||||
| `RUSTFS_KEYSTONE_URL` | Keystone authentication endpoint URL |
|
||||
| `RUSTFS_KEYSTONE_ADMIN_TENANT` | Admin tenant/project name |
|
||||
| `RUSTFS_KEYSTONE_ADMIN_USER` | Admin username |
|
||||
| `RUSTFS_KEYSTONE_ADMIN_PASSWORD` | Admin password |
|
||||
|
||||
## API Endpoints
|
||||
|
||||
Swift API endpoints follow the pattern: `/v1/AUTH_{project_id}/...`
|
||||
|
||||
### Account Operations
|
||||
- `GET /v1/AUTH_{project}` - List containers (JSON)
|
||||
- `HEAD /v1/AUTH_{project}` - Get account metadata (returns 501, not yet implemented)
|
||||
- `POST /v1/AUTH_{project}` - Update account metadata and TempURL key
|
||||
- `DELETE /v1/AUTH_{project}?bulk-delete` - Bulk delete
|
||||
- `GET /v1/AUTH_{project}` - List containers
|
||||
- `HEAD /v1/AUTH_{project}` - Get account metadata (not yet implemented)
|
||||
- `POST /v1/AUTH_{project}` - Update account metadata (not yet implemented)
|
||||
|
||||
### Container Operations
|
||||
- `PUT /v1/AUTH_{project}/{container}` - Create container (`?extract-archive=` for bulk upload)
|
||||
- `GET /v1/AUTH_{project}/{container}` - List objects (JSON; `limit`, `marker`, `end_marker`, `prefix`, `delimiter`)
|
||||
- `PUT /v1/AUTH_{project}/{container}` - Create container
|
||||
- `GET /v1/AUTH_{project}/{container}` - List objects
|
||||
- `HEAD /v1/AUTH_{project}/{container}` - Get container metadata
|
||||
- `POST /v1/AUTH_{project}/{container}` - Update container metadata, ACLs, versioning location; FormPost when `multipart/form-data`
|
||||
- `POST /v1/AUTH_{project}/{container}` - Update container metadata
|
||||
- `DELETE /v1/AUTH_{project}/{container}` - Delete container
|
||||
- `OPTIONS /v1/AUTH_{project}/{container}` - CORS preflight
|
||||
|
||||
### Object Operations
|
||||
- `PUT /v1/AUTH_{project}/{container}/{object}` - Upload object (SLO manifest with `?multipart-manifest=put`, DLO with `X-Object-Manifest`, symlink with `X-Symlink-Target`)
|
||||
- `GET /v1/AUTH_{project}/{container}/{object}` - Download object (Range, SLO/DLO assembly, symlink resolution, `?multipart-manifest=get`)
|
||||
- `PUT /v1/AUTH_{project}/{container}/{object}` - Upload object
|
||||
- `GET /v1/AUTH_{project}/{container}/{object}` - Download object
|
||||
- `HEAD /v1/AUTH_{project}/{container}/{object}` - Get object metadata
|
||||
- `POST /v1/AUTH_{project}/{container}/{object}` - Update object metadata
|
||||
- `DELETE /v1/AUTH_{project}/{container}/{object}` - Delete object (`?multipart-manifest=delete` removes SLO segments)
|
||||
- `DELETE /v1/AUTH_{project}/{container}/{object}` - Delete object
|
||||
- `COPY /v1/AUTH_{project}/{container}/{object}` - Server-side copy
|
||||
- `OPTIONS /v1/AUTH_{project}/{container}/{object}` - CORS preflight
|
||||
|
||||
Object GET, HEAD, and PUT also accept TempURL query parameters without an auth token.
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -165,18 +96,11 @@ Handler (fallback)
|
||||
|
||||
### Key Components
|
||||
|
||||
- **handler.rs** - Main service implementing Tower's Service trait and method dispatch
|
||||
- **handler.rs** - Main service implementing Tower's Service trait
|
||||
- **router.rs** - URL routing and parsing for Swift paths
|
||||
- **container.rs** - Container operations with tenant isolation
|
||||
- **object.rs** - Object operations including copy and range requests
|
||||
- **account.rs** - Account validation, tenant access control, account metadata and TempURL key
|
||||
- **acl.rs**, **cors.rs** - Container ACL evaluation and CORS config
|
||||
- **slo.rs**, **dlo.rs** - Static and dynamic large objects
|
||||
- **tempurl.rs**, **formpost.rs** - Signed URL and form upload validation
|
||||
- **bulk.rs** - Bulk delete and archive extraction
|
||||
- **versioning.rs**, **symlink.rs**, **quota.rs**, **staticweb.rs**, **expiration.rs** - Per-feature helpers called from the handler
|
||||
- **expiration_worker.rs**, **sync.rs** - Background workers that are not started by the server (see above)
|
||||
- **metadata_update.rs** - Additive account/container metadata merge
|
||||
- **account.rs** - Account validation and tenant access control
|
||||
- **errors.rs** - Swift-specific error types
|
||||
- **types.rs** - Data structures for Swift API responses
|
||||
|
||||
@@ -197,15 +121,13 @@ This ensures:
|
||||
|
||||
## Documentation
|
||||
|
||||
There is no separate Swift reference document in the repository. Use these
|
||||
sources instead:
|
||||
See the `docs/` directory for detailed documentation:
|
||||
|
||||
- Module-level `//!` comments in each `crates/protocols/src/swift/*.rs` file
|
||||
describe the headers and metadata keys that feature reads
|
||||
- `crates/protocols/tests/swift_*.rs` and `rustfs/tests/swift_*_integration_test.rs`
|
||||
show the expected request and response shapes
|
||||
- `docs/testing/ci-gates.md` describes the CI lane that builds and tests with
|
||||
`--features swift`
|
||||
- `SWIFT_API.md` - Complete API reference
|
||||
- `TESTING_GUIDE.md` - Manual testing procedures
|
||||
- `COMPLETION_ANALYSIS.md` - Protocol coverage tracking
|
||||
- `COPY_IMPLEMENTATION.md` - Server-side copy documentation
|
||||
- `RANGE_REQUESTS.md` - Range request implementation details
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@ use dav_server::fs::{
|
||||
};
|
||||
use futures_util::{FutureExt, StreamExt, stream};
|
||||
use percent_encoding::percent_decode_str;
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_utils::MaskedAccessKey;
|
||||
use rustfs_utils::path;
|
||||
use s3s::S3ErrorCode;
|
||||
@@ -199,7 +198,15 @@ where
|
||||
let key = self.key.clone();
|
||||
|
||||
async move {
|
||||
match storage.head_object(&bucket, &key, session_context.credentials()).await {
|
||||
match storage
|
||||
.head_object(
|
||||
&bucket,
|
||||
&key,
|
||||
&session_context.principal.user_identity.credentials.access_key,
|
||||
&session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(output) => {
|
||||
let size = output.content_length.unwrap_or(0) as u64;
|
||||
let modified = output
|
||||
@@ -281,7 +288,14 @@ where
|
||||
async move {
|
||||
let start_pos = *position.read().await;
|
||||
match storage
|
||||
.get_object_range(&bucket, &key, session_context.credentials(), start_pos, count as u64)
|
||||
.get_object_range(
|
||||
&bucket,
|
||||
&key,
|
||||
&session_context.principal.user_identity.credentials.access_key,
|
||||
&session_context.principal.user_identity.credentials.secret_key,
|
||||
start_pos,
|
||||
count as u64,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(output) => {
|
||||
@@ -393,7 +407,14 @@ where
|
||||
.build()
|
||||
.map_err(|_| FsError::GeneralFailure)?;
|
||||
|
||||
match storage.put_object(put_input, session_context.credentials()).await {
|
||||
match storage
|
||||
.put_object(
|
||||
put_input,
|
||||
&session_context.principal.user_identity.credentials.access_key,
|
||||
&session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
event = EVENT_WEBDAV_OBJECT_WRITE_STATE,
|
||||
@@ -501,8 +522,11 @@ where
|
||||
self
|
||||
}
|
||||
|
||||
fn credentials(&self) -> &Credentials {
|
||||
self.session_context.credentials()
|
||||
fn credentials(&self) -> (&str, &str) {
|
||||
(
|
||||
&self.session_context.principal.user_identity.credentials.access_key,
|
||||
&self.session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
}
|
||||
|
||||
fn is_missing_head_object_error(error: &str) -> bool {
|
||||
@@ -514,7 +538,7 @@ where
|
||||
}
|
||||
|
||||
async fn prefix_has_entries(&self, bucket: &str, prefix: &str) -> FsResult<bool> {
|
||||
let credentials = self.credentials();
|
||||
let (access_key, secret_key) = self.credentials();
|
||||
let list_input = ListObjectsV2Input::builder()
|
||||
.bucket(bucket.to_string())
|
||||
.prefix(Some(prefix.to_string()))
|
||||
@@ -522,28 +546,32 @@ where
|
||||
.build()
|
||||
.map_err(|_| FsError::GeneralFailure)?;
|
||||
|
||||
let output = self.storage.list_objects_v2(list_input, credentials).await.map_err(|e| {
|
||||
error!(
|
||||
event = EVENT_WEBDAV_LIST_FAILED,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
|
||||
bucket = %bucket,
|
||||
prefix = %prefix,
|
||||
error = %e,
|
||||
"webdav list failed"
|
||||
);
|
||||
FsError::GeneralFailure
|
||||
})?;
|
||||
let output = self
|
||||
.storage
|
||||
.list_objects_v2(list_input, access_key, secret_key)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(
|
||||
event = EVENT_WEBDAV_LIST_FAILED,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
|
||||
bucket = %bucket,
|
||||
prefix = %prefix,
|
||||
error = %e,
|
||||
"webdav list failed"
|
||||
);
|
||||
FsError::GeneralFailure
|
||||
})?;
|
||||
|
||||
Ok(output.contents.map(|c| !c.is_empty()).unwrap_or(false)
|
||||
|| output.common_prefixes.map(|c| !c.is_empty()).unwrap_or(false))
|
||||
}
|
||||
|
||||
async fn copy_object_streaming(&self, src_bucket: &str, src_key: &str, dst_bucket: &str, dst_key: &str) -> FsResult<()> {
|
||||
let credentials = self.credentials();
|
||||
let (access_key, secret_key) = self.credentials();
|
||||
let get_output = self
|
||||
.storage
|
||||
.get_object(src_bucket, src_key, credentials, None)
|
||||
.get_object(src_bucket, src_key, access_key, secret_key, None)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(
|
||||
@@ -597,21 +625,24 @@ where
|
||||
|
||||
let put_input = put_builder.build().map_err(|_| FsError::GeneralFailure)?;
|
||||
|
||||
self.storage.put_object(put_input, credentials).await.map_err(|e| {
|
||||
error!(
|
||||
event = EVENT_WEBDAV_COPY_FAILED,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
|
||||
state = "destination_write_failed",
|
||||
src_bucket = %src_bucket,
|
||||
src_object = %src_key,
|
||||
dst_bucket = %dst_bucket,
|
||||
dst_object = %dst_key,
|
||||
error = %e,
|
||||
"webdav copy failed"
|
||||
);
|
||||
FsError::GeneralFailure
|
||||
})?;
|
||||
self.storage
|
||||
.put_object(put_input, access_key, secret_key)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(
|
||||
event = EVENT_WEBDAV_COPY_FAILED,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
|
||||
state = "destination_write_failed",
|
||||
src_bucket = %src_bucket,
|
||||
src_object = %src_key,
|
||||
dst_bucket = %dst_bucket,
|
||||
dst_object = %dst_key,
|
||||
error = %e,
|
||||
"webdav copy failed"
|
||||
);
|
||||
FsError::GeneralFailure
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -622,7 +653,7 @@ where
|
||||
dst_bucket: &str,
|
||||
rename_pairs: &[(String, String)],
|
||||
) -> FsResult<()> {
|
||||
let credentials = self.credentials();
|
||||
let (access_key, secret_key) = self.credentials();
|
||||
|
||||
for (src_obj_key, dst_obj_key) in rename_pairs {
|
||||
self.copy_object_streaming(src_bucket, src_obj_key, dst_bucket, dst_obj_key)
|
||||
@@ -631,7 +662,7 @@ where
|
||||
|
||||
for (src_obj_key, _) in rename_pairs {
|
||||
self.storage
|
||||
.delete_object(src_bucket, src_obj_key, credentials)
|
||||
.delete_object(src_bucket, src_obj_key, access_key, secret_key)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(
|
||||
@@ -652,7 +683,7 @@ where
|
||||
}
|
||||
|
||||
async fn probe_head_object(&self, bucket: &str, key: &str) -> FsResult<HeadObjectProbe> {
|
||||
let credentials = self.credentials();
|
||||
let (access_key, secret_key) = self.credentials();
|
||||
|
||||
if authorize_operation(&self.session_context, &S3Action::HeadObject, bucket, Some(key))
|
||||
.await
|
||||
@@ -661,7 +692,7 @@ where
|
||||
return Ok(HeadObjectProbe::Forbidden);
|
||||
}
|
||||
|
||||
match self.storage.head_object(bucket, key, credentials).await {
|
||||
match self.storage.head_object(bucket, key, access_key, secret_key).await {
|
||||
Ok(output) => Ok(HeadObjectProbe::Found(Box::new(output))),
|
||||
Err(e) => {
|
||||
let err_msg = e.to_string();
|
||||
@@ -785,8 +816,8 @@ where
|
||||
async fn list_buckets(&self) -> FsResult<Vec<WebDavDirEntry>> {
|
||||
match authorize_operation(&self.session_context, &S3Action::ListBuckets, "", None).await {
|
||||
Ok(()) => {
|
||||
let credentials = self.credentials();
|
||||
return match self.storage.list_buckets(credentials).await {
|
||||
let (access_key, secret_key) = self.credentials();
|
||||
return match self.storage.list_buckets(access_key, secret_key).await {
|
||||
Ok(output) => Ok(Self::bucket_entries(output)),
|
||||
Err(error) => {
|
||||
error!(
|
||||
@@ -794,7 +825,7 @@ where
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
|
||||
error = %error,
|
||||
access_key = %MaskedAccessKey(credentials.access_key.as_str()),
|
||||
access_key = %MaskedAccessKey(access_key),
|
||||
"webdav bucket list failed"
|
||||
);
|
||||
Err(FsError::GeneralFailure)
|
||||
@@ -877,7 +908,15 @@ where
|
||||
.build()
|
||||
.map_err(|_| FsError::GeneralFailure)?;
|
||||
|
||||
match self.storage.list_objects_v2(list_input, self.credentials()).await {
|
||||
match self
|
||||
.storage
|
||||
.list_objects_v2(
|
||||
list_input,
|
||||
&self.session_context.principal.user_identity.credentials.access_key,
|
||||
&self.session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(output) => {
|
||||
let mut entries = Vec::new();
|
||||
|
||||
@@ -1015,7 +1054,15 @@ where
|
||||
|
||||
let list_input = list_input.build().map_err(|_| FsError::GeneralFailure)?;
|
||||
|
||||
if let Ok(output) = self.storage.list_objects_v2(list_input, self.credentials()).await {
|
||||
if let Ok(output) = self
|
||||
.storage
|
||||
.list_objects_v2(
|
||||
list_input,
|
||||
&self.session_context.principal.user_identity.credentials.access_key,
|
||||
&self.session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
// Delete all objects in this page
|
||||
if let Some(objects) = output.contents {
|
||||
for obj in objects {
|
||||
@@ -1024,7 +1071,15 @@ where
|
||||
.await
|
||||
.map_err(|_| FsError::Forbidden)?;
|
||||
|
||||
let _ = self.storage.delete_object(bucket, &obj_key, self.credentials()).await;
|
||||
let _ = self
|
||||
.storage
|
||||
.delete_object(
|
||||
bucket,
|
||||
&obj_key,
|
||||
&self.session_context.principal.user_identity.credentials.access_key,
|
||||
&self.session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1040,7 +1095,15 @@ where
|
||||
}
|
||||
|
||||
// Then delete the bucket
|
||||
match self.storage.delete_bucket(bucket, self.credentials()).await {
|
||||
match self
|
||||
.storage
|
||||
.delete_bucket(
|
||||
bucket,
|
||||
&self.session_context.principal.user_identity.credentials.access_key,
|
||||
&self.session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) if e.to_string().contains("NoSuchBucket") => Ok(()),
|
||||
Err(e) => {
|
||||
@@ -1187,7 +1250,15 @@ where
|
||||
.await
|
||||
.map_err(|_| FsError::Forbidden)?;
|
||||
|
||||
match self.storage.head_bucket(&bucket, self.credentials()).await {
|
||||
match self
|
||||
.storage
|
||||
.head_bucket(
|
||||
&bucket,
|
||||
&self.session_context.principal.user_identity.credentials.access_key,
|
||||
&self.session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(Box::new(WebDavMetaData {
|
||||
size: 0,
|
||||
modified: SystemTime::now(),
|
||||
@@ -1247,7 +1318,15 @@ where
|
||||
.build()
|
||||
.map_err(|_| FsError::GeneralFailure)?;
|
||||
|
||||
match self.storage.put_object(put_input, self.credentials()).await {
|
||||
match self
|
||||
.storage
|
||||
.put_object(
|
||||
put_input,
|
||||
&self.session_context.principal.user_identity.credentials.access_key,
|
||||
&self.session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
event = EVENT_WEBDAV_DIRECTORY_STATE,
|
||||
@@ -1281,7 +1360,15 @@ where
|
||||
.await
|
||||
.map_err(|_| FsError::Forbidden)?;
|
||||
|
||||
match self.storage.create_bucket(&bucket, self.credentials()).await {
|
||||
match self
|
||||
.storage
|
||||
.create_bucket(
|
||||
&bucket,
|
||||
&self.session_context.principal.user_identity.credentials.access_key,
|
||||
&self.session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
event = EVENT_WEBDAV_DIRECTORY_STATE,
|
||||
@@ -1351,7 +1438,15 @@ where
|
||||
|
||||
let list_input = list_input.build().map_err(|_| FsError::GeneralFailure)?;
|
||||
|
||||
if let Ok(output) = self.storage.list_objects_v2(list_input, self.credentials()).await {
|
||||
if let Ok(output) = self
|
||||
.storage
|
||||
.list_objects_v2(
|
||||
list_input,
|
||||
&self.session_context.principal.user_identity.credentials.access_key,
|
||||
&self.session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
if let Some(objects) = output.contents {
|
||||
for obj in objects {
|
||||
if let Some(obj_key) = obj.key {
|
||||
@@ -1359,7 +1454,15 @@ where
|
||||
.await
|
||||
.map_err(|_| FsError::Forbidden)?;
|
||||
|
||||
let _ = self.storage.delete_object(&bucket, &obj_key, self.credentials()).await;
|
||||
let _ = self
|
||||
.storage
|
||||
.delete_object(
|
||||
&bucket,
|
||||
&obj_key,
|
||||
&self.session_context.principal.user_identity.credentials.access_key,
|
||||
&self.session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1376,7 +1479,12 @@ where
|
||||
// Also delete the directory marker itself
|
||||
let _ = self
|
||||
.storage
|
||||
.delete_object(&bucket, &prefix_with_slash, self.credentials())
|
||||
.delete_object(
|
||||
&bucket,
|
||||
&prefix_with_slash,
|
||||
&self.session_context.principal.user_identity.credentials.access_key,
|
||||
&self.session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await;
|
||||
|
||||
return Ok(());
|
||||
@@ -1407,7 +1515,16 @@ where
|
||||
.await
|
||||
.map_err(|_| FsError::Forbidden)?;
|
||||
|
||||
match self.storage.delete_object(&bucket, &key, self.credentials()).await {
|
||||
match self
|
||||
.storage
|
||||
.delete_object(
|
||||
&bucket,
|
||||
&key,
|
||||
&self.session_context.principal.user_identity.credentials.access_key,
|
||||
&self.session_context.principal.user_identity.credentials.secret_key,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
event = EVENT_WEBDAV_OBJECT_DELETE_STATE,
|
||||
@@ -1449,7 +1566,7 @@ where
|
||||
|
||||
let src_key = src_key.ok_or(FsError::Forbidden)?;
|
||||
let dst_key = dst_key.ok_or(FsError::Forbidden)?;
|
||||
let credentials = self.credentials();
|
||||
let (access_key, secret_key) = self.credentials();
|
||||
let resolved_src = self.resolve_path(&src_bucket, &src_key).await?;
|
||||
let (src_prefix, include_src_marker) = match resolved_src {
|
||||
ResolvedPath::File(_) => {
|
||||
@@ -1467,7 +1584,7 @@ where
|
||||
.await?;
|
||||
|
||||
self.storage
|
||||
.delete_object(&src_bucket, &src_key, credentials)
|
||||
.delete_object(&src_bucket, &src_key, access_key, secret_key)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(
|
||||
@@ -1539,21 +1656,25 @@ where
|
||||
}
|
||||
|
||||
let list_input = list_builder.build().map_err(|_| FsError::GeneralFailure)?;
|
||||
let output = self.storage.list_objects_v2(list_input, credentials).await.map_err(|e| {
|
||||
error!(
|
||||
event = EVENT_WEBDAV_RENAME_STATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
|
||||
state = "directory_list_failed",
|
||||
src_bucket = %src_bucket,
|
||||
src_prefix = %src_prefix,
|
||||
dst_bucket = %dst_bucket,
|
||||
dst_prefix = %dst_prefix,
|
||||
error = %e,
|
||||
"WebDAV rename directory listing failed"
|
||||
);
|
||||
FsError::GeneralFailure
|
||||
})?;
|
||||
let output = self
|
||||
.storage
|
||||
.list_objects_v2(list_input, access_key, secret_key)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(
|
||||
event = EVENT_WEBDAV_RENAME_STATE,
|
||||
component = LOG_COMPONENT_PROTOCOLS,
|
||||
subsystem = LOG_SUBSYSTEM_WEBDAV_DRIVER,
|
||||
state = "directory_list_failed",
|
||||
src_bucket = %src_bucket,
|
||||
src_prefix = %src_prefix,
|
||||
dst_bucket = %dst_bucket,
|
||||
dst_prefix = %dst_prefix,
|
||||
error = %e,
|
||||
"WebDAV rename directory listing failed"
|
||||
);
|
||||
FsError::GeneralFailure
|
||||
})?;
|
||||
|
||||
let mut page_pairs: Vec<(String, String)> = Vec::new();
|
||||
if let Some(objects) = output.contents {
|
||||
@@ -1664,7 +1785,8 @@ mod tests {
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
_start_pos: Option<u64>,
|
||||
) -> Result<GetObjectOutput, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
@@ -1674,14 +1796,20 @@ mod tests {
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
_start_pos: u64,
|
||||
_length: u64,
|
||||
) -> Result<GetObjectOutput, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
}
|
||||
|
||||
async fn put_object(&self, _input: PutObjectInput, _credentials: &Credentials) -> Result<PutObjectOutput, Self::Error> {
|
||||
async fn put_object(
|
||||
&self,
|
||||
_input: PutObjectInput,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<PutObjectOutput, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
}
|
||||
|
||||
@@ -1689,7 +1817,8 @@ mod tests {
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<DeleteObjectOutput, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
}
|
||||
@@ -1698,39 +1827,57 @@ mod tests {
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<HeadObjectOutput, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
}
|
||||
|
||||
async fn head_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<HeadBucketOutput, Self::Error> {
|
||||
async fn head_bucket(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<HeadBucketOutput, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
}
|
||||
|
||||
async fn list_objects_v2(
|
||||
&self,
|
||||
_input: ListObjectsV2Input,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<ListObjectsV2Output, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
}
|
||||
|
||||
async fn list_buckets(&self, _credentials: &Credentials) -> Result<ListBucketsOutput, Self::Error> {
|
||||
async fn list_buckets(&self, _access_key: &str, _secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
}
|
||||
|
||||
async fn create_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<CreateBucketOutput, Self::Error> {
|
||||
async fn create_bucket(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<CreateBucketOutput, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
}
|
||||
|
||||
async fn delete_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<DeleteBucketOutput, Self::Error> {
|
||||
async fn delete_bucket(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<DeleteBucketOutput, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
}
|
||||
|
||||
async fn copy_object(
|
||||
&self,
|
||||
_input: CopyObjectInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<CopyObjectOutput, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
}
|
||||
@@ -1738,7 +1885,8 @@ mod tests {
|
||||
async fn create_multipart_upload(
|
||||
&self,
|
||||
_input: CreateMultipartUploadInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<CreateMultipartUploadOutput, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
}
|
||||
@@ -1746,7 +1894,8 @@ mod tests {
|
||||
async fn upload_part(
|
||||
&self,
|
||||
_input: UploadPartInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<UploadPartOutput, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
}
|
||||
@@ -1754,7 +1903,8 @@ mod tests {
|
||||
async fn complete_multipart_upload(
|
||||
&self,
|
||||
_input: CompleteMultipartUploadInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<CompleteMultipartUploadOutput, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
}
|
||||
@@ -1762,7 +1912,8 @@ mod tests {
|
||||
async fn abort_multipart_upload(
|
||||
&self,
|
||||
_input: AbortMultipartUploadInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<AbortMultipartUploadOutput, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
}
|
||||
@@ -1770,7 +1921,8 @@ mod tests {
|
||||
async fn upload_part_copy(
|
||||
&self,
|
||||
_input: UploadPartCopyInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<UploadPartCopyOutput, Self::Error> {
|
||||
unreachable!("parse_path tests should not hit storage")
|
||||
}
|
||||
@@ -1947,7 +2099,8 @@ mod tests {
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
_start_pos: Option<u64>,
|
||||
) -> Result<GetObjectOutput, Self::Error> {
|
||||
let data = self
|
||||
@@ -1974,7 +2127,8 @@ mod tests {
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
_start_pos: u64,
|
||||
_length: u64,
|
||||
) -> Result<GetObjectOutput, Self::Error> {
|
||||
@@ -1984,7 +2138,8 @@ mod tests {
|
||||
async fn put_object(
|
||||
&self,
|
||||
mut input: PutObjectInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<PutObjectOutput, Self::Error> {
|
||||
let bucket = input.bucket.clone();
|
||||
let key = input.key.clone();
|
||||
@@ -2008,7 +2163,8 @@ mod tests {
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<DeleteObjectOutput, Self::Error> {
|
||||
let mut state = self.state.lock().expect("recording storage lock poisoned");
|
||||
state.delete_keys.push(key.to_string());
|
||||
@@ -2023,19 +2179,26 @@ mod tests {
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<HeadObjectOutput, Self::Error> {
|
||||
unreachable!("head_object is not used in rename regression tests")
|
||||
}
|
||||
|
||||
async fn head_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<HeadBucketOutput, Self::Error> {
|
||||
async fn head_bucket(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<HeadBucketOutput, Self::Error> {
|
||||
unreachable!("head_bucket is not used in rename regression tests")
|
||||
}
|
||||
|
||||
async fn list_objects_v2(
|
||||
&self,
|
||||
input: ListObjectsV2Input,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<ListObjectsV2Output, Self::Error> {
|
||||
let prefix = input.prefix.unwrap_or_default();
|
||||
let mut keys: Vec<String> = self
|
||||
@@ -2063,15 +2226,25 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
async fn list_buckets(&self, _credentials: &Credentials) -> Result<ListBucketsOutput, Self::Error> {
|
||||
async fn list_buckets(&self, _access_key: &str, _secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
|
||||
unreachable!("list_buckets is not used in rename regression tests")
|
||||
}
|
||||
|
||||
async fn create_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<CreateBucketOutput, Self::Error> {
|
||||
async fn create_bucket(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<CreateBucketOutput, Self::Error> {
|
||||
unreachable!("create_bucket is not used in rename regression tests")
|
||||
}
|
||||
|
||||
async fn delete_bucket(&self, bucket: &str, _credentials: &Credentials) -> Result<DeleteBucketOutput, Self::Error> {
|
||||
async fn delete_bucket(
|
||||
&self,
|
||||
bucket: &str,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<DeleteBucketOutput, Self::Error> {
|
||||
self.state
|
||||
.lock()
|
||||
.expect("recording storage lock poisoned")
|
||||
@@ -2083,7 +2256,8 @@ mod tests {
|
||||
async fn copy_object(
|
||||
&self,
|
||||
_input: CopyObjectInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<CopyObjectOutput, Self::Error> {
|
||||
unreachable!("copy_object is not used in rename regression tests")
|
||||
}
|
||||
@@ -2091,7 +2265,8 @@ mod tests {
|
||||
async fn create_multipart_upload(
|
||||
&self,
|
||||
_input: CreateMultipartUploadInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<CreateMultipartUploadOutput, Self::Error> {
|
||||
unreachable!("create_multipart_upload is not used in rename regression tests")
|
||||
}
|
||||
@@ -2099,7 +2274,8 @@ mod tests {
|
||||
async fn upload_part(
|
||||
&self,
|
||||
_input: UploadPartInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<UploadPartOutput, Self::Error> {
|
||||
unreachable!("upload_part is not used in rename regression tests")
|
||||
}
|
||||
@@ -2107,7 +2283,8 @@ mod tests {
|
||||
async fn complete_multipart_upload(
|
||||
&self,
|
||||
_input: CompleteMultipartUploadInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<CompleteMultipartUploadOutput, Self::Error> {
|
||||
unreachable!("complete_multipart_upload is not used in rename regression tests")
|
||||
}
|
||||
@@ -2115,7 +2292,8 @@ mod tests {
|
||||
async fn abort_multipart_upload(
|
||||
&self,
|
||||
_input: AbortMultipartUploadInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<AbortMultipartUploadOutput, Self::Error> {
|
||||
unreachable!("abort_multipart_upload is not used in rename regression tests")
|
||||
}
|
||||
@@ -2123,7 +2301,8 @@ mod tests {
|
||||
async fn upload_part_copy(
|
||||
&self,
|
||||
_input: UploadPartCopyInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<UploadPartCopyOutput, Self::Error> {
|
||||
unreachable!("upload_part_copy is not used in rename regression tests")
|
||||
}
|
||||
|
||||
@@ -687,7 +687,6 @@ mod tests {
|
||||
use futures_util::stream;
|
||||
use http_body_util::StreamBody;
|
||||
use hyper::body::Frame;
|
||||
use rustfs_credentials::Credentials;
|
||||
use s3s::dto::*;
|
||||
use std::fmt::{Debug, Formatter};
|
||||
use std::net::{Ipv4Addr, SocketAddr};
|
||||
@@ -716,7 +715,8 @@ mod tests {
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
_start_pos: Option<u64>,
|
||||
) -> Result<GetObjectOutput, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
@@ -726,14 +726,20 @@ mod tests {
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
_start_pos: u64,
|
||||
_length: u64,
|
||||
) -> Result<GetObjectOutput, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
}
|
||||
|
||||
async fn put_object(&self, _input: PutObjectInput, _credentials: &Credentials) -> Result<PutObjectOutput, Self::Error> {
|
||||
async fn put_object(
|
||||
&self,
|
||||
_input: PutObjectInput,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<PutObjectOutput, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
}
|
||||
|
||||
@@ -741,7 +747,8 @@ mod tests {
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<DeleteObjectOutput, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
}
|
||||
@@ -750,39 +757,57 @@ mod tests {
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_key: &str,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<HeadObjectOutput, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
}
|
||||
|
||||
async fn head_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<HeadBucketOutput, Self::Error> {
|
||||
async fn head_bucket(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<HeadBucketOutput, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
}
|
||||
|
||||
async fn list_objects_v2(
|
||||
&self,
|
||||
_input: ListObjectsV2Input,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<ListObjectsV2Output, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
}
|
||||
|
||||
async fn list_buckets(&self, _credentials: &Credentials) -> Result<ListBucketsOutput, Self::Error> {
|
||||
async fn list_buckets(&self, _access_key: &str, _secret_key: &str) -> Result<ListBucketsOutput, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
}
|
||||
|
||||
async fn create_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<CreateBucketOutput, Self::Error> {
|
||||
async fn create_bucket(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<CreateBucketOutput, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
}
|
||||
|
||||
async fn delete_bucket(&self, _bucket: &str, _credentials: &Credentials) -> Result<DeleteBucketOutput, Self::Error> {
|
||||
async fn delete_bucket(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<DeleteBucketOutput, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
}
|
||||
|
||||
async fn copy_object(
|
||||
&self,
|
||||
_input: CopyObjectInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<CopyObjectOutput, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
}
|
||||
@@ -790,7 +815,8 @@ mod tests {
|
||||
async fn create_multipart_upload(
|
||||
&self,
|
||||
_input: CreateMultipartUploadInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<CreateMultipartUploadOutput, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
}
|
||||
@@ -798,7 +824,8 @@ mod tests {
|
||||
async fn upload_part(
|
||||
&self,
|
||||
_input: UploadPartInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<UploadPartOutput, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
}
|
||||
@@ -806,7 +833,8 @@ mod tests {
|
||||
async fn complete_multipart_upload(
|
||||
&self,
|
||||
_input: CompleteMultipartUploadInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<CompleteMultipartUploadOutput, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
}
|
||||
@@ -814,7 +842,8 @@ mod tests {
|
||||
async fn abort_multipart_upload(
|
||||
&self,
|
||||
_input: AbortMultipartUploadInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<AbortMultipartUploadOutput, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
}
|
||||
@@ -822,7 +851,8 @@ mod tests {
|
||||
async fn upload_part_copy(
|
||||
&self,
|
||||
_input: UploadPartCopyInput,
|
||||
_credentials: &Credentials,
|
||||
_access_key: &str,
|
||||
_secret_key: &str,
|
||||
) -> Result<UploadPartCopyOutput, Self::Error> {
|
||||
unreachable!("connection tests should not hit storage")
|
||||
}
|
||||
|
||||
@@ -88,8 +88,6 @@ impl From<Priority> for HealChannelPriority {
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct StartCommand {
|
||||
disk: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
heal_endpoints: Vec<String>,
|
||||
bucket: String,
|
||||
object_prefix: Option<String>,
|
||||
object_version_id: Option<String>,
|
||||
@@ -115,7 +113,6 @@ impl TryFrom<HealChannelRequest> for StartCommand {
|
||||
fn try_from(request: HealChannelRequest) -> Result<Self, Self::Error> {
|
||||
Ok(Self {
|
||||
disk: request.disk,
|
||||
heal_endpoints: request.heal_endpoints,
|
||||
bucket: request.bucket,
|
||||
object_prefix: request.object_prefix,
|
||||
object_version_id: request.object_version_id,
|
||||
@@ -149,7 +146,6 @@ impl StartCommand {
|
||||
Ok(HealChannelRequest {
|
||||
id: request_id,
|
||||
disk: self.disk,
|
||||
heal_endpoints: self.heal_endpoints,
|
||||
bucket: self.bucket,
|
||||
object_prefix: self.object_prefix,
|
||||
object_version_id: self.object_version_id,
|
||||
@@ -636,12 +632,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn replacement_test_request(request_id: String) -> HealChannelRequest {
|
||||
let mut request = test_request(request_id);
|
||||
request.heal_endpoints = vec!["http://node1:9000/drive2".to_string()];
|
||||
request
|
||||
}
|
||||
|
||||
fn metadata(byte: u8, epoch: u64) -> RequestMetadata {
|
||||
RequestMetadata::new([byte; 16], 1_000, 2_000, epoch)
|
||||
}
|
||||
@@ -649,7 +639,7 @@ mod tests {
|
||||
#[test]
|
||||
fn round_trips_all_commands_and_results() {
|
||||
let request_id = uuid::Uuid::new_v4().to_string();
|
||||
let start = Envelope::start(replacement_test_request(request_id), metadata(1, 7)).unwrap();
|
||||
let start = Envelope::start(test_request(request_id), metadata(1, 7)).unwrap();
|
||||
let query = Envelope::query(
|
||||
uuid::Uuid::new_v4().to_string(),
|
||||
metadata(2, 7),
|
||||
|
||||
@@ -59,9 +59,8 @@ pub use mrf::{
|
||||
MrfV2Envelope, MrfV2Error, MrfV2Reader, MrfV2Readiness, decode_mrf_file, encode_mrf_file,
|
||||
};
|
||||
pub use multipart::{
|
||||
REPLICATION_MAX_SINGLE_PUT_SIZE, ReplicationMultipartPartInput, ReplicationMultipartPartPlan, ReplicationMultipartPlanError,
|
||||
ReplicationMultipartRange, replication_multipart_complete_actual_size, replication_multipart_part_plan,
|
||||
replication_single_put_size_error,
|
||||
ReplicationMultipartPartInput, ReplicationMultipartPartPlan, ReplicationMultipartPlanError, ReplicationMultipartRange,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan,
|
||||
};
|
||||
pub use object::{
|
||||
ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate, content_matches_by_etag,
|
||||
|
||||
@@ -109,49 +109,15 @@ pub fn replication_multipart_complete_actual_size(user_defined: &HashMap<String,
|
||||
get_internal_metadata(user_defined, SUFFIX_ACTUAL_SIZE).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Largest body S3 accepts on a single `PutObject`. Anything above this has to
|
||||
/// be uploaded as multipart; the limit is part of the S3 API, not a RustFS
|
||||
/// tunable, so every generic S3 target enforces it.
|
||||
pub const REPLICATION_MAX_SINGLE_PUT_SIZE: i64 = 5 * 1024 * 1024 * 1024;
|
||||
|
||||
/// Reject a single-`PutObject` replication transfer the target can never accept.
|
||||
///
|
||||
/// Replication mirrors the object's *source-side storage shape*: an object
|
||||
/// written to the source with one `PutObject` replicates with one `PutObject`
|
||||
/// whatever its size, and a multipart object replays the source's own part
|
||||
/// layout. So a source object larger than [`REPLICATION_MAX_SINGLE_PUT_SIZE`]
|
||||
/// that was not written as multipart can never reach a generic S3 target — the
|
||||
/// remote rejects it with `EntityTooLarge`, but only after the whole body has
|
||||
/// been streamed to it (rustfs#6825).
|
||||
///
|
||||
/// Returning the failure up front turns an unbounded wasted transfer plus an
|
||||
/// opaque remote error into a stated, diagnosable limit. RustFS deliberately
|
||||
/// does not re-chunk such an object into multipart on the replication side:
|
||||
/// the target's part layout is the source's, and rewriting it would break the
|
||||
/// ETag/part identity that heal and delete convergence address.
|
||||
pub fn replication_single_put_size_error(is_multipart: bool, transfer_size: i64) -> Option<String> {
|
||||
if is_multipart || transfer_size <= REPLICATION_MAX_SINGLE_PUT_SIZE {
|
||||
return None;
|
||||
}
|
||||
Some(format!(
|
||||
"object of {transfer_size} bytes was not written as multipart on the source and exceeds the \
|
||||
{REPLICATION_MAX_SINGLE_PUT_SIZE} byte single-PutObject limit of an S3 target; \
|
||||
re-upload it with multipart to make it replicable"
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
REPLICATION_MAX_SINGLE_PUT_SIZE, ReplicationMultipartPartInput, ReplicationMultipartPartPlan,
|
||||
ReplicationMultipartPlanError, ReplicationMultipartRange, replication_multipart_complete_actual_size,
|
||||
replication_multipart_part_plan, replication_single_put_size_error,
|
||||
ReplicationMultipartPartInput, ReplicationMultipartPartPlan, ReplicationMultipartPlanError, ReplicationMultipartRange,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan,
|
||||
};
|
||||
use crate::http::{SUFFIX_ACTUAL_SIZE, insert_internal_metadata};
|
||||
use std::collections::HashMap;
|
||||
|
||||
const MIB: i64 = 1024 * 1024;
|
||||
|
||||
#[test]
|
||||
fn multipart_part_plan_builds_range_and_next_offset() {
|
||||
assert_eq!(
|
||||
@@ -253,44 +219,4 @@ mod tests {
|
||||
assert_eq!(replication_multipart_complete_actual_size(&user_defined), "123");
|
||||
assert!(replication_multipart_complete_actual_size(&HashMap::new()).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_put_size_guard_admits_transfers_a_target_can_accept() {
|
||||
for size in [
|
||||
0,
|
||||
1,
|
||||
MIB,
|
||||
REPLICATION_MAX_SINGLE_PUT_SIZE - 1,
|
||||
REPLICATION_MAX_SINGLE_PUT_SIZE,
|
||||
] {
|
||||
assert_eq!(
|
||||
replication_single_put_size_error(false, size),
|
||||
None,
|
||||
"single PUT of {size} bytes is within the S3 limit and must not be rejected"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_put_size_guard_rejects_an_oversized_single_put() {
|
||||
let size = REPLICATION_MAX_SINGLE_PUT_SIZE + 1;
|
||||
let err = replication_single_put_size_error(false, size).expect("oversized single PUT must be rejected");
|
||||
|
||||
// The message is the operator's diagnosis: it has to name the actual
|
||||
// size, the limit, and the reason the object is on this route at all.
|
||||
assert!(err.contains(&size.to_string()), "message must name the object size: {err}");
|
||||
assert!(
|
||||
err.contains(&REPLICATION_MAX_SINGLE_PUT_SIZE.to_string()),
|
||||
"message must name the limit: {err}"
|
||||
);
|
||||
assert!(err.contains("multipart"), "message must name the remedy: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_put_size_guard_never_rejects_the_multipart_route() {
|
||||
// Multipart replays the source part layout, so object size alone says
|
||||
// nothing about whether the target will accept it; the per-part limits
|
||||
// are the target's to enforce.
|
||||
assert_eq!(replication_single_put_size_error(true, REPLICATION_MAX_SINGLE_PUT_SIZE * 1024), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,15 +90,8 @@ s3s = { workspace = true, features = ["minio"] }
|
||||
hex-simd.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
tokio-test = { workspace = true }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
proptest = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
hyper = { workspace = true, features = ["http2", "server"] }
|
||||
hyper-util = { workspace = true, features = ["tokio"] }
|
||||
http-body-util = { workspace = true }
|
||||
|
||||
[[bench]]
|
||||
name = "tee_reader"
|
||||
harness = false
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Throughput of `tee_reader` versus reading the same source directly:
|
||||
//! 64 MiB of data served in 1 MiB chunks, consumed with 1 MiB reads.
|
||||
|
||||
use bytes::Bytes;
|
||||
use criterion::{Criterion, Throughput, criterion_group, criterion_main};
|
||||
use rustfs_rio::tee_reader;
|
||||
use std::hint::black_box;
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf};
|
||||
|
||||
const CHUNK_BYTES: usize = 1024 * 1024;
|
||||
const TOTAL_BYTES: usize = 64 * 1024 * 1024;
|
||||
const TEE_BUFFER_BYTES: usize = 4 * CHUNK_BYTES;
|
||||
|
||||
/// In-memory source that serves at most `CHUNK_BYTES` per poll.
|
||||
struct ChunkedSource {
|
||||
data: Bytes,
|
||||
pos: usize,
|
||||
}
|
||||
|
||||
impl AsyncRead for ChunkedSource {
|
||||
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
let remaining = self.data.len() - self.pos;
|
||||
let n = CHUNK_BYTES.min(remaining).min(buf.remaining());
|
||||
buf.put_slice(&self.data[self.pos..self.pos + n]);
|
||||
self.pos += n;
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
async fn consume<R: AsyncRead + Unpin>(mut reader: R) -> usize {
|
||||
let mut buf = vec![0u8; CHUNK_BYTES];
|
||||
let mut total = 0;
|
||||
loop {
|
||||
let n = reader.read(&mut buf).await.expect("read");
|
||||
if n == 0 {
|
||||
return total;
|
||||
}
|
||||
total += n;
|
||||
}
|
||||
}
|
||||
|
||||
fn bench_tee_reader(c: &mut Criterion) {
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.worker_threads(2)
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("build tokio runtime for tee_reader benchmark");
|
||||
let data = Bytes::from(vec![0xA5u8; TOTAL_BYTES]);
|
||||
|
||||
let mut group = c.benchmark_group("tee_reader_64mib_1mib_chunks");
|
||||
group.throughput(Throughput::Bytes(TOTAL_BYTES as u64));
|
||||
group.sample_size(10);
|
||||
|
||||
group.bench_function("direct_read", |b| {
|
||||
b.iter(|| {
|
||||
let source = ChunkedSource {
|
||||
data: data.clone(),
|
||||
pos: 0,
|
||||
};
|
||||
let total = runtime.block_on(consume(source));
|
||||
black_box(total)
|
||||
})
|
||||
});
|
||||
|
||||
group.bench_function("tee_primary_plus_secondary", |b| {
|
||||
b.iter(|| {
|
||||
let source = ChunkedSource {
|
||||
data: data.clone(),
|
||||
pos: 0,
|
||||
};
|
||||
let (primary, secondary) = tee_reader(source, TEE_BUFFER_BYTES);
|
||||
let totals = runtime.block_on(async {
|
||||
let secondary_task = tokio::spawn(consume(secondary));
|
||||
let primary_total = consume(primary).await;
|
||||
let secondary_total = secondary_task.await.expect("secondary task");
|
||||
(primary_total, secondary_total)
|
||||
});
|
||||
black_box(totals)
|
||||
})
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_tee_reader);
|
||||
criterion_main!(benches);
|
||||
@@ -118,12 +118,6 @@ pub use hardlimit_reader::HardLimitReader;
|
||||
|
||||
mod hash_reader;
|
||||
pub use hash_reader::*;
|
||||
|
||||
mod tee_reader;
|
||||
pub use tee_reader::{
|
||||
DEFAULT_TEE_MAX_DRAIN_BYTES, TeeDrainLimitExceeded, TeeOptions, TeePrimary, TeeSecondary, TeeStream, tee_reader,
|
||||
tee_reader_with_options,
|
||||
};
|
||||
mod checksum;
|
||||
pub use checksum::*;
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -552,21 +552,18 @@ pub(super) fn data_usage_info_has_persisted_baseline_identity(info: &DataUsageIn
|
||||
}
|
||||
|
||||
pub(super) fn data_usage_info_is_bootstrap_pending(info: &DataUsageInfo) -> bool {
|
||||
let Some(last_update) = info.last_update else {
|
||||
if info.last_update.is_none() || info.scanner_cycle.is_some() {
|
||||
return false;
|
||||
};
|
||||
}
|
||||
|
||||
info == &scanner_usage_bootstrap_marker(last_update, info.scanner_epoch)
|
||||
}
|
||||
|
||||
pub(super) fn scanner_usage_bootstrap_marker(last_update: std::time::SystemTime, scanner_epoch: Option<u64>) -> DataUsageInfo {
|
||||
DataUsageInfo {
|
||||
last_update: Some(last_update),
|
||||
scanner_epoch,
|
||||
let expected = DataUsageInfo {
|
||||
last_update: info.last_update,
|
||||
scanner_epoch: info.scanner_epoch,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_bootstrap_pending: true,
|
||||
..Default::default()
|
||||
}
|
||||
};
|
||||
info == &expected
|
||||
}
|
||||
|
||||
fn usage_cache_needs_prompt_scan(authoritative: &DataUsageInfo, observed: Option<&DataUsageInfo>) -> bool {
|
||||
@@ -918,8 +915,8 @@ fn prepare_cycle_for_usage_floor_bootstrap(
|
||||
},
|
||||
)
|
||||
}
|
||||
PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence => {
|
||||
// The legacy incomplete fence proves only its leader epoch, not
|
||||
PersistedUsageFloorStartup::RecoveredLegacyEmptyFence => {
|
||||
// The legacy empty fence proves only its leader epoch, not
|
||||
// namespace coverage. Clear coverage while retaining the durable
|
||||
// cycle number so surviving caches cannot force a regression.
|
||||
let next = cycle_info.next;
|
||||
@@ -1441,7 +1438,6 @@ async fn fence_scanner_epoch_after_cycle_timeout<Store, LockLost>(
|
||||
cycle_info: &mut CurrentCycle,
|
||||
cycle_revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: &mut u64,
|
||||
allow_bootstrap_pending: bool,
|
||||
lock_lost: LockLost,
|
||||
) -> bool
|
||||
where
|
||||
@@ -1455,7 +1451,7 @@ where
|
||||
cycle_info,
|
||||
cycle_revision,
|
||||
leader_epoch,
|
||||
allow_bootstrap_pending,
|
||||
false,
|
||||
ScannerCycleResetPolicy::None,
|
||||
);
|
||||
tokio::pin!(claim);
|
||||
@@ -1477,7 +1473,6 @@ struct ScannerCycleDeadlineState<'a> {
|
||||
cycle_revision: &'a mut DataUsageCacheRevision,
|
||||
leader_epoch: &'a mut u64,
|
||||
cycle_budget: &'a ScannerCycleBudget,
|
||||
allow_bootstrap_pending: bool,
|
||||
}
|
||||
|
||||
fn cycle_timeout_requires_recovery(worker_stopped: bool, cycle_state_persisted: bool, generation_fenced: bool) -> bool {
|
||||
@@ -1499,7 +1494,6 @@ async fn handle_scanner_cycle_deadline<Store>(
|
||||
state.cycle_info,
|
||||
state.cycle_revision,
|
||||
state.leader_epoch,
|
||||
state.allow_bootstrap_pending,
|
||||
guard.lock_lost_notified(),
|
||||
)
|
||||
.await;
|
||||
@@ -2581,7 +2575,7 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
match usage_floor_startup {
|
||||
PersistedUsageFloorStartup::Authoritative
|
||||
| PersistedUsageFloorStartup::BootstrapPending
|
||||
| PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence => {}
|
||||
| PersistedUsageFloorStartup::RecoveredLegacyEmptyFence => {}
|
||||
PersistedUsageFloorStartup::Missing => {
|
||||
if ctx.is_cancelled() || guard.is_lock_lost() {
|
||||
global_metrics().set_cycle(None).await;
|
||||
@@ -2676,8 +2670,8 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
finish_scanner_leader_iteration(false, "epoch_claim_failed", "leadership epoch claim failed".to_string()).await;
|
||||
return Ok(());
|
||||
}
|
||||
if usage_floor_startup == PersistedUsageFloorStartup::RecoveredLegacyIncompleteFence
|
||||
&& let Err(err) = complete_legacy_incomplete_usage_floor_recovery(storeapi.clone(), leader_epoch).await
|
||||
if usage_floor_startup == PersistedUsageFloorStartup::RecoveredLegacyEmptyFence
|
||||
&& let Err(err) = complete_legacy_empty_usage_floor_recovery(storeapi.clone(), leader_epoch).await
|
||||
{
|
||||
let error = err.to_string();
|
||||
warn!(
|
||||
@@ -2750,7 +2744,6 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
cycle_revision: &mut cycle_revision,
|
||||
leader_epoch: &mut leader_epoch,
|
||||
cycle_budget: &cycle_budget,
|
||||
allow_bootstrap_pending: allow_usage_floor_bootstrap_pending,
|
||||
},
|
||||
worker_stopped,
|
||||
&mut guard,
|
||||
@@ -3040,7 +3033,6 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
cycle_revision: &mut cycle_revision,
|
||||
leader_epoch: &mut leader_epoch,
|
||||
cycle_budget: &cycle_budget,
|
||||
allow_bootstrap_pending: allow_usage_floor_bootstrap_pending,
|
||||
},
|
||||
worker_stopped,
|
||||
&mut guard,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user