build(make): resolve a Python 3.11+ interpreter for guard scripts (#7004)
scripts/check_test_wiring.py and scripts/check_security_coverage.py import tomllib, which landed in Python 3.11. macOS ships /usr/bin/python3 at 3.9, so `make pre-commit` failed on a clean machine with `ModuleNotFoundError: No module named 'tomllib'` in test-wiring-check, even though the checkers themselves are fine. Add scripts/python_bin.sh, which resolves an interpreter (explicit RUSTFS_PYTHON, then python3.14..3.11/python3/python on PATH, then a `uv run --python 3.12 --no-project` fallback) and execs it, failing with the concrete remediation when nothing usable exists. Route the Make call sites through RUSTFS_PYTHON_BIN. CI workflows keep calling python3 directly because their runners already provide 3.11+.
This commit is contained in:
唐小鸭
@@ -23,4 +23,4 @@ coverage: core-deps ## Workspace line coverage (cargo-llvm-cov + nextest; slow,
|
||||
@mkdir -p target/llvm-cov
|
||||
cargo llvm-cov report --lcov --output-path target/llvm-cov/lcov.info
|
||||
cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
|
||||
python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json
|
||||
$(RUSTFS_PYTHON_BIN) scripts/coverage_per_crate.py target/llvm-cov/coverage.json
|
||||
|
||||
@@ -88,7 +88,7 @@ offline-enrollment-e2e-check: core-deps ## Build and exercise the dedicated offl
|
||||
.PHONY: test-wiring-check
|
||||
test-wiring-check: ## Check tests stay registered and selected by their intended runners
|
||||
@echo "🧪 Checking test wiring..."
|
||||
python3 ./scripts/check_test_wiring.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py
|
||||
|
||||
.PHONY: log-analyzer-rules-check
|
||||
log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source
|
||||
|
||||
@@ -35,13 +35,14 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_pinned_paired_abba_bench.sh
|
||||
./scripts/test_manual_transition_runbooks.sh
|
||||
./scripts/test_fuzz_runner.sh
|
||||
./scripts/test_python_bin.sh
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
python3 ./scripts/check_test_wiring.py --self-test
|
||||
python3 ./scripts/check_security_coverage.py --self-test
|
||||
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
python3 ./scripts/s3-tests/test_report_compat.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
|
||||
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
||||
python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||
./scripts/validate_object_data_cache_cold_stampede.sh --self-test
|
||||
|
||||
.PHONY: test
|
||||
|
||||
@@ -68,6 +68,14 @@ make pre-pr
|
||||
|
||||
> `make test` requires [cargo-nextest](https://nexte.st) (CI runs it and only nextest honours `.config/nextest.toml` test-groups). Install it with `cargo install cargo-nextest --locked` or a prebuilt binary (see https://nexte.st/docs/installation/). To run the plain `cargo test` fallback anyway (results not authoritative — serialization semantics differ from CI), set `RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1`.
|
||||
|
||||
> Some guard checks are Python (`test-wiring-check` in `make pre-commit`, plus the
|
||||
> security-coverage and scheduled-validation self-tests in `make test`) and import
|
||||
> `tomllib`, so they need **Python 3.11+**. Make resolves the interpreter through
|
||||
> `scripts/python_bin.sh`, which prefers a `python3.11`+ on `PATH` and otherwise falls
|
||||
> back to `uv run --python 3.12`. macOS ships `/usr/bin/python3` at 3.9, so install a
|
||||
> newer one (`brew install python@3.12`) or [uv](https://docs.astral.sh/uv/); pin a
|
||||
> specific interpreter with `RUSTFS_PYTHON=/path/to/python3.12`.
|
||||
|
||||
> For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md).
|
||||
|
||||
> For the event, timeout, required-status, and local reproduction matrix, see [docs/testing/ci-gates.md](docs/testing/ci-gates.md).
|
||||
|
||||
@@ -23,6 +23,12 @@ SHELL := $(shell which bash)
|
||||
.SHELLFLAGS = -eu -o pipefail -c
|
||||
|
||||
DOCKER_CLI ?= docker
|
||||
# Python interpreter for the repository's helper scripts. They import tomllib
|
||||
# (Python 3.11+), while macOS still ships /usr/bin/python3 at 3.9, so calls go
|
||||
# through a resolver that picks a new-enough interpreter (or falls back to uv).
|
||||
# Override with RUSTFS_PYTHON=/path/to/python3.12, or replace the resolver via
|
||||
# RUSTFS_PYTHON_BIN=<command>.
|
||||
RUSTFS_PYTHON_BIN ?= ./scripts/python_bin.sh
|
||||
IMAGE_NAME ?= rustfs:v1.0.0
|
||||
CONTAINER_NAME ?= rustfs-dev
|
||||
# Docker build configurations
|
||||
|
||||
Executable
+99
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env bash
|
||||
# Resolve a Python interpreter new enough for this repository's helper scripts
|
||||
# and exec it with the given arguments.
|
||||
#
|
||||
# Why this exists: scripts/check_test_wiring.py and
|
||||
# scripts/check_security_coverage.py import `tomllib`, which only landed in
|
||||
# Python 3.11. macOS ships /usr/bin/python3 at 3.9, so a plain
|
||||
# `python3 ./scripts/check_test_wiring.py` makes `make pre-commit` fail on a
|
||||
# clean machine with `ModuleNotFoundError: No module named 'tomllib'` even
|
||||
# though the checker itself is fine. Make targets go through this wrapper so
|
||||
# the gate works wherever a new-enough interpreter (or `uv`) is reachable.
|
||||
#
|
||||
# CI runners already provide Python 3.11+ as `python3`, so .github/workflows
|
||||
# keeps calling `python3` directly.
|
||||
#
|
||||
# Resolution order:
|
||||
# 1. $RUSTFS_PYTHON, if set (must itself be >= the minimum version)
|
||||
# 2. python3.14 / python3.13 / python3.12 / python3.11 / python3 / python
|
||||
# 3. `uv run --python <spec> --no-project python`
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/python_bin.sh ./scripts/check_test_wiring.py --self-test
|
||||
# ./scripts/python_bin.sh --print-interpreter # report what would be used
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
MIN_MAJOR=3
|
||||
MIN_MINOR=11
|
||||
UV_PYTHON_SPEC="${RUSTFS_UV_PYTHON:-3.12}"
|
||||
|
||||
version_ok() {
|
||||
"$1" -c "import sys; raise SystemExit(0 if sys.version_info >= (${MIN_MAJOR}, ${MIN_MINOR}) else 1)" \
|
||||
>/dev/null 2>&1
|
||||
}
|
||||
|
||||
if [ "${1:---help}" = "--help" ] || [ "${1:-}" = "-h" ]; then
|
||||
cat <<USAGE
|
||||
Usage: scripts/python_bin.sh [--print-interpreter] [python arguments...]
|
||||
|
||||
Execs a Python >= ${MIN_MAJOR}.${MIN_MINOR} interpreter (the repository's checkers import
|
||||
tomllib) with the given arguments. Resolution order:
|
||||
1. \$RUSTFS_PYTHON, if set
|
||||
2. python3.14 / python3.13 / python3.12 / python3.11 / python3 / python
|
||||
3. uv run --python \${RUSTFS_UV_PYTHON:-3.12} --no-project python
|
||||
|
||||
--print-interpreter Print the interpreter that would be used and exit.
|
||||
USAGE
|
||||
exit 0
|
||||
fi
|
||||
|
||||
print_only=0
|
||||
if [ "${1:-}" = "--print-interpreter" ]; then
|
||||
print_only=1
|
||||
shift
|
||||
fi
|
||||
|
||||
if [ -n "${RUSTFS_PYTHON:-}" ]; then
|
||||
if ! command -v "${RUSTFS_PYTHON}" >/dev/null 2>&1; then
|
||||
echo >&2 "❌ RUSTFS_PYTHON='${RUSTFS_PYTHON}' is not an executable command."
|
||||
exit 1
|
||||
fi
|
||||
if ! version_ok "${RUSTFS_PYTHON}"; then
|
||||
echo >&2 "❌ RUSTFS_PYTHON='${RUSTFS_PYTHON}' is older than Python ${MIN_MAJOR}.${MIN_MINOR}."
|
||||
echo >&2 " The repository's checkers import tomllib (Python ${MIN_MAJOR}.${MIN_MINOR}+)."
|
||||
exit 1
|
||||
fi
|
||||
if [ "${print_only}" = "1" ]; then
|
||||
command -v "${RUSTFS_PYTHON}"
|
||||
exit 0
|
||||
fi
|
||||
exec "${RUSTFS_PYTHON}" "$@"
|
||||
fi
|
||||
|
||||
for candidate in python3.14 python3.13 python3.12 python3.11 python3 python; do
|
||||
if command -v "${candidate}" >/dev/null 2>&1 && version_ok "${candidate}"; then
|
||||
if [ "${print_only}" = "1" ]; then
|
||||
command -v "${candidate}"
|
||||
exit 0
|
||||
fi
|
||||
exec "${candidate}" "$@"
|
||||
fi
|
||||
done
|
||||
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
echo >&2 "ℹ️ No Python ${MIN_MAJOR}.${MIN_MINOR}+ on PATH; using 'uv run --python ${UV_PYTHON_SPEC}'."
|
||||
if [ "${print_only}" = "1" ]; then
|
||||
echo "uv run --python ${UV_PYTHON_SPEC} --no-project python"
|
||||
exit 0
|
||||
fi
|
||||
exec uv run --python "${UV_PYTHON_SPEC}" --no-project python "$@"
|
||||
fi
|
||||
|
||||
echo >&2 "❌ No Python ${MIN_MAJOR}.${MIN_MINOR}+ interpreter found."
|
||||
echo >&2 " The repository's checkers import tomllib, added in Python ${MIN_MAJOR}.${MIN_MINOR}."
|
||||
echo >&2 " Fix it with any of:"
|
||||
echo >&2 " brew install [email protected] # macOS: /usr/bin/python3 is 3.9"
|
||||
echo >&2 " curl -LsSf https://astral.sh/uv/install.sh | sh # then re-run"
|
||||
echo >&2 " make ... RUSTFS_PYTHON=/path/to/python3.12"
|
||||
exit 1
|
||||
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/bin/sh
|
||||
# Tests for scripts/python_bin.sh: the interpreter resolver that keeps
|
||||
# `make pre-commit` working on machines whose `python3` is older than 3.11
|
||||
# (notably macOS, which ships /usr/bin/python3 at 3.9).
|
||||
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)
|
||||
RESOLVER="$REPO_ROOT/scripts/python_bin.sh"
|
||||
TMP_ROOT=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP_ROOT"' EXIT HUP INT TERM
|
||||
|
||||
fail() {
|
||||
echo "❌ $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# A resolved interpreter must be able to import tomllib, which is the module
|
||||
# the repository's checkers need and the reason the resolver exists.
|
||||
"$RESOLVER" -c 'import tomllib, sys; assert sys.version_info >= (3, 11)' \
|
||||
|| fail "resolver produced an interpreter without tomllib"
|
||||
|
||||
"$RESOLVER" --print-interpreter >/dev/null \
|
||||
|| fail "--print-interpreter failed"
|
||||
|
||||
# An explicit RUSTFS_PYTHON override wins over PATH discovery.
|
||||
selected=$("$RESOLVER" --print-interpreter)
|
||||
RUSTFS_PYTHON="$selected" "$RESOLVER" -c 'import tomllib' \
|
||||
|| fail "RUSTFS_PYTHON override rejected a valid interpreter"
|
||||
|
||||
# A too-old RUSTFS_PYTHON must fail loudly instead of silently falling back to
|
||||
# a newer interpreter, so the operator learns their override is unusable.
|
||||
mkdir -p "$TMP_ROOT/bin"
|
||||
cat > "$TMP_ROOT/bin/fake-old-python" <<'STUB'
|
||||
#!/bin/sh
|
||||
# Pretends to be Python 3.9: the version probe exits non-zero.
|
||||
exit 1
|
||||
STUB
|
||||
chmod +x "$TMP_ROOT/bin/fake-old-python"
|
||||
|
||||
if RUSTFS_PYTHON="$TMP_ROOT/bin/fake-old-python" "$RESOLVER" -c 'pass' \
|
||||
>"$TMP_ROOT/old.out" 2>"$TMP_ROOT/old.err"; then
|
||||
fail "resolver accepted a too-old RUSTFS_PYTHON"
|
||||
fi
|
||||
grep -q 'older than Python' "$TMP_ROOT/old.err" \
|
||||
|| fail "too-old RUSTFS_PYTHON did not explain the version requirement"
|
||||
|
||||
# A missing RUSTFS_PYTHON must be reported as such.
|
||||
if RUSTFS_PYTHON="$TMP_ROOT/bin/definitely-absent" "$RESOLVER" -c 'pass' \
|
||||
>"$TMP_ROOT/absent.out" 2>"$TMP_ROOT/absent.err"; then
|
||||
fail "resolver accepted a nonexistent RUSTFS_PYTHON"
|
||||
fi
|
||||
grep -q 'is not an executable command' "$TMP_ROOT/absent.err" \
|
||||
|| fail "nonexistent RUSTFS_PYTHON did not explain what was wrong"
|
||||
|
||||
# With no usable interpreter and no uv on PATH, the failure must name the fix
|
||||
# rather than surfacing a bare ModuleNotFoundError from a 3.9 interpreter.
|
||||
cat > "$TMP_ROOT/bin/python3" <<'STUB'
|
||||
#!/bin/sh
|
||||
exit 1
|
||||
STUB
|
||||
chmod +x "$TMP_ROOT/bin/python3"
|
||||
|
||||
SANDBOX_PATH="$TMP_ROOT/bin:/usr/bin:/bin"
|
||||
if PATH="$SANDBOX_PATH" command -v uv >/dev/null 2>&1; then
|
||||
# uv is reachable even from the sandbox PATH, so the resolver would
|
||||
# legitimately fall back to it instead of failing. Skip this case.
|
||||
echo "ℹ️ uv is on the sandbox PATH; skipping the no-interpreter case"
|
||||
else
|
||||
if PATH="$SANDBOX_PATH" "$RESOLVER" -c 'pass' \
|
||||
>"$TMP_ROOT/none.out" 2>"$TMP_ROOT/none.err"; then
|
||||
fail "resolver succeeded with no usable interpreter on PATH"
|
||||
fi
|
||||
grep -q 'No Python 3.11+ interpreter found' "$TMP_ROOT/none.err" \
|
||||
|| fail "missing-interpreter failure did not name the requirement"
|
||||
grep -q 'RUSTFS_PYTHON=' "$TMP_ROOT/none.err" \
|
||||
|| fail "missing-interpreter failure did not point at the override"
|
||||
fi
|
||||
|
||||
echo "✅ scripts/python_bin.sh resolver checks passed"
|
||||
Reference in New Issue
Block a user