diff --git a/.config/make/coverage.mak b/.config/make/coverage.mak index 366535d85..661552bc8 100644 --- a/.config/make/coverage.mak +++ b/.config/make/coverage.mak @@ -23,4 +23,4 @@ coverage: core-deps ## Workspace line coverage (cargo-llvm-cov + nextest; slow, @mkdir -p target/llvm-cov cargo llvm-cov report --lcov --output-path target/llvm-cov/lcov.info cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json - python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json + $(RUSTFS_PYTHON_BIN) scripts/coverage_per_crate.py target/llvm-cov/coverage.json diff --git a/.config/make/lint-fmt.mak b/.config/make/lint-fmt.mak index 391a06226..e0b2ca87c 100644 --- a/.config/make/lint-fmt.mak +++ b/.config/make/lint-fmt.mak @@ -88,7 +88,7 @@ offline-enrollment-e2e-check: core-deps ## Build and exercise the dedicated offl .PHONY: test-wiring-check test-wiring-check: ## Check tests stay registered and selected by their intended runners @echo "๐Ÿงช Checking test wiring..." - python3 ./scripts/check_test_wiring.py + $(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py .PHONY: log-analyzer-rules-check log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 3e485058a..3ee3337fa 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -35,13 +35,14 @@ script-tests: ## Run shell script tests ./scripts/test_pinned_paired_abba_bench.sh ./scripts/test_manual_transition_runbooks.sh ./scripts/test_fuzz_runner.sh + ./scripts/test_python_bin.sh ./scripts/check_embedded_secrets.sh --self-test - python3 ./scripts/check_test_wiring.py --self-test - python3 ./scripts/check_security_coverage.py --self-test - python3 ./scripts/check_scheduled_validation_freshness.py --self-test - python3 ./scripts/s3-tests/test_report_compat.py + $(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test + $(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test + $(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test + $(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py bash -n ./scripts/validate_object_data_cache_cold_stampede.sh - python3 ./scripts/check_object_data_cache_follower_samples.py --self-test + $(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test ./scripts/validate_object_data_cache_cold_stampede.sh --self-test .PHONY: test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 059bd4731..13bf35c08 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,6 +68,14 @@ make pre-pr > `make test` requires [cargo-nextest](https://nexte.st) (CI runs it and only nextest honours `.config/nextest.toml` test-groups). Install it with `cargo install cargo-nextest --locked` or a prebuilt binary (see https://nexte.st/docs/installation/). To run the plain `cargo test` fallback anyway (results not authoritative โ€” serialization semantics differ from CI), set `RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1`. +> Some guard checks are Python (`test-wiring-check` in `make pre-commit`, plus the +> security-coverage and scheduled-validation self-tests in `make test`) and import +> `tomllib`, so they need **Python 3.11+**. Make resolves the interpreter through +> `scripts/python_bin.sh`, which prefers a `python3.11`+ on `PATH` and otherwise falls +> back to `uv run --python 3.12`. macOS ships `/usr/bin/python3` at 3.9, so install a +> newer one (`brew install python@3.12`) or [uv](https://docs.astral.sh/uv/); pin a +> specific interpreter with `RUSTFS_PYTHON=/path/to/python3.12`. + > For the full test-layer taxonomy (unit / ecstore black-box / e2e / s3s-e2e / S3 compatibility / chaos / fuzz / bench), each layer's entry command, the naming conventions the migration gate depends on, and the serial/nextest rules, see [docs/testing/README.md](docs/testing/README.md). > For the event, timeout, required-status, and local reproduction matrix, see [docs/testing/ci-gates.md](docs/testing/ci-gates.md). diff --git a/Makefile b/Makefile index 94add2643..59cff44aa 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,12 @@ SHELL := $(shell which bash) .SHELLFLAGS = -eu -o pipefail -c DOCKER_CLI ?= docker +# Python interpreter for the repository's helper scripts. They import tomllib +# (Python 3.11+), while macOS still ships /usr/bin/python3 at 3.9, so calls go +# through a resolver that picks a new-enough interpreter (or falls back to uv). +# Override with RUSTFS_PYTHON=/path/to/python3.12, or replace the resolver via +# RUSTFS_PYTHON_BIN=. +RUSTFS_PYTHON_BIN ?= ./scripts/python_bin.sh IMAGE_NAME ?= rustfs:v1.0.0 CONTAINER_NAME ?= rustfs-dev # Docker build configurations diff --git a/scripts/python_bin.sh b/scripts/python_bin.sh new file mode 100755 index 000000000..b36b35a05 --- /dev/null +++ b/scripts/python_bin.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Resolve a Python interpreter new enough for this repository's helper scripts +# and exec it with the given arguments. +# +# Why this exists: scripts/check_test_wiring.py and +# scripts/check_security_coverage.py import `tomllib`, which only landed in +# Python 3.11. macOS ships /usr/bin/python3 at 3.9, so a plain +# `python3 ./scripts/check_test_wiring.py` makes `make pre-commit` fail on a +# clean machine with `ModuleNotFoundError: No module named 'tomllib'` even +# though the checker itself is fine. Make targets go through this wrapper so +# the gate works wherever a new-enough interpreter (or `uv`) is reachable. +# +# CI runners already provide Python 3.11+ as `python3`, so .github/workflows +# keeps calling `python3` directly. +# +# Resolution order: +# 1. $RUSTFS_PYTHON, if set (must itself be >= the minimum version) +# 2. python3.14 / python3.13 / python3.12 / python3.11 / python3 / python +# 3. `uv run --python --no-project python` +# +# Usage: +# ./scripts/python_bin.sh ./scripts/check_test_wiring.py --self-test +# ./scripts/python_bin.sh --print-interpreter # report what would be used + +set -euo pipefail + +MIN_MAJOR=3 +MIN_MINOR=11 +UV_PYTHON_SPEC="${RUSTFS_UV_PYTHON:-3.12}" + +version_ok() { + "$1" -c "import sys; raise SystemExit(0 if sys.version_info >= (${MIN_MAJOR}, ${MIN_MINOR}) else 1)" \ + >/dev/null 2>&1 +} + +if [ "${1:---help}" = "--help" ] || [ "${1:-}" = "-h" ]; then + cat <= ${MIN_MAJOR}.${MIN_MINOR} interpreter (the repository's checkers import +tomllib) with the given arguments. Resolution order: + 1. \$RUSTFS_PYTHON, if set + 2. python3.14 / python3.13 / python3.12 / python3.11 / python3 / python + 3. uv run --python \${RUSTFS_UV_PYTHON:-3.12} --no-project python + + --print-interpreter Print the interpreter that would be used and exit. +USAGE + exit 0 +fi + +print_only=0 +if [ "${1:-}" = "--print-interpreter" ]; then + print_only=1 + shift +fi + +if [ -n "${RUSTFS_PYTHON:-}" ]; then + if ! command -v "${RUSTFS_PYTHON}" >/dev/null 2>&1; then + echo >&2 "โŒ RUSTFS_PYTHON='${RUSTFS_PYTHON}' is not an executable command." + exit 1 + fi + if ! version_ok "${RUSTFS_PYTHON}"; then + echo >&2 "โŒ RUSTFS_PYTHON='${RUSTFS_PYTHON}' is older than Python ${MIN_MAJOR}.${MIN_MINOR}." + echo >&2 " The repository's checkers import tomllib (Python ${MIN_MAJOR}.${MIN_MINOR}+)." + exit 1 + fi + if [ "${print_only}" = "1" ]; then + command -v "${RUSTFS_PYTHON}" + exit 0 + fi + exec "${RUSTFS_PYTHON}" "$@" +fi + +for candidate in python3.14 python3.13 python3.12 python3.11 python3 python; do + if command -v "${candidate}" >/dev/null 2>&1 && version_ok "${candidate}"; then + if [ "${print_only}" = "1" ]; then + command -v "${candidate}" + exit 0 + fi + exec "${candidate}" "$@" + fi +done + +if command -v uv >/dev/null 2>&1; then + echo >&2 "โ„น๏ธ No Python ${MIN_MAJOR}.${MIN_MINOR}+ on PATH; using 'uv run --python ${UV_PYTHON_SPEC}'." + if [ "${print_only}" = "1" ]; then + echo "uv run --python ${UV_PYTHON_SPEC} --no-project python" + exit 0 + fi + exec uv run --python "${UV_PYTHON_SPEC}" --no-project python "$@" +fi + +echo >&2 "โŒ No Python ${MIN_MAJOR}.${MIN_MINOR}+ interpreter found." +echo >&2 " The repository's checkers import tomllib, added in Python ${MIN_MAJOR}.${MIN_MINOR}." +echo >&2 " Fix it with any of:" +echo >&2 " brew install python@3.12 # macOS: /usr/bin/python3 is 3.9" +echo >&2 " curl -LsSf https://astral.sh/uv/install.sh | sh # then re-run" +echo >&2 " make ... RUSTFS_PYTHON=/path/to/python3.12" +exit 1 diff --git a/scripts/test_python_bin.sh b/scripts/test_python_bin.sh new file mode 100755 index 000000000..4c5e149fb --- /dev/null +++ b/scripts/test_python_bin.sh @@ -0,0 +1,81 @@ +#!/bin/sh +# Tests for scripts/python_bin.sh: the interpreter resolver that keeps +# `make pre-commit` working on machines whose `python3` is older than 3.11 +# (notably macOS, which ships /usr/bin/python3 at 3.9). + +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd) +RESOLVER="$REPO_ROOT/scripts/python_bin.sh" +TMP_ROOT=$(mktemp -d) +trap 'rm -rf "$TMP_ROOT"' EXIT HUP INT TERM + +fail() { + echo "โŒ $1" >&2 + exit 1 +} + +# A resolved interpreter must be able to import tomllib, which is the module +# the repository's checkers need and the reason the resolver exists. +"$RESOLVER" -c 'import tomllib, sys; assert sys.version_info >= (3, 11)' \ + || fail "resolver produced an interpreter without tomllib" + +"$RESOLVER" --print-interpreter >/dev/null \ + || fail "--print-interpreter failed" + +# An explicit RUSTFS_PYTHON override wins over PATH discovery. +selected=$("$RESOLVER" --print-interpreter) +RUSTFS_PYTHON="$selected" "$RESOLVER" -c 'import tomllib' \ + || fail "RUSTFS_PYTHON override rejected a valid interpreter" + +# A too-old RUSTFS_PYTHON must fail loudly instead of silently falling back to +# a newer interpreter, so the operator learns their override is unusable. +mkdir -p "$TMP_ROOT/bin" +cat > "$TMP_ROOT/bin/fake-old-python" <<'STUB' +#!/bin/sh +# Pretends to be Python 3.9: the version probe exits non-zero. +exit 1 +STUB +chmod +x "$TMP_ROOT/bin/fake-old-python" + +if RUSTFS_PYTHON="$TMP_ROOT/bin/fake-old-python" "$RESOLVER" -c 'pass' \ + >"$TMP_ROOT/old.out" 2>"$TMP_ROOT/old.err"; then + fail "resolver accepted a too-old RUSTFS_PYTHON" +fi +grep -q 'older than Python' "$TMP_ROOT/old.err" \ + || fail "too-old RUSTFS_PYTHON did not explain the version requirement" + +# A missing RUSTFS_PYTHON must be reported as such. +if RUSTFS_PYTHON="$TMP_ROOT/bin/definitely-absent" "$RESOLVER" -c 'pass' \ + >"$TMP_ROOT/absent.out" 2>"$TMP_ROOT/absent.err"; then + fail "resolver accepted a nonexistent RUSTFS_PYTHON" +fi +grep -q 'is not an executable command' "$TMP_ROOT/absent.err" \ + || fail "nonexistent RUSTFS_PYTHON did not explain what was wrong" + +# With no usable interpreter and no uv on PATH, the failure must name the fix +# rather than surfacing a bare ModuleNotFoundError from a 3.9 interpreter. +cat > "$TMP_ROOT/bin/python3" <<'STUB' +#!/bin/sh +exit 1 +STUB +chmod +x "$TMP_ROOT/bin/python3" + +SANDBOX_PATH="$TMP_ROOT/bin:/usr/bin:/bin" +if PATH="$SANDBOX_PATH" command -v uv >/dev/null 2>&1; then + # uv is reachable even from the sandbox PATH, so the resolver would + # legitimately fall back to it instead of failing. Skip this case. + echo "โ„น๏ธ uv is on the sandbox PATH; skipping the no-interpreter case" +else + if PATH="$SANDBOX_PATH" "$RESOLVER" -c 'pass' \ + >"$TMP_ROOT/none.out" 2>"$TMP_ROOT/none.err"; then + fail "resolver succeeded with no usable interpreter on PATH" + fi + grep -q 'No Python 3.11+ interpreter found' "$TMP_ROOT/none.err" \ + || fail "missing-interpreter failure did not name the requirement" + grep -q 'RUSTFS_PYTHON=' "$TMP_ROOT/none.err" \ + || fail "missing-interpreter failure did not point at the override" +fi + +echo "โœ… scripts/python_bin.sh resolver checks passed"