fran/rustfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ci(functional): retry chain handoffs and alert on stall (#7023)

Browse Source 
The repository_dispatch handoff step was continue-on-error with a single
attempt: if the call failed (token lacking contents:write, transient API
error), the chain stalled silently while every job stayed green.

Each handoff now retries 3x and, if all attempts fail, files an alert
issue in rustfs/backlog with the exact recovery command before exiting 1
(still continue-on-error, so suite workflows themselves never fail).
This commit is contained in:
hector
2026-09-01 23:11:25 +08:00
committed by GitHub
parent cebe57a2f0
commit bba934723a
7 changed files with 266 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/rustfs-heal-test.yml
+38 -7
View File
@@ -284,21 +284,52 @@ jobs:
- name: "Continue functional chain (next: Pool expansion)"
# Only chain-triggered runs forward to the next suite; standalone
# workflow_dispatch runs stop after their own cleanup.
# workflow_dispatch runs stop after their own cleanup. A failed
# handoff must never pass silently: it retries, then files an alert
# issue in rustfs/backlog so a stalled chain is visible.
if: ${{ always() && github.event_name == 'repository_dispatch' }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
set -uo pipefail
if [ -z "${{GH_TOKEN:-}}" ]; then
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
exit 1
fi
echo "Dispatching next functional suite: Pool expansion"
gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-pool' \
-F 'client_payload[from_suite]=heal'
DISPATCHED=0
for attempt in 1 2 3; do
if gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-pool' \
-F 'client_payload[from_suite]=heal'; then
echo "dispatched next suite Pool expansion (attempt ${{attempt}})"
DISPATCHED=1
break
fi
echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
sleep "${{attempt}}0"
done
if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
echo "ERROR: functional chain stalled: could not dispatch Pool expansion after 3 attempts" >&2
TITLE="[functional][chain] stalled after heal (run ${{GITHUB_RUN_ID}})"
BODY_FILE="$(mktemp)"
{
echo "The functional chain could not hand off from **heal** to **Pool expansion** after 3 attempts."
echo ""
echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
echo "- Expected next event: `rustfs-chain-pool`"
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
echo "- Recovery: re-dispatch manually with"
echo " ```"
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-pool'"
echo " ```"
} > "${{BODY_FILE}}"
gh issue create -R rustfs/backlog --title "${{TITLE}}" \
--body-file "${{BODY_FILE}}" --label functional-test \
|| gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
|| echo "could not file the stall alert issue either; check the token" >&2
exit 1
fi
- name: Notify on failure
if: failure()
.github/workflows/rustfs-kms-test.yml
+38 -7
View File
@@ -340,21 +340,52 @@ jobs:
- name: "Continue functional chain (next: Tier)"
# Only chain-triggered runs forward to the next suite; standalone
# workflow_dispatch runs stop after their own cleanup.
# workflow_dispatch runs stop after their own cleanup. A failed
# handoff must never pass silently: it retries, then files an alert
# issue in rustfs/backlog so a stalled chain is visible.
if: ${{ always() && github.event_name == 'repository_dispatch' }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
set -uo pipefail
if [ -z "${{GH_TOKEN:-}}" ]; then
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
exit 1
fi
echo "Dispatching next functional suite: Tier"
gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-tier' \
-F 'client_payload[from_suite]=kms'
DISPATCHED=0
for attempt in 1 2 3; do
if gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-tier' \
-F 'client_payload[from_suite]=kms'; then
echo "dispatched next suite Tier (attempt ${{attempt}})"
DISPATCHED=1
break
fi
echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
sleep "${{attempt}}0"
done
if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
echo "ERROR: functional chain stalled: could not dispatch Tier after 3 attempts" >&2
TITLE="[functional][chain] stalled after kms (run ${{GITHUB_RUN_ID}})"
BODY_FILE="$(mktemp)"
{
echo "The functional chain could not hand off from **kms** to **Tier** after 3 attempts."
echo ""
echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
echo "- Expected next event: `rustfs-chain-tier`"
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
echo "- Recovery: re-dispatch manually with"
echo " ```"
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-tier'"
echo " ```"
} > "${{BODY_FILE}}"
gh issue create -R rustfs/backlog --title "${{TITLE}}" \
--body-file "${{BODY_FILE}}" --label functional-test \
|| gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
|| echo "could not file the stall alert issue either; check the token" >&2
exit 1
fi
- name: Notify on failure
if: failure()
.github/workflows/rustfs-pool-expand-test.yml
+38 -7
View File
@@ -596,21 +596,52 @@ jobs:
- name: "Continue functional chain (next: Security)"
# Only chain-triggered runs forward to the next suite; standalone
# workflow_dispatch runs stop after their own cleanup.
# workflow_dispatch runs stop after their own cleanup. A failed
# handoff must never pass silently: it retries, then files an alert
# issue in rustfs/backlog so a stalled chain is visible.
if: ${{ always() && github.event_name == 'repository_dispatch' }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
set -uo pipefail
if [ -z "${{GH_TOKEN:-}}" ]; then
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
exit 1
fi
echo "Dispatching next functional suite: Security"
gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-security' \
-F 'client_payload[from_suite]=pool'
DISPATCHED=0
for attempt in 1 2 3; do
if gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-security' \
-F 'client_payload[from_suite]=pool'; then
echo "dispatched next suite Security (attempt ${{attempt}})"
DISPATCHED=1
break
fi
echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
sleep "${{attempt}}0"
done
if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
echo "ERROR: functional chain stalled: could not dispatch Security after 3 attempts" >&2
TITLE="[functional][chain] stalled after pool (run ${{GITHUB_RUN_ID}})"
BODY_FILE="$(mktemp)"
{
echo "The functional chain could not hand off from **pool** to **Security** after 3 attempts."
echo ""
echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
echo "- Expected next event: `rustfs-chain-security`"
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
echo "- Recovery: re-dispatch manually with"
echo " ```"
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-security'"
echo " ```"
} > "${{BODY_FILE}}"
gh issue create -R rustfs/backlog --title "${{TITLE}}" \
--body-file "${{BODY_FILE}}" --label functional-test \
|| gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
|| echo "could not file the stall alert issue either; check the token" >&2
exit 1
fi
- name: Notify on failure
if: failure()
.github/workflows/rustfs-s3-compat-test.yml
+38 -7
View File
@@ -320,21 +320,52 @@ jobs:
- name: "Continue functional chain (next: KMS)"
# Only chain-triggered runs forward to the next suite; standalone
# workflow_dispatch runs stop after their own cleanup.
# workflow_dispatch runs stop after their own cleanup. A failed
# handoff must never pass silently: it retries, then files an alert
# issue in rustfs/backlog so a stalled chain is visible.
if: ${{ always() && github.event_name == 'repository_dispatch' }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
set -uo pipefail
if [ -z "${{GH_TOKEN:-}}" ]; then
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
exit 1
fi
echo "Dispatching next functional suite: KMS"
gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-kms' \
-F 'client_payload[from_suite]=s3'
DISPATCHED=0
for attempt in 1 2 3; do
if gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-kms' \
-F 'client_payload[from_suite]=s3'; then
echo "dispatched next suite KMS (attempt ${{attempt}})"
DISPATCHED=1
break
fi
echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
sleep "${{attempt}}0"
done
if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
echo "ERROR: functional chain stalled: could not dispatch KMS after 3 attempts" >&2
TITLE="[functional][chain] stalled after s3 (run ${{GITHUB_RUN_ID}})"
BODY_FILE="$(mktemp)"
{
echo "The functional chain could not hand off from **s3** to **KMS** after 3 attempts."
echo ""
echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
echo "- Expected next event: `rustfs-chain-kms`"
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
echo "- Recovery: re-dispatch manually with"
echo " ```"
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-kms'"
echo " ```"
} > "${{BODY_FILE}}"
gh issue create -R rustfs/backlog --title "${{TITLE}}" \
--body-file "${{BODY_FILE}}" --label functional-test \
|| gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
|| echo "could not file the stall alert issue either; check the token" >&2
exit 1
fi
- name: Notify on failure
if: failure()
.github/workflows/rustfs-storage-test.yml
+38 -7
View File
@@ -335,21 +335,52 @@ jobs:
- name: "Continue functional chain (next: Heal)"
# Only chain-triggered runs forward to the next suite; standalone
# workflow_dispatch runs stop after their own cleanup.
# workflow_dispatch runs stop after their own cleanup. A failed
# handoff must never pass silently: it retries, then files an alert
# issue in rustfs/backlog so a stalled chain is visible.
if: ${{ always() && github.event_name == 'repository_dispatch' }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
set -uo pipefail
if [ -z "${{GH_TOKEN:-}}" ]; then
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
exit 1
fi
echo "Dispatching next functional suite: Heal"
gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-heal' \
-F 'client_payload[from_suite]=storage'
DISPATCHED=0
for attempt in 1 2 3; do
if gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-heal' \
-F 'client_payload[from_suite]=storage'; then
echo "dispatched next suite Heal (attempt ${{attempt}})"
DISPATCHED=1
break
fi
echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
sleep "${{attempt}}0"
done
if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
echo "ERROR: functional chain stalled: could not dispatch Heal after 3 attempts" >&2
TITLE="[functional][chain] stalled after storage (run ${{GITHUB_RUN_ID}})"
BODY_FILE="$(mktemp)"
{
echo "The functional chain could not hand off from **storage** to **Heal** after 3 attempts."
echo ""
echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
echo "- Expected next event: `rustfs-chain-heal`"
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
echo "- Recovery: re-dispatch manually with"
echo " ```"
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-heal'"
echo " ```"
} > "${{BODY_FILE}}"
gh issue create -R rustfs/backlog --title "${{TITLE}}" \
--body-file "${{BODY_FILE}}" --label functional-test \
|| gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
|| echo "could not file the stall alert issue either; check the token" >&2
exit 1
fi
- name: Notify on failure
if: failure()
.github/workflows/rustfs-tier-test.yml
+38 -7
View File
@@ -373,21 +373,52 @@ jobs:
- name: "Continue functional chain (next: Storage engine)"
# Only chain-triggered runs forward to the next suite; standalone
# workflow_dispatch runs stop after their own cleanup.
# workflow_dispatch runs stop after their own cleanup. A failed
# handoff must never pass silently: it retries, then files an alert
# issue in rustfs/backlog so a stalled chain is visible.
if: ${{ always() && github.event_name == 'repository_dispatch' }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
set -uo pipefail
if [ -z "${{GH_TOKEN:-}}" ]; then
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
exit 1
fi
echo "Dispatching next functional suite: Storage engine"
gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-storage' \
-F 'client_payload[from_suite]=tier'
DISPATCHED=0
for attempt in 1 2 3; do
if gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-storage' \
-F 'client_payload[from_suite]=tier'; then
echo "dispatched next suite Storage engine (attempt ${{attempt}})"
DISPATCHED=1
break
fi
echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
sleep "${{attempt}}0"
done
if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
echo "ERROR: functional chain stalled: could not dispatch Storage engine after 3 attempts" >&2
TITLE="[functional][chain] stalled after tier (run ${{GITHUB_RUN_ID}})"
BODY_FILE="$(mktemp)"
{
echo "The functional chain could not hand off from **tier** to **Storage engine** after 3 attempts."
echo ""
echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
echo "- Expected next event: `rustfs-chain-storage`"
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
echo "- Recovery: re-dispatch manually with"
echo " ```"
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-storage'"
echo " ```"
} > "${{BODY_FILE}}"
gh issue create -R rustfs/backlog --title "${{TITLE}}" \
--body-file "${{BODY_FILE}}" --label functional-test \
|| gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
|| echo "could not file the stall alert issue either; check the token" >&2
exit 1
fi
- name: Notify on failure
if: failure()
.github/workflows/rustfs-upgrade-test.yml
+38 -7
View File
@@ -388,21 +388,52 @@ jobs:
- name: "Continue functional chain (next: S3 compatibility)"
# Only chain-triggered runs forward to the next suite; standalone
# workflow_dispatch runs stop after their own cleanup.
# workflow_dispatch runs stop after their own cleanup. A failed
# handoff must never pass silently: it retries, then files an alert
# issue in rustfs/backlog so a stalled chain is visible.
if: ${{ always() && github.event_name == 'repository_dispatch' }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: |
set -euo pipefail
if [ -z "${GH_TOKEN:-}" ]; then
set -uo pipefail
if [ -z "${{GH_TOKEN:-}}" ]; then
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
exit 1
fi
echo "Dispatching next functional suite: S3 compatibility"
gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-s3' \
-F 'client_payload[from_suite]=upgrade'
DISPATCHED=0
for attempt in 1 2 3; do
if gh api --method POST repos/rustfs/rustfs/dispatches \
-f event_type='rustfs-chain-s3' \
-F 'client_payload[from_suite]=upgrade'; then
echo "dispatched next suite S3 compatibility (attempt ${{attempt}})"
DISPATCHED=1
break
fi
echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
sleep "${{attempt}}0"
done
if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
echo "ERROR: functional chain stalled: could not dispatch S3 compatibility after 3 attempts" >&2
TITLE="[functional][chain] stalled after upgrade (run ${{GITHUB_RUN_ID}})"
BODY_FILE="$(mktemp)"
{
echo "The functional chain could not hand off from **upgrade** to **S3 compatibility** after 3 attempts."
echo ""
echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
echo "- Expected next event: `rustfs-chain-s3`"
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
echo "- Recovery: re-dispatch manually with"
echo " ```"
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-s3'"
echo " ```"
} > "${{BODY_FILE}}"
gh issue create -R rustfs/backlog --title "${{TITLE}}" \
--body-file "${{BODY_FILE}}" --label functional-test \
|| gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
|| echo "could not file the stall alert issue either; check the token" >&2
exit 1
fi
- name: Notify on failure
if: failure()