# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Warp A/B budget gate for the hotpath series (rustfs/backlog#935 HP-14, item 4).
#
# Two entry points, honestly scoped:
#   * schedule (nightly, on main): post-merge detection — catches a regression
#     within 24h of landing, not before merge.
#   * workflow_dispatch: an explicitly selected trusted ref.
# The dispatch input can run the gate with --allow-regression so a deliberate
# correctness cost (e.g. the #4221 fsync durability fix) is recorded, not
# blocked (rustfs/backlog#935 correction 1).

# DISABLED. This workflow is switched off in the repository's Actions settings
# (state: disabled_manually) and does not run on any trigger, including its cron
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
# reading this file, which has already misled at least one audit — hence this
# banner. Re-enabling is a UI action; anyone doing so should first check that the
# workflow still matches the current CI layout. See rustfs/backlog#1603.
#
name: Performance A/B

on:
  schedule:
    - cron: "0 6 * * *" # 06:00 UTC nightly, against main
  workflow_dispatch:
    inputs:
      duration:
        description: "warp duration per round (short by default to fit the double-build budget)"
        required: false
        default: "12s"
        type: string
      allow_regression:
        description: "Pass the gate despite a FAIL (deliberate tradeoff)"
        required: false
        default: false
        type: boolean
  push:
    # Every main commit pre-builds and caches its release binary (perf-3) so the
    # nightly A/B restores a ready baseline instead of paying the double build.
    branches: [main]

permissions:
  contents: read

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1

jobs:
  # perf-3: on every push to main, build the release binary once and cache it
  # keyed by commit SHA (rustfs-baseline-<sha>). The warp-ab measurements
  # restore this instead of paying the ~32min-per-side source
  # build. That double build is what pushed the expanded 24-cell nightly past its
  # ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental
  # builds off the shared cargo cache keep each push cheap, and building on the
  # same sm-standard-2 runner the A/B measures on guarantees the cached binary is
  # ABI-identical. Do NOT source this from build.yml's per-merge artifact: those
  # are cancelled ~7/8 of the time and are not a reliable baseline.
  build-baseline-cache:
    name: Build + cache baseline binary
    if: github.event_name == 'push'
    runs-on: sm-standard-2
    # Latest-wins: consumers only ever restore the binary for the *current*
    # origin/main tip, so when pushes land faster than the ~65min build, a
    # superseded build's output is dead weight — cancel it instead of stacking
    # hour-long jobs on the shared runner pool. A skipped intermediate SHA at
    # most costs one same-commit self-heal in the A/B job.
    concurrency:
      group: perf-baseline-build-main
      cancel-in-progress: true
    # #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a
    # single release build past 60min on this runner — every cache build on
    # 2026-07-15 died on the old 60min ceiling ("exceeded the maximum execution
    # time of 1h0m0s") and the cache never populated. The measured binary must
    # keep the production profile, so the budget absorbs the build instead.
    timeout-minutes: 100
    steps:
      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          persist-credentials: false

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
          cache-save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Build release rustfs
        run: cargo build --release --bin rustfs

      - name: Stage binary for cache
        run: |
          set -euo pipefail
          mkdir -p baseline-bin
          cp target/release/rustfs baseline-bin/rustfs

      - name: Cache baseline binary by SHA
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
        with:
          path: baseline-bin/rustfs
          key: rustfs-baseline-${{ github.sha }}

  warp-ab:
    name: Warp A/B budget gate
    # Always run on schedule / manual dispatch. Never on push — that event only
    # feeds build-baseline-cache above.
    if: >-
      github.event_name == 'schedule' ||
      github.event_name == 'workflow_dispatch'
    runs-on: sm-standard-2
    # With perf-3's cached baseline binary the common (cache-hit) nightly is
    # measurement-only and finishes well under 50min. This ceiling stays
    # generous only to absorb the same-commit cache-miss self-heal (~65min
    # single build with the post-#4806 LTO profile + measurement). A timeout
    # surfaces via the alert-on-failure job (it fires on cancelled/timed-out,
    # not just failure). perf-6 recalibrates the budget once the noise study
    # lands.
    timeout-minutes: 120
    steps:
      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          persist-credentials: false
          fetch-depth: 0 # baseline is built from origin/main

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
          cache-save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Install warp
        run: |
          set -euo pipefail
          WARP_VERSION="1.0.0"
          curl -fsSL "https://github.com/minio/warp/releases/download/v${WARP_VERSION}/warp_Linux_x86_64.tar.gz" \
            | sudo tar -xz -C /usr/local/bin warp
          warp --version

      - name: Decide exemption
        id: exempt
        env:
          INPUT_ALLOW_REGRESSION: ${{ github.event.inputs.allow_regression }}
        run: |
          allow="false"
          if [[ "$INPUT_ALLOW_REGRESSION" == "true" ]]; then
            allow="true"
          fi
          echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"

      # perf-3: resolve the commits so the cache can be keyed by SHA. The
      # baseline is origin/main; the candidate is the checked-out ref. On the
      # nightly (checkout == main) they are the same commit, so one cached binary
      # serves both phases and the run does zero source builds.
      - name: Resolve baseline / candidate commits
        id: commits
        run: |
          set -euo pipefail
          baseline_sha="$(git rev-parse origin/main)"
          candidate_sha="$(git rev-parse HEAD)"
          echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
          echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
          echo "baseline commit: $baseline_sha"
          echo "candidate commit: $candidate_sha"

      # Exact-key restore of the baseline binary built by build-baseline-cache
      # when origin/main last landed. A miss (binary evicted or not built yet)
      # leaves cache-hit unset and the rig falls back to a source build.
      - name: Restore cached baseline binary
        id: baseline_cache
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
        with:
          path: baseline-bin/rustfs
          key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}

      # Self-heal: on a nightly/dispatch run where the candidate commit IS the
      # baseline commit, a cache miss would make the rig build the same commit
      # twice (~65min per side with the post-#4806 LTO profile — no job budget
      # fits that). Build it once here, reuse it for both phases, and save it
      # back to the cache so the next run hits.
      - name: Build baseline on cache miss (same-commit self-heal)
        id: selfheal
        if: >-
          steps.baseline_cache.outputs.cache-hit != 'true' &&
          steps.commits.outputs.baseline_sha == steps.commits.outputs.candidate_sha
        run: |
          set -euo pipefail
          cargo build --release --bin rustfs
          mkdir -p baseline-bin
          cp target/release/rustfs baseline-bin/rustfs
          echo "built=true" >> "$GITHUB_OUTPUT"

      - name: Build baseline on cache miss (different candidate)
        id: baseline_build
        if: >-
          steps.baseline_cache.outputs.cache-hit != 'true' &&
          steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
        run: |
          set -euo pipefail
          baseline_root="$RUNNER_TEMP/rustfs-baseline-${{ github.run_id }}"
          baseline_target="$RUNNER_TEMP/rustfs-baseline-target-${{ github.run_id }}"
          git worktree add --detach "$baseline_root" "${{ steps.commits.outputs.baseline_sha }}"
          cargo build --release --manifest-path "$baseline_root/Cargo.toml" --bin rustfs --target-dir "$baseline_target"
          mkdir -p baseline-bin
          cp "$baseline_target/release/rustfs" baseline-bin/rustfs
          git worktree remove --force "$baseline_root"
          echo "built=true" >> "$GITHUB_OUTPUT"

      - name: Build candidate binary
        id: candidate_build
        if: steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
        run: |
          set -euo pipefail
          cargo build --release --bin rustfs
          mkdir -p candidate-bin
          cp target/release/rustfs candidate-bin/rustfs
          echo "built=true" >> "$GITHUB_OUTPUT"

      - name: Save self-healed baseline to cache
        if: steps.selfheal.outputs.built == 'true' || steps.baseline_build.outputs.built == 'true'
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
        with:
          path: baseline-bin/rustfs
          key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}

      - name: Run warp A/B and gate
        id: ab
        env:
          INPUT_DURATION: ${{ github.event.inputs.duration }}
        run: |
          set -euo pipefail
          # The formal runner executes A1 baseline -> B1 candidate -> B2 candidate
          # -> A2 baseline for each workload and drive-sync cell. It requires three
          # rounds per leg to emit tail latency and error-rate evidence.
          # --health-timeout 180 outlasts the server's own 120s startup-readiness
          # budget, which the rig's previous 60s health poll undershot (the first
          # two nightly failures). perf-6 recalibrates these once the noise study
          # lands.
          duration="${INPUT_DURATION:-12s}"
          baseline_sha="${{ steps.commits.outputs.baseline_sha }}"
          candidate_sha="${{ steps.commits.outputs.candidate_sha }}"
          baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}"
          selfheal_built="${{ steps.selfheal.outputs.built }}"
          baseline_built="${{ steps.baseline_build.outputs.built }}"
          candidate_built="${{ steps.candidate_build.outputs.built }}"

          args=(--duration "$duration" --rounds 3 --cooldown 5 --health-timeout 180 --baseline-revision "$baseline_sha" --candidate-revision "$candidate_sha")

          if [[ "$baseline_hit" == "true" || "$selfheal_built" == "true" || "$baseline_built" == "true" ]]; then
            chmod +x baseline-bin/rustfs
            base_bin="$PWD/baseline-bin/rustfs"
            args+=(--baseline-bin "$base_bin")
            if [[ "$baseline_hit" == "true" ]]; then
              base_src="actions-cache (rustfs-baseline-$baseline_sha)"
            elif [[ "$selfheal_built" == "true" ]]; then
              base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
            else
              base_src="isolated origin/main source build (saved as rustfs-baseline-$baseline_sha)"
            fi
            if [[ "$candidate_sha" == "$baseline_sha" ]]; then
              # Nightly on main: the candidate is the same commit as the baseline,
              # so reuse the one binary for both phases and skip all builds.
              args+=(--candidate-bin "$base_bin")
              cand_src="same binary as baseline (same commit)"
            elif [[ "$candidate_built" == "true" ]]; then
              chmod +x candidate-bin/rustfs
              args+=(--candidate-bin "$PWD/candidate-bin/rustfs")
              cand_src="source build of the checked-out ref"
            else
              echo "::error::candidate binary was not built" >&2
              exit 2
            fi
          else
            echo "::error::baseline binary was not restored or built" >&2
            exit 2
          fi

          echo "baseline binary: $base_src"
          echo "candidate binary: $cand_src"

          if [[ "${{ steps.exempt.outputs.allow_regression }}" == "true" ]]; then
            args+=(--allow-regression --exemption-reason "workflow dispatch override")
          fi
          # Do not let a gate FAIL abort the job here; capture status and surface
          # it after the step summary is written.
          set +e
          bash scripts/run_hotpath_warp_abba.sh "${args[@]}"
          echo "status=$?" >> "$GITHUB_OUTPUT"
          set -e
          # Locate the newest run dir + gate.md for the summary/comment/artifact
          # steps. On a startup failure there is no gate.md, but the run dir still
          # holds server-logs/ for diagnosis.
          # Run dirs are UTC-timestamp names (no special chars); ls is safe here.
          # shellcheck disable=SC2012
          run_dir="$(ls -td target/hotpath-abba/*/ 2>/dev/null | head -n1 || true)"
          echo "run_dir=${run_dir%/}" >> "$GITHUB_OUTPUT"
          # shellcheck disable=SC2012
          gate_md="$(ls -t target/hotpath-abba/*/candidate_gate.md 2>/dev/null | head -n1 || true)"
          echo "gate_md=$gate_md" >> "$GITHUB_OUTPUT"

      - name: Upload A/B results
        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
          name: hotpath-warp-ab-${{ github.run_number }}
          # Includes per-cell median_summary.csv / baseline_compare.csv, both gates,
          # and server-logs/ (rustfs.log + startup env per phase) so a failed run
          # is diagnosable. Short retention: this is churny nightly debug data.
          path: target/hotpath-abba/
          if-no-files-found: warn
          retention-days: 14

      - name: Write gate summary
        if: always()
        run: |
          set -euo pipefail
          status="${{ steps.ab.outputs.status }}"
          gate_md="${{ steps.ab.outputs.gate_md }}"
          run_dir="${{ steps.ab.outputs.run_dir }}"
          {
            echo "## Hotpath warp A/B — run ${{ github.run_number }}"
            echo
            if [[ "$status" == "0" ]]; then
              echo "Rig/gate exit: \`0\` (pass or warn)."
            else
              echo "Rig/gate exit: \`${status:-unknown}\` — **FAILED**."
            fi
            echo
            if [[ -n "$gate_md" && -f "$gate_md" ]]; then
              cat "$gate_md"
            else
              echo "No \`gate.md\` produced — the rig failed **before** the gate"
              echo "(most likely server startup / health). Failing phase(s) below;"
              echo "full logs in the \`hotpath-warp-ab-${{ github.run_number }}\` artifact."
              if [[ -n "$run_dir" && -d "$run_dir/server-logs" ]]; then
                for f in "$run_dir"/server-logs/*.log; do
                  [[ -f "$f" ]] || continue
                  echo
                  echo "<details><summary>$(basename "$f")</summary>"
                  echo
                  echo '```'
                  tail -n 30 "$f"
                  echo '```'
                  echo "</details>"
                done
              fi
            fi
          } >> "$GITHUB_STEP_SUMMARY"

      # Scheduled failure alerting is handled by the alert-on-failure job below
      # (perf-2 consuming ci-8's schedule-failure-issue composite action).

      - name: Enforce gate
        if: always()
        run: |
          status="${{ steps.ab.outputs.status }}"
          if [[ "$status" != "0" ]]; then
            echo "::error::warp A/B budget gate failed (exit $status). See the step summary / gate.md artifact." >&2
            exit "$status"
          fi
          echo "warp A/B budget gate passed."

  alert-on-failure:
    name: Alert on scheduled failure
    needs: [warp-ab]
    # `always()` is required: without it this job is skipped when a needed
    # job fails. Alerts only for scheduled (nightly) runs (backlog#1149
    # ci-8); manual dispatch failures are already watched by a human.
    # `cancelled` is included alongside `failure` on purpose: a job that hits
    # timeout-minutes ends as `cancelled`, and the 2026-07-11..07-14 nightly
    # timeouts went silent precisely because the guard was failure-only. The
    # composite action already reports cancelled/timed-out jobs in the issue
    # body.
    if: >-
      always() && github.event_name == 'schedule' &&
      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          persist-credentials: false
      - name: Open or update failure-tracking issue
        uses: ./.github/actions/schedule-failure-issue
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
