name: Coach eval

# B0 (v1.21.3) — the Coach evaluation harness, two layers.
#
#   1. The DETERMINISTIC suite (golden set + red-team battery + the grounding
#      grader) runs ALWAYS. It is free, offline, and the per-change floor — the
#      same suite `pnpm test` runs, pinned here as a standing nightly check.
#   2. The LIVE JUDGE step runs ONLY when the `COACH_EVAL_API_KEY` repository
#      secret is present, and is NON-BLOCKING (`continue-on-error: true`). With
#      the secret absent it no-ops with a clear log line — the workflow stays
#      green. The operator flips the secret on later; no rebuild needed.
#
# Schedule + manual only. This is NOT a required check.

on:
  schedule:
    # Nightly at 04:20 UTC — off the busy hour, after any evening merges.
    - cron: "20 4 * * *"
  workflow_dispatch:

permissions:
  contents: read

jobs:
  coach-eval:
    name: Coach eval — deterministic floor (always) + live judge (opt-in)
    runs-on: ubuntu-latest
    timeout-minutes: 15

    # Map the opt-in secret to a job-level env so the live-judge steps can gate
    # on it. The `secrets` context is NOT available in `if:` expressions — using
    # it there is a workflow-file validation error that surfaces as a 0-second
    # startup failure on every push. `env` is allowed in `if:`, so gate on that.
    env:
      COACH_EVAL_API_KEY: ${{ secrets.COACH_EVAL_API_KEY }}

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10

      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: 22
          cache: pnpm

      - run: pnpm install --frozen-lockfile

      - run: pnpm db:generate

      # Layer 1 — the deterministic floor. Free, offline, ALWAYS runs. A
      # failure here reds the workflow: a regression in the golden set or the
      # red-team battery is a real find.
      - name: Deterministic floor (golden set + red team)
        run: pnpm exec vitest run src/lib/ai/coach/eval/__tests__/

      # Layer 2 — the live judge. Runs ONLY when the secret is present, and is
      # non-blocking either way. With the secret absent the script no-ops with a
      # clear line and exits 0.
      - name: Live judge (opt-in, non-blocking)
        if: ${{ env.COACH_EVAL_API_KEY != '' }}
        continue-on-error: true
        run: pnpm dlx tsx scripts/run-coach-eval.ts

      # When the secret is absent, surface a clear no-op line so the run log
      # explains why the live judge did not execute.
      - name: Live judge skipped (no secret)
        if: ${{ env.COACH_EVAL_API_KEY == '' }}
        run: echo "COACH_EVAL_API_KEY not set — live Coach judge skipped (deterministic floor still gated above)."