From 75f8a648dc4427178a213760c29dfe9e2c8fff52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 15 Nov 2026 10:45:21 +0100 Subject: [PATCH 1/7] fix(jobs): stop counting an unusable provider as a failing briefing run The nightly briefing pass settled an account as failed whenever its generation failed, including when every provider of its chain was benched for background work and no request was sent, and when the provider refused the account's own credential. A key revoked at its issuer stays saved and is refused every night, so a single-account instance read as all-failed night after night and the repeated-failure alert reported a broken job. - A chain that sent nothing settles as skipped (provider_benched), with no 45-minute retry. - A refused credential of the account's own settles as auth_failed, which the run reports as partial but does not fail over; a refusal of the operator's shared key still fails the run. - The generator's failed outcome names the provider that failed first, whether its credential was refused and whether anything was sent. - The run's facts carry failure_cause and failure_provider. The admin Background jobs card listed only pg-boss failures, while the alert also counts runs that completed with every account failed, so an operator sent to the card read that nothing had failed. The card now lists those runs, and the card and the alert describe them in words with the cause and provider instead of quoting the run's JSON. The warm tally counted a status card that fell back to its deterministic text as a warmed assessment; it now counts model-written notes only. Refs #1384 --- CHANGELOG.md | 15 ++ messages/de.json | 2 +- messages/en.json | 2 +- messages/es.json | 2 +- messages/fr.json | 2 +- messages/it.json | 2 +- messages/ko.json | 2 +- messages/nl.json | 2 +- messages/pl.json | 2 +- messages/pt-BR.json | 2 +- messages/ru.json | 2 +- src/__tests__/baselines/test-as-never.json | 2 +- .../baselines/version-stamp-comments.json | 6 +- .../error-level-discipline-guard.test.ts | 5 + .../comprehensive-warm-timeout.test.ts | 55 ++++++++ .../insights/__tests__/status-batch.test.ts | 12 ++ src/lib/insights/chain-failure-facts.ts | 37 +++++ src/lib/insights/comprehensive-generate.ts | 22 +-- src/lib/insights/status-batch.ts | 45 +++++- .../__tests__/insight-pregenerate.test.ts | 130 ++++++++++++++++-- src/lib/jobs/__tests__/job-outcome.test.ts | 33 +++++ src/lib/jobs/__tests__/job-run-report.test.ts | 56 ++++++-- .../jobs/__tests__/system-job-alert.test.ts | 28 ++++ src/lib/jobs/insight-pregenerate-failure.ts | 93 +++++++++++++ src/lib/jobs/insight-pregenerate.ts | 56 ++++---- src/lib/jobs/job-failures.ts | 56 +++++++- src/lib/jobs/job-outcome.ts | 21 +++ src/lib/jobs/job-run-report.ts | 30 +++- src/lib/jobs/reminder/insights-handlers.ts | 8 +- .../__golden__/status-parity/batch.json | 3 + .../background-switch-freshness.test.ts | 1 + tests/integration/job-failures.test.ts | 35 +++++ 32 files changed, 675 insertions(+), 94 deletions(-) create mode 100644 src/lib/insights/chain-failure-facts.ts create mode 100644 src/lib/jobs/insight-pregenerate-failure.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d4156079..de935e7cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,21 @@ default was rejected as in the future, and onset dates could display as the wrong calendar day in non-UTC time zones. Saving now anchors to the current time when before local noon and preserves stored timestamps on edit. +- **No more nightly "job still failing" alert over a provider that cannot + answer.** When the only account on an instance still had an AI provider + saved whose key the provider no longer accepted (removing the key from a + Local or OpenAI-compatible provider keeps its address, so it is still + called), or whose provider HealthLog had paused after repeated errors, the + nightly briefing pass counted that account as a failure, and every night + the admins were told the job was broken. A key the provider refuses is now + that account's matter: the run notes it but does not fail over it, unless + it is the instance's own shared key. An account whose providers are all + paused is skipped, since nothing was sent. The Background jobs card on the + admin status page now lists a run that finished with every account + failing, the same runs the alert counts, and the alert and the card say in + words what failed and which provider, instead of quoting the run's + counters. Those counters also stop counting a fallback text as a warmed + assessment. (#1384) ### Security diff --git a/messages/de.json b/messages/de.json index 66fa04236..06ca76108 100644 --- a/messages/de.json +++ b/messages/de.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Hintergrundjobs", - "description": "Queues, deren Jobs nach allen Wiederholungen aufgegeben haben.", + "description": "Queues, deren Jobs nach allen Wiederholungen aufgaben oder deren Lauf bei jedem versuchten Konto scheiterte.", "none": "In den letzten {hours} Stunden ist kein Job fehlgeschlagen.", "unavailable": "Keine Job-Queue lesbar. Dieses Deployment läuft ohne Hintergrund-Worker.", "countOne": "{count} Fehlschlag", diff --git a/messages/en.json b/messages/en.json index 765ba808b..de6c5d6a6 100644 --- a/messages/en.json +++ b/messages/en.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Background jobs", - "description": "Queues whose jobs gave up after exhausting their retries.", + "description": "Queues whose jobs gave up after their retries, or whose run ended with every account it tried failing.", "none": "No job has failed in the last {hours} hours.", "unavailable": "No job queue to read. This deployment runs without the background worker.", "countOne": "{count} failure", diff --git a/messages/es.json b/messages/es.json index e5f751113..41be21086 100644 --- a/messages/es.json +++ b/messages/es.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Trabajos en segundo plano", - "description": "Colas cuyos trabajos se rindieron tras agotar sus reintentos.", + "description": "Colas cuyos trabajos se rindieron tras sus reintentos o cuya ejecución falló en cada cuenta intentada.", "none": "Ningún trabajo ha fallado en las últimas {hours} horas.", "unavailable": "No hay ninguna cola que consultar: este despliegue funciona sin el worker en segundo plano.", "countOne": "{count} fallo", diff --git a/messages/fr.json b/messages/fr.json index b5e9d4ccf..84226f8ee 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Tâches en arrière-plan", - "description": "Files dont les tâches ont abandonné après avoir épuisé leurs tentatives.", + "description": "Files dont les tâches ont abandonné après leurs tentatives, ou dont l’exécution a échoué pour chaque compte.", "none": "Aucune tâche n’a échoué au cours des {hours} dernières heures.", "unavailable": "Aucune file à consulter : ce déploiement fonctionne sans le worker en arrière-plan.", "countOne": "{count} échec", diff --git a/messages/it.json b/messages/it.json index 8e5512ee2..506eb23f8 100644 --- a/messages/it.json +++ b/messages/it.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Processi in background", - "description": "Code i cui processi hanno rinunciato dopo aver esaurito i tentativi.", + "description": "Code i cui processi hanno rinunciato dopo i tentativi o la cui esecuzione è fallita per ogni account.", "none": "Nessun processo è fallito nelle ultime {hours} ore.", "unavailable": "Nessuna coda da leggere: questo deployment funziona senza il worker in background.", "countOne": "{count} errore", diff --git a/messages/ko.json b/messages/ko.json index 6ce331f81..89fe1732c 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -8743,7 +8743,7 @@ }, "failingJobs": { "title": "백그라운드 작업", - "description": "재시도를 다 쓰고 포기한 작업이 있는 큐예요.", + "description": "재시도를 다 쓰고 포기했거나, 실행에서 시도한 모든 계정이 실패한 큐예요.", "none": "최근 {hours}시간 동안 실패한 작업이 없어요.", "unavailable": "읽을 작업 큐가 없어요. 이 배포본은 백그라운드 워커 없이 돌고 있어요.", "countOne": "실패 {count}건", diff --git a/messages/nl.json b/messages/nl.json index 657a2e61e..ddf85f494 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Achtergrondtaken", - "description": "Wachtrijen waarvan taken het hebben opgegeven nadat alle nieuwe pogingen waren verbruikt.", + "description": "Wachtrijen waarvan taken na alle pogingen opgaven, of waarvan een run voor elk geprobeerd account mislukte.", "none": "Er is de afgelopen {hours} uur geen taak mislukt.", "unavailable": "Geen takenwachtrij om te lezen. Deze installatie draait zonder de achtergrondworker.", "countOne": "{count} fout", diff --git a/messages/pl.json b/messages/pl.json index 5ced39f79..aa2b039d2 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Zadania w tle", - "description": "Kolejki, których zadania poddały się po wyczerpaniu ponowień.", + "description": "Kolejki, których zadania poddały się po ponowieniach albo których przebieg zawiódł dla każdego konta.", "none": "W ciągu ostatnich {hours} godzin żadne zadanie nie zawiodło.", "unavailable": "Brak kolejki zadań do odczytu: to wdrożenie działa bez workera w tle.", "countOne": "{count} niepowodzenie", diff --git a/messages/pt-BR.json b/messages/pt-BR.json index 01ab83ec1..351ec8538 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Tarefas em segundo plano", - "description": "Filas cujas tarefas desistiram depois de esgotar as tentativas.", + "description": "Filas cujas tarefas desistiram após as tentativas, ou cuja execução falhou em todas as contas tentadas.", "none": "Nenhuma tarefa falhou nas últimas {hours} horas.", "unavailable": "Nenhuma fila de tarefas para ler. Esta implantação roda sem o worker em segundo plano.", "countOne": "{count} falha", diff --git a/messages/ru.json b/messages/ru.json index c8ba334ab..acd103c36 100644 --- a/messages/ru.json +++ b/messages/ru.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Фоновые задания", - "description": "Очереди, задания которых сдались после исчерпания повторных попыток.", + "description": "Очереди, задания которых сдались после всех попыток или чей запуск не удался ни для одного аккаунта.", "none": "За последние {hours} часов ни одно задание не завершилось неудачей.", "unavailable": "Нет очереди заданий для чтения. Это развёртывание работает без фонового воркера.", "countOne": "{count} неудача", diff --git a/src/__tests__/baselines/test-as-never.json b/src/__tests__/baselines/test-as-never.json index 25d657b68..cddd240c4 100644 --- a/src/__tests__/baselines/test-as-never.json +++ b/src/__tests__/baselines/test-as-never.json @@ -435,7 +435,7 @@ "lib/jobs/__tests__/document-summary-reaper.test.ts": 1, "lib/jobs/__tests__/document-summary.test.ts": 19, "lib/jobs/__tests__/geo-backfill.test.ts": 1, - "lib/jobs/__tests__/insight-pregenerate.test.ts": 45, + "lib/jobs/__tests__/insight-pregenerate.test.ts": 43, "lib/jobs/__tests__/insight-status-generate.test.ts": 2, "lib/jobs/__tests__/measurement-maintenance-schedule.test.ts": 2, "lib/jobs/__tests__/measurement-reminder-byhour.test.ts": 2, diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index 6bbcd8aee..bbc180535 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", "count": 2863, - "total": 9122, + "total": 9118, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -2134,7 +2134,7 @@ "lib/insights/chart-tokens.ts": 24, "lib/insights/coach-launch-context.tsx": 10, "lib/insights/comprehensive-aggregator.ts": 27, - "lib/insights/comprehensive-generate.ts": 45, + "lib/insights/comprehensive-generate.ts": 44, "lib/insights/correlation-channel-series.ts": 16, "lib/insights/correlation-discovery.ts": 30, "lib/insights/correlation-series-builders.ts": 1, @@ -2299,7 +2299,7 @@ "lib/jobs/google-health-oauth-state-cleanup.ts": 1, "lib/jobs/google-health-sleep-repair.ts": 1, "lib/jobs/insight-pregenerate-shared.ts": 1, - "lib/jobs/insight-pregenerate.ts": 36, + "lib/jobs/insight-pregenerate.ts": 33, "lib/jobs/insight-status-generate-shared.ts": 3, "lib/jobs/insight-status-generate.ts": 3, "lib/jobs/intake-auto-skip.ts": 8, diff --git a/src/__tests__/error-level-discipline-guard.test.ts b/src/__tests__/error-level-discipline-guard.test.ts index c3432c8be..5376df965 100644 --- a/src/__tests__/error-level-discipline-guard.test.ts +++ b/src/__tests__/error-level-discipline-guard.test.ts @@ -168,6 +168,11 @@ const ALLOWED: Readonly> = { reason: "No pgboss schema on a web-only deployment; null means no queue to ask, and the evaluation then judges nothing.", }, + "lib/jobs/job-failures.ts::explainAllFailedRun": { + count: 1, + reason: + "A pg-boss output that does not parse as a run's outcome is not an all-failed run; presentError then shows the raw text as it always did.", + }, "lib/jobs/job-failures.ts::readActiveJobsStartedBefore": { count: 1, reason: diff --git a/src/lib/insights/__tests__/comprehensive-warm-timeout.test.ts b/src/lib/insights/__tests__/comprehensive-warm-timeout.test.ts index 5468a5de7..18bb396c5 100644 --- a/src/lib/insights/__tests__/comprehensive-warm-timeout.test.ts +++ b/src/lib/insights/__tests__/comprehensive-warm-timeout.test.ts @@ -209,4 +209,59 @@ describe("comprehensive warm-path timeout + failure observability", () => { (failAnnotation![0] as { meta?: { reason?: string } }).meta?.reason, ).toBe("all-providers-failed"); }); + + // #1384 — the nightly pass needs to know which configured provider failed, + // whose credential a refusal was, and whether anything was sent at all. + it("names the provider that failed first and whether its credential was refused", async () => { + runRawCompletionWithFallback.mockRejectedValue( + Object.assign(new AllProvidersFailedError([]), { + attempts: [ + { + providerType: "openai-compatible", + attempt: 1, + failureReason: "401", + httpStatus: 401, + }, + ], + primaryCredentialExpired: true, + sentNothing: false, + }), + ); + + const outcome = await generateComprehensiveInsight("u1", { locale: "de" }); + + expect(outcome).toEqual({ + status: "failed", + reason: "all-providers-failed", + credentialExpired: true, + provider: "openai-compatible", + }); + }); + + it("says so when every provider was benched and nothing was sent (#1279)", async () => { + runRawCompletionWithFallback.mockRejectedValue( + Object.assign(new AllProvidersFailedError([]), { + attempts: [ + { + providerType: "local", + attempt: 1, + failureReason: "benched-for-background-work", + httpStatus: null, + failureKind: "paused", + }, + ], + primaryCredentialExpired: false, + sentNothing: true, + }), + ); + + const outcome = await generateComprehensiveInsight("u1", { locale: "de" }); + + expect(outcome).toEqual({ + status: "failed", + reason: "all-providers-failed", + provider: "local", + sentNothing: true, + }); + }); }); diff --git a/src/lib/insights/__tests__/status-batch.test.ts b/src/lib/insights/__tests__/status-batch.test.ts index 6aaf2dff5..e9c26b9e0 100644 --- a/src/lib/insights/__tests__/status-batch.test.ts +++ b/src/lib/insights/__tests__/status-batch.test.ts @@ -134,6 +134,7 @@ describe("generateStatusBatchForUser — the statusText capability comes first", served: 0, batched: 0, fellBack: 0, + fellBackAnswered: 0, batchCallMade: false, }); }); @@ -340,6 +341,15 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => { vi.mocked(runStatusCompletion).mockResolvedValue({ kind: "error", } as never); + // #1384 — one card's own call came back with a model note; the other six + // served their fallback. A nightly warm over a provider that only failed + // reported `assessments_warmed: 6`, because every fallback counted. + vi.mocked(runPreparedStatusCard).mockResolvedValueOnce({ + hasProvider: true, + text: "note", + cached: false, + updatedAt: "t", + }); const result = await generateStatusBatchForUser("u1", { locale: "en" }); @@ -347,6 +357,7 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => { for (const f of finalizers) expect(f).not.toHaveBeenCalled(); expect(runPreparedStatusCard).toHaveBeenCalledTimes(7); expect(result.fellBack).toBe(7); + expect(result.fellBackAnswered).toBe(1); expect(result.batched).toBe(0); }); @@ -411,6 +422,7 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => { for (const f of finalizers) expect(f).not.toHaveBeenCalled(); expect(runPreparedStatusCard).not.toHaveBeenCalled(); expect(result.fellBack).toBe(7); + expect(result.fellBackAnswered).toBe(0); expect(result.batchCallMade).toBe(false); }); }); diff --git a/src/lib/insights/chain-failure-facts.ts b/src/lib/insights/chain-failure-facts.ts new file mode 100644 index 000000000..1ac5a0bfb --- /dev/null +++ b/src/lib/insights/chain-failure-facts.ts @@ -0,0 +1,37 @@ +/** + * What a failed briefing generation knows about its provider chain beyond + * the reason (#1384): which configured provider failed first, whether that + * provider refused the credential, and whether anything was sent at all. + * + * The nightly pass needs all three to settle an account honestly. A refusal + * of the account's own credential is the account's matter, a refusal of the + * operator's shared key is the operator's, and a chain whose every entry the + * provider health ledger benched for background work (#1279) sent no request, + * so the account was not attempted. Without them every such account read as + * a failure, night after night. + */ +import { AllProvidersFailedError } from "@/lib/ai/provider-runner"; + +export interface ChainFailureFacts { + /** + * The chain's first provider refused the credential (a dead Codex + * sign-in, a revoked key). No retry can succeed until someone reconnects + * or replaces it. + */ + credentialExpired?: boolean; + /** The chain entry that failed first, e.g. `openai-compatible`. */ + provider?: string; + /** Every entry was benched for background work; no request left. */ + sentNothing?: boolean; +} + +/** The facts of a thrown chain failure; empty for any other error. */ +export function chainFailureFacts(error: unknown): ChainFailureFacts { + if (!(error instanceof AllProvidersFailedError)) return {}; + const provider = error.attempts?.[0]?.providerType; + return { + ...(error.primaryCredentialExpired ? { credentialExpired: true } : {}), + ...(provider ? { provider } : {}), + ...(error.sentNothing ? { sentNothing: true } : {}), + }; +} diff --git a/src/lib/insights/comprehensive-generate.ts b/src/lib/insights/comprehensive-generate.ts index 5d1f6a0f1..3911d54eb 100644 --- a/src/lib/insights/comprehensive-generate.ts +++ b/src/lib/insights/comprehensive-generate.ts @@ -93,6 +93,10 @@ import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; import { insightResultSchema, type InsightResult } from "@/lib/ai/types"; import { resolveProviderChain } from "@/lib/ai/provider"; import { AllProvidersFailedError } from "@/lib/ai/provider-runner"; +import { + chainFailureFacts, + type ChainFailureFacts, +} from "@/lib/insights/chain-failure-facts"; import { BriefingBudgetExceededError, runBriefingCompletion, @@ -178,17 +182,7 @@ export type GenerateOutcome = | "scope-changed" | "profile-scope-changed"; } - | { - status: "failed"; - reason: string; - /** - * v1.42 — the chain's first provider refused the credential (a dead - * Codex sign-in, a revoked key). The nightly pass reports the account - * as `auth_failed` and does not queue a retry that cannot succeed until - * the person reconnects. - */ - credentialExpired?: boolean; - }; + | ({ status: "failed"; reason: string } & ChainFailureFacts); /** Higher, seedless temperature for the daily-briefing phrasing re-roll. */ const BRIEFING_REROLL_TEMPERATURE = 0.6; @@ -1290,11 +1284,7 @@ export async function generateComprehensiveInsight( ? (e.attempts?.[0]?.failureKind ?? null) : providerFailureKind(e).failure, }); - const credentialExpired = - e instanceof AllProvidersFailedError && e.primaryCredentialExpired; - return credentialExpired - ? { status: "failed", reason, credentialExpired } - : { status: "failed", reason }; + return { status: "failed", reason, ...chainFailureFacts(e) }; } // Anthropic + local have no native JSON mode, so a ```json-fenced or diff --git a/src/lib/insights/status-batch.ts b/src/lib/insights/status-batch.ts index a5a9a27e6..48aa11f8c 100644 --- a/src/lib/insights/status-batch.ts +++ b/src/lib/insights/status-batch.ts @@ -37,6 +37,7 @@ import { runPreparedStatusCard, type PendingStatusCard, type PreparedStatusCard, + type StatusCardResult, } from "@/lib/insights/status-card-generation"; import { runStatusCompletion } from "@/lib/insights/status-provider"; import { @@ -77,10 +78,27 @@ export interface StatusBatchResult { /** Metrics that fell back to a single-card call (omitted by the batch, or * the batch failed). */ fellBack: number; + /** + * Of `fellBack`, the metrics whose own single-card call came back with a + * model-written assessment. The rest served their deterministic fallback: + * no provider, a failed or cooling-down provider, or a screened reply. + * `batched + fellBackAnswered` is what a warm pass may call warmed (#1384); + * `fellBack` alone counted a night of six failed calls as six assessments. + */ + fellBackAnswered: number; /** True when the one batched provider call was issued. */ batchCallMade: boolean; } +/** + * Whether a single-card run ended in a model-written assessment: the persisted + * shape is the only one with a provider and no cache flag. The no-key, the + * timeout / error stub and the screened fallback all carry `cached: true`. + */ +function modelAnswered(result: StatusCardResult): boolean { + return result.hasProvider === true && result.cached === false; +} + /** Pull `perMetric` out of a (possibly fenced) model completion. */ function parsePerMetric(content: string): Record | null { const tryParse = (raw: string): Record | null => { @@ -119,6 +137,7 @@ export async function generateStatusBatchForUser( served: 0, batched: 0, fellBack: 0, + fellBackAnswered: 0, batchCallMade: false, }; @@ -171,11 +190,19 @@ export async function generateStatusBatchForUser( // One metric still needing the LLM is not worth a batch envelope — run it // through its own single-card path (identical persisted result). if (pending.length === 1) { - await runPreparedStatusCard(pending[0]); + if (modelAnswered(await runPreparedStatusCard(pending[0]))) { + result.fellBackAnswered++; + } result.fellBack++; annotate({ action: { name: "insights.status.batch" }, - meta: { served: result.served, batched: 0, fell_back: 1, single: true }, + meta: { + served: result.served, + batched: 0, + fell_back: 1, + fell_back_answered: result.fellBackAnswered, + single: true, + }, }); return result; } @@ -272,14 +299,19 @@ export async function generateStatusBatchForUser( // Batch missed this metric (omitted key, or timeout/error/invalid-json): // fall back to the single-card path. On a batch timeout/error this writes // the per-card negative stub via the card's own `timeout`. - await runPreparedStatusCard(card); - return "fellBack" as const; + return modelAnswered(await runPreparedStatusCard(card)) + ? ("fellBackAnswered" as const) + : ("fellBack" as const); }); const outcomes = await Promise.all(fanOuts); for (const o of outcomes) { - if (o === "batched") result.batched++; - else result.fellBack++; + if (o === "batched") { + result.batched++; + continue; + } + result.fellBack++; + if (o === "fellBackAnswered") result.fellBackAnswered++; } annotate({ @@ -288,6 +320,7 @@ export async function generateStatusBatchForUser( served: result.served, batched: result.batched, fell_back: result.fellBack, + fell_back_answered: result.fellBackAnswered, pending: pending.length, }, }); diff --git a/src/lib/jobs/__tests__/insight-pregenerate.test.ts b/src/lib/jobs/__tests__/insight-pregenerate.test.ts index 80e182903..f889746eb 100644 --- a/src/lib/jobs/__tests__/insight-pregenerate.test.ts +++ b/src/lib/jobs/__tests__/insight-pregenerate.test.ts @@ -103,6 +103,7 @@ vi.mock("@/lib/insights/status-batch", () => ({ served: 0, batched: 0, fellBack: 0, + fellBackAnswered: 0, batchCallMade: false, })), })); @@ -132,6 +133,7 @@ import { INSIGHT_PREGENERATE_EXPIRE_SECONDS, } from "../insight-pregenerate-shared"; import { handleInsightPregenerateJob } from "../reminder/insights-handlers"; +import { judgeJobRun, type reportJobRun } from "../job-run-report"; import { insightRetryOptions } from "../reminder/registrar-shared"; import { groupOf, @@ -1556,15 +1558,9 @@ describe("runInsightPregenerate — warm budget honours the response timeout (v1 // withheld (screened) briefing and a dead sign-in queue no provider retry, // and a time-budget cut-off reports the unreached accounts as deferred. describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => { - type Reported = { - queue: string; - runId: string; - candidates: Array<{ key: string; outcome: string; cause?: string }>; - }; - function harness(users: Array<{ id: string; locale: string }>) { const { prisma } = makePrisma(users); - const reportRun = vi.fn(async (input: Reported) => ({ + const reportRun = vi.fn(async (input) => ({ total: input.candidates.length, attempted: 0, failed: 0, @@ -1606,7 +1602,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => { statusGenerators: h.statusGenerators, warmGenericMetrics: h.warmGenericMetrics, enqueueRetry: h.enqueueRetry, - reportRun: h.reportRun as never, + reportRun: h.reportRun, }); expect(h.reportRun).toHaveBeenCalledTimes(1); @@ -1639,7 +1635,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => { statusGenerators: h.statusGenerators, warmGenericMetrics: h.warmGenericMetrics, enqueueRetry: h.enqueueRetry, - reportRun: h.reportRun as never, + reportRun: h.reportRun, }); expect(h.enqueueRetry).toHaveBeenCalledTimes(1); @@ -1668,7 +1664,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => { statusGenerators: h.statusGenerators, warmGenericMetrics: h.warmGenericMetrics, enqueueRetry: h.enqueueRetry, - reportRun: h.reportRun as never, + reportRun: h.reportRun, shouldStop: () => ++calls > 1, }); @@ -1677,3 +1673,117 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => { ).toEqual(["ok", "deferred", "deferred"]); }); }); + +// #1384 — a self-hoster whose only account kept a provider that no longer +// works was told every night that the job was failing. The run used the real +// verdict here, so what the alert reads (`all_failed`) is what is asserted. +describe("runInsightPregenerate — a provider that cannot serve is not a failing run (#1384)", () => { + async function runOne(outcome: Record) { + const { prisma } = makePrisma([{ id: "only-account", locale: "en" }]); + const reportRun = vi.fn(async (input) => + judgeJobRun(input.candidates), + ); + const enqueueRetry = vi.fn().mockResolvedValue(undefined); + const result = await runInsightPregenerate(prisma as never, { + generate: vi.fn().mockResolvedValue(outcome), + statusGenerators: Array.from({ length: 7 }, () => + vi.fn().mockResolvedValue({ hasProvider: true, cached: true }), + ), + warmGenericMetrics: vi.fn().mockResolvedValue(0), + enqueueRetry, + reportRun, + }); + return { + result, + enqueueRetry, + candidate: reportRun.mock.calls[0][0].candidates[0], + }; + } + + it("skips an account whose every provider was benched, so nothing was sent", async () => { + const { result, enqueueRetry, candidate } = await runOne({ + status: "failed", + reason: "all-providers-failed", + provider: "local", + sentNothing: true, + }); + + expect(candidate).toEqual( + expect.objectContaining({ + outcome: "skipped", + cause: "provider_benched", + }), + ); + expect(result.skipped).toBe(1); + expect(result.failed).toBe(0); + expect(result.allFailed).toBe(false); + expect(result.firstFailure).toBeNull(); + // The retry would find the same bench. + expect(enqueueRetry).not.toHaveBeenCalled(); + expect(annotateSpy).toHaveBeenCalledWith( + expect.objectContaining({ + action: { name: "insights.pregenerate.provider_benched" }, + }), + ); + }); + + it("reports a refused credential of the account's own as auth_failed, naming the provider, without failing the run", async () => { + const { result, enqueueRetry, candidate } = await runOne({ + status: "failed", + reason: "all-providers-failed", + provider: "openai-compatible", + credentialExpired: true, + }); + + expect(candidate).toEqual( + expect.objectContaining({ + outcome: "auth_failed", + cause: "all-providers-failed:openai-compatible", + }), + ); + expect(result.allFailed).toBe(false); + expect(enqueueRetry).not.toHaveBeenCalled(); + expect(annotateSpy).toHaveBeenCalledWith( + expect.objectContaining({ + action: { name: "insights.pregenerate.comprehensive_failed" }, + meta: expect.objectContaining({ + provider: "openai-compatible", + credential_refused: true, + }), + }), + ); + }); + + it("still fails the run when the operator's own key was refused, and says which", async () => { + const { result, enqueueRetry, candidate } = await runOne({ + status: "failed", + reason: "all-providers-failed", + provider: "admin-openai", + credentialExpired: true, + }); + + expect(candidate.outcome).toBe("failed"); + expect(result.allFailed).toBe(true); + expect(result.firstFailure).toEqual({ + reason: "all-providers-failed", + provider: "admin-openai", + }); + // A refused key does not heal in 45 minutes. + expect(enqueueRetry).not.toHaveBeenCalled(); + }); + + it("carries the cause and provider of an ordinary failure into the run's facts", async () => { + const { result, enqueueRetry } = await runOne({ + status: "failed", + reason: "provider-error", + provider: "local", + }); + + expect(result.allFailed).toBe(true); + expect(result.firstFailure).toEqual({ + reason: "provider-error", + provider: "local", + }); + expect(enqueueRetry).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/jobs/__tests__/job-outcome.test.ts b/src/lib/jobs/__tests__/job-outcome.test.ts index 9f0d79d65..437634129 100644 --- a/src/lib/jobs/__tests__/job-outcome.test.ts +++ b/src/lib/jobs/__tests__/job-outcome.test.ts @@ -193,3 +193,36 @@ describe("bounded generic JobOutcome serialization", () => { }); }); }); + +// #1384 — a generator reason and a provider type become facts the admin page +// and the repeated-failure alert can show. +describe("codeFact", () => { + it("turns our own hyphenated codes into stable fact codes", async () => { + const { codeFact, serializeJobOutcome } = await vi.importActual< + typeof import("@/lib/jobs/job-outcome") + >("@/lib/jobs/job-outcome"); + expect(codeFact("all-providers-failed")).toBe("all_providers_failed"); + expect(codeFact("openai-compatible")).toBe("openai_compatible"); + expect(codeFact("generator:invalid-json")).toBe("generator_invalid_json"); + expect(codeFact("timeout")).toBe("timeout"); + // Nothing usable: the caller leaves the fact out. + expect(codeFact("")).toBeNull(); + expect(codeFact("---")).toBeNull(); + expect(codeFact(null)).toBeNull(); + // What it produces is a fact the serializer accepts. + expect( + serializeJobOutcome( + jobDone({ + failure_cause: codeFact("all-providers-failed")!, + failure_provider: codeFact("admin-openai")!, + }), + ), + ).toEqual({ + ok: true, + did: { + failure_cause: "all_providers_failed", + failure_provider: "admin_openai", + }, + }); + }); +}); diff --git a/src/lib/jobs/__tests__/job-run-report.test.ts b/src/lib/jobs/__tests__/job-run-report.test.ts index 2ccd2dd19..3f17db911 100644 --- a/src/lib/jobs/__tests__/job-run-report.test.ts +++ b/src/lib/jobs/__tests__/job-run-report.test.ts @@ -47,13 +47,27 @@ describe("judgeJobRun", () => { }); }); - it("counts a dead credential as a failure", () => { - expect( - judgeJobRun([ - { key: "a", outcome: "auth_failed" }, - { key: "b", outcome: "failed" }, - ]).allFailed, - ).toBe(true); + // #1384 — a key the issuer revoked stays saved and is refused every night. + // Counted as a failure, it made a single-account instance's nightly pass + // read as all-failed forever and alerted the operator about the job. + it("does not fail a run over a refused credential of the account's own", () => { + expect(judgeJobRun([{ key: "a", outcome: "auth_failed" }])).toEqual({ + total: 1, + attempted: 1, + failed: 0, + allFailed: false, + partial: true, + }); + }); + + it("reads a refused credential beside a real failure as partial", () => { + const verdict = judgeJobRun([ + { key: "a", outcome: "auth_failed" }, + { key: "b", outcome: "failed" }, + ]); + expect(verdict.allFailed).toBe(false); + expect(verdict.partial).toBe(true); + expect(verdict.failed).toBe(1); }); it("reads one success among failures as partial, not failed", () => { @@ -84,7 +98,7 @@ describe("reportJobRun", () => { queue: "insight-pregenerate", runId: "r1", candidates: [ - { key: "a", outcome: "auth_failed", cause: "all-providers-failed" }, + { key: "a", outcome: "failed", cause: "all-providers-failed" }, { key: "b", outcome: "skipped", cause: "no-consent" }, ], }); @@ -101,7 +115,31 @@ describe("reportJobRun", () => { consecutiveFailures: 3, failed: 1, total: 1, - causes: ["auth_failed:all-providers-failed"], + causes: ["failed:all-providers-failed"], + }, + }); + }); + + it("emits job.run.partial, never job.run.failed, for a run whose only attempt was a refused credential (#1384)", async () => { + const verdict = await reportJobRun({ + queue: "insight-pregenerate", + runId: "r1", + candidates: [ + { + key: "a", + outcome: "auth_failed", + cause: "all-providers-failed:openai-compatible", + }, + ], + }); + expect(verdict.allFailed).toBe(false); + expect(readConsecutiveRunFailures).not.toHaveBeenCalled(); + expect(emitSignal).toHaveBeenCalledTimes(1); + expect(emitSignal.mock.calls[0][0]).toMatchObject({ + action: "job.run.partial", + level: "warn", + meta: { + causes: ["auth_failed:all-providers-failed:openai-compatible"], }, }); }); diff --git a/src/lib/jobs/__tests__/system-job-alert.test.ts b/src/lib/jobs/__tests__/system-job-alert.test.ts index 224fb0209..14c390d6f 100644 --- a/src/lib/jobs/__tests__/system-job-alert.test.ts +++ b/src/lib/jobs/__tests__/system-job-alert.test.ts @@ -87,6 +87,34 @@ describe("judgeQueueRuns", () => { ).toBe("failing"); }); + // #1384 — the alert quoted the completed run's own outcome JSON, and the + // operator it sent to the admin page found nothing there. + it("says in words what an all-failed night was, with its cause and provider", () => { + const output = JSON.stringify({ + ok: true, + did: { + total: 1, + failed: 1, + all_failed: true, + assessments_warmed: 0, + failure_cause: "all_providers_failed", + failure_provider: "admin_openai", + }, + }); + const verdict = judgeQueueRuns( + [0, 1, 2].map((ago) => + run(ago, { state: "completed", allFailed: true, error: output }), + ), + ); + expect(verdict.state).toBe("failing"); + expect(verdict.lastError).toBe( + "The run completed, but every account it attempted failed (1 of 1). " + + "First cause: all_providers_failed from provider admin_openai. " + + "Accounts without a usable provider are skipped and not counted; " + + "job.run.failed in the app log lists each account's outcome.", + ); + }); + it("explains a bare timeout the way the admin card does", () => { const verdict = judgeQueueRuns( [0, 1, 2].map((ago) => diff --git a/src/lib/jobs/insight-pregenerate-failure.ts b/src/lib/jobs/insight-pregenerate-failure.ts new file mode 100644 index 000000000..bcad68b35 --- /dev/null +++ b/src/lib/jobs/insight-pregenerate-failure.ts @@ -0,0 +1,93 @@ +/** + * How the nightly briefing pass settles an account whose generation came back + * `failed` (#1384). + * + * A failure used to be one of three things: screened, a dead sign-in, or a + * provider failure worth a retry in 45 minutes. Two cases were missing, and + * both made a self-hoster's only account read as a failing run every night, + * which the repeated-failure alert then reported as a broken job: + * + * - Every provider of the chain was benched for background work by the + * provider health ledger (#1279), so no request was sent. The account was + * not attempted tonight; its earlier failures were counted by the runs + * that made them, and a retry would find the same bench. It is `skipped`. + * - The provider refused the credential. A refused credential does not heal + * in 45 minutes, so no retry is queued, and whose credential it was + * decides who has to act. The operator's shared key or central sign-in is + * the operator's to fix, and a run where it was refused for everyone is a + * failing run: `failed`. The account's own key, gateway or sign-in is its + * owner's to fix: `auth_failed`, which the run reports but does not fail + * over (`job-run-report.ts`). An API key revoked at its issuer stays saved + * and is refused again every night, unlike a ChatGPT sign-in, which the + * refusal parks. + * + * The model answered and the safety screen withheld it (`outbound-screened`) + * stays `screened` with no retry: asking the same model again in 45 minutes + * is the same question, and it was screened eight times a night. + */ +import { isOperatorHeldProvider } from "@/lib/ai/provider-egress"; +import type { GenerateOutcome } from "@/lib/insights/comprehensive-generate"; +import type { JobRunCandidate, JobRunOutcome } from "@/lib/jobs/job-run-report"; + +type FailedGeneration = Extract; + +export interface FailedGenerationVerdict { + outcome: JobRunOutcome; + /** The run line's cause: the reason, and the provider that failed first. */ + cause: string; + /** False when nothing was sent: the account is tallied as skipped. */ + attempted: boolean; + /** Queue the bounded 45-minute retry for this account. */ + retry: boolean; +} + +export function settleFailedGeneration( + generated: FailedGeneration, +): FailedGenerationVerdict { + if (generated.sentNothing) { + return { + outcome: "skipped", + cause: "provider_benched", + attempted: false, + retry: false, + }; + } + const cause = generated.provider + ? `${generated.reason}:${generated.provider}` + : generated.reason; + if (generated.reason === "outbound-screened") { + return { outcome: "screened", cause, attempted: true, retry: false }; + } + if (generated.credentialExpired) { + const operatorHeld = + generated.provider !== undefined && + isOperatorHeldProvider(generated.provider); + return { + outcome: operatorHeld ? "failed" : "auth_failed", + cause, + attempted: true, + retry: false, + }; + } + return { outcome: "failed", cause, attempted: true, retry: true }; +} + +/** The first failed account of a run: its reason and the provider, if known. */ +export interface FirstFailure { + reason: string; + provider: string | null; +} + +/** + * The first failed account's reason and provider, read back off the run's + * causes, for the job's facts: what the admin page and the repeated-failure + * alert show for a run that completed with every account failed. + */ +export function firstFailure( + candidates: readonly JobRunCandidate[], +): FirstFailure | null { + const cause = candidates.find((c) => c.outcome === "failed")?.cause; + if (!cause) return null; + const [reason, provider] = cause.split(":"); + return { reason, provider: provider ?? null }; +} diff --git a/src/lib/jobs/insight-pregenerate.ts b/src/lib/jobs/insight-pregenerate.ts index 59f453448..5c09b727c 100644 --- a/src/lib/jobs/insight-pregenerate.ts +++ b/src/lib/jobs/insight-pregenerate.ts @@ -81,6 +81,10 @@ import { } from "@/lib/insights/metric-status-registry"; import { withTimeout } from "@/lib/insights/with-timeout"; import { comprehensiveWarmBudgetMs } from "@/lib/jobs/model-time"; +import { + firstFailure, + settleFailedGeneration, +} from "@/lib/jobs/insight-pregenerate-failure"; import { INSIGHT_PREGENERATE_QUEUE, enqueueForceWarm, @@ -235,6 +239,8 @@ export interface PregenerateRunResult { * seven specialised cards. */ metricAssessmentsWarmed: number; + /** #1384 — the first failed account's reason and provider, as facts. */ + firstFailure: ReturnType; } /** @@ -302,9 +308,9 @@ async function warmPerStatusCaches( * once and issues ONE provider call for the metrics still needing the LLM, * fanning the response back into the same per-metric cache rows the * standalone generators wrote. Replaces the prior seven independent warm - * calls. Returns the count of metrics that got a fresh (batched or - * fallback) assessment so the wide-event tally is comparable to the old - * per-card count. + * calls. Returns the count of metrics that got a fresh model-written + * assessment, batched or through a card's own fallback call, so the + * wide-event tally is comparable to the old per-card count. * * Refill-only (`force: false`): a card already generated today and a card * whose content-hash is unchanged both resolve inside the batch as `served` @@ -322,7 +328,7 @@ async function warmStatusBatch( locale, force: false, }); - warmed += result.batched + result.fellBack; + warmed += result.batched + result.fellBackAnswered; } catch (err) { // The batch must never throw the cron's user loop off course — but // a swallowed batch failure left ~7 cards cold with zero signal. @@ -555,6 +561,7 @@ export async function runInsightPregenerate( allFailed: false, assessmentsWarmed: 0, metricAssessmentsWarmed: 0, + firstFailure: null, }; // Master kill-switch — when the operator disabled the briefing @@ -619,6 +626,7 @@ export async function runInsightPregenerate( candidates: runOutcomes, }); result.allFailed = verdict.allFailed; + result.firstFailure = firstFailure(runOutcomes); return result; @@ -783,42 +791,34 @@ export async function runInsightPregenerate( result.skipped++; settle("skipped", generated.reason); break; - case "failed": - result.failed++; - // v1.28.30 — THE previously-silent failure path. A generator - // that resolves with `{ status: "failed" }` (features error, - // oversize payload, invalid JSON after retry, abort) bumped - // the tally with no annotation at all — the nightly summary - // showed `failed: 1` and nothing else was greppable. Reuse - // the `comprehensive_failed` shape with the generator's own - // reason as the cause. + case "failed": { + // Benched, screened, a refused credential and whose, or a + // retryable failure: `insight-pregenerate-failure.ts` (#1384). + const verdict = settleFailedGeneration(generated); + if (verdict.attempted) result.failed++; + else result.skipped++; annotate({ - action: { name: "insights.pregenerate.comprehensive_failed" }, + action: { + name: verdict.attempted + ? "insights.pregenerate.comprehensive_failed" + : "insights.pregenerate.provider_benched", + }, meta: { locale, stage: "nightly.generator", cause: `generator:${generated.reason}`, + provider: generated.provider ?? null, + credential_refused: generated.credentialExpired === true, budget_ms: warmBudgetMs, message: null, }, }); - if (generated.reason === "outbound-screened") { - // v1.42 — the model answered and the safety screen withheld - // it, after the generator's own repair. Asking the same model - // again in 45 minutes is not a provider retry; it is the same - // question, and it was screened eight times a night. - settle("screened", generated.reason); - } else if (generated.credentialExpired) { - // v1.42 — a dead sign-in (a Codex refresh token the issuer - // revoked) does not heal in 45 minutes either; the person has - // to reconnect. The refusal parked the connection as expired, - // which Settings shows, and the next run no longer counts it. - settle("auth_failed", generated.reason); - } else { - settle("failed", generated.reason); + settle(verdict.outcome, verdict.cause); + if (verdict.retry) { await enqueueRetry({ userId: candidate.id, locale }); } break; + } } } } diff --git a/src/lib/jobs/job-failures.ts b/src/lib/jobs/job-failures.ts index c9bf7a82b..b96cfd929 100644 --- a/src/lib/jobs/job-failures.ts +++ b/src/lib/jobs/job-failures.ts @@ -92,7 +92,7 @@ const MAX_ERROR_CHARS = 300; export interface FailingQueue { /** The pg-boss queue name, e.g. `environment-fetch`. */ queue: string; - /** Jobs that exhausted their retries inside the window. */ + /** Failed runs inside the window: exhausted retries, or all-failed passes. */ failures: number; /** ISO instant of the newest failure. */ lastFailedAt: string; @@ -140,10 +140,48 @@ function explainTimeout(message: string, expireSeconds: number | null): string { ); } +/** + * Say what a completed run that failed every account means (#1384). Such a + * run's `output` is its outcome, `{"ok":true,"did":{…,"all_failed":true}}`, + * and quoting that JSON told the operator nothing: not that the job ran, not + * which provider failed. Null when `raw` is not such an outcome. + */ +function explainAllFailedRun(raw: string): string | null { + if (!raw.startsWith("{")) return null; + let did: Record; + try { + const parsed = JSON.parse(raw) as { ok?: unknown; did?: unknown }; + if (parsed?.ok !== true || typeof parsed.did !== "object") return null; + did = (parsed.did ?? {}) as Record; + } catch { + return null; + } + if (did.all_failed !== true) return null; + // Every attempted account failed, so the count is both sides of "of". + const failed = + typeof did.failed === "number" && did.failed > 0 + ? ` (${did.failed} of ${did.failed})` + : ""; + // "none" is how the run says it has no cause or provider to name. + const named = (value: unknown): string | null => + typeof value === "string" && value !== "none" ? value : null; + const cause = named(did.failure_cause); + const provider = named(did.failure_provider); + const why = cause + ? ` First cause: ${cause}${provider ? ` from provider ${provider}` : ""}.` + : ""; + return ( + `The run completed, but every account it attempted failed${failed}.${why} ` + + `Accounts without a usable provider are skipped and not counted; ` + + `job.run.failed in the app log lists each account's outcome.` + ); +} + /** * Turn whatever pg-boss stored in `output` into one line. A thrown `Error` * serialises to `{ message, stack, … }`; a thrown non-Error can be any JSON at * all, so the raw text is the fallback rather than an invented placeholder. + * A completed run that reported every account failed is said in words. */ export function presentError( raw: string | null, @@ -152,6 +190,8 @@ export function presentError( if (raw === null) return ""; const trimmed = raw.trim(); if (trimmed.length === 0) return ""; + const allFailed = explainAllFailedRun(trimmed); + if (allFailed !== null) return allFailed; if (TIMEOUT_MESSAGE.test(trimmed)) return explainTimeout(trimmed, expireSeconds); if (HEARTBEAT_MESSAGE.test(trimmed)) { @@ -168,8 +208,13 @@ export function presentError( } /** - * Every queue with at least one exhausted-retry failure in the window, newest - * first. `null` when the queue schema is not present at all. + * Every queue with at least one failed run in the window, newest first: + * a job that exhausted its retries, or a fan-out pass that completed but + * reported that every account it attempted failed (`did.all_failed`). The + * second kind is what the repeated-failure alert counts too + * (`readConsecutiveRunFailures`, `system-job-alert.ts`); the page used to + * list only the first, so an operator the alert sent here read "no job has + * failed" (#1384). `null` when the queue schema is not present at all. */ export async function readFailingQueues( withinHours: number = JOB_FAILURE_WINDOW_HOURS, @@ -194,7 +239,10 @@ export async function readFailingQueues( array_agg(expire_seconds ORDER BY completed_on DESC NULLS LAST) )[1] AS last_expire_seconds FROM pgboss.job - WHERE state = 'failed' + WHERE ( + state = 'failed' + OR (state = 'completed' AND output->'did'->>'all_failed' = 'true') + ) AND completed_on > now() - make_interval(hours => ${withinHours}::int) GROUP BY name ORDER BY max(completed_on) DESC diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index 942b5fde6..f47efcf38 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -130,6 +130,11 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "duration_ms", "errored", "failed", + // #1384 — a fan-out pass's first failed candidate: its cause and the + // provider that failed for it, as stable codes. What the admin page and + // the repeated-failure alert show for a run that completed all-failed. + "failure_cause", + "failure_provider", "failures", "feedback_buckets", "feedback_total_rows", @@ -423,6 +428,22 @@ export function percentFact(ratio: number | null | undefined): number { return wholeFact(ratio * 100); } +/** + * A short code from our own vocabulary (a generator reason, a provider type) + * as a fact: `all-providers-failed` becomes `all_providers_failed`. Null when + * nothing of it survives, so the caller leaves the fact out. + */ +export function codeFact(value: string | null | undefined): string | null { + if (value == null) return null; + const code = value + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^[^a-z]+/, "") + .slice(0, 64) + .replace(/_+$/, ""); + return STABLE_CODE_PATTERN.test(code) ? code : null; +} + /** Why one fact is refused, or null when it is a valid fact. */ function invalidFact(key: string, value: unknown): string | null { if (!JOB_FACT_ALLOWLIST.has(key)) diff --git a/src/lib/jobs/job-run-report.ts b/src/lib/jobs/job-run-report.ts index 4dadbd0f4..f38f8b4d2 100644 --- a/src/lib/jobs/job-run-report.ts +++ b/src/lib/jobs/job-run-report.ts @@ -21,6 +21,18 @@ * attempted but is not a provider failure, so a run of only screened * candidates is partial, not failed. * + * `auth_failed` is attempted and is not a failure of the run either (#1384). + * It means the provider refused the account's own credential: a key revoked + * at its issuer, a gateway that now wants a key the account no longer + * stores. Only that account's owner can change it, and the run itself worked. + * It used to count as a failure on the assumption that a refusal is reported + * once and then parked; that holds for a ChatGPT sign-in and for nothing else, + * so an API key the issuer had revoked failed every night for as long as it + * stayed saved. On an instance where that account was the only candidate the + * run then read as all-failed, and the operator was told the job was broken + * night after night. A refusal of the operator's own credential is reported by + * the caller as `failed`, because there the operator is the one who can act. + * * The verdict comes back to the caller, which writes `all_failed` into its * `jobDone` facts. That fact is what lets `readConsecutiveRunFailures` count a * completed-but-all-failed night as part of a streak. @@ -46,18 +58,26 @@ export interface JobRunVerdict { total: number; /** Candidates it actually attempted (not skipped, not deferred). */ attempted: number; - /** `failed` + `auth_failed`. */ + /** Candidates that failed (`failed`; a refused credential is not one). */ failed: number; /** Every attempted candidate failed (and at least one was attempted). */ allFailed: boolean; - /** Something failed or was withheld, but not everything. */ + /** + * Something failed, was withheld or had its credential refused, but not + * everything failed. + */ partial: boolean; } /** Candidates listed on the run's line; the counts always cover all of them. */ const MAX_LISTED_CANDIDATES = 50; -const FAILED: ReadonlySet = new Set(["failed", "auth_failed"]); +const FAILED: ReadonlySet = new Set(["failed"]); +/** Attempted, not a failure of the run, but worth a `partial` line. */ +const WITHHELD: ReadonlySet = new Set([ + "screened", + "auth_failed", +]); const NOT_ATTEMPTED: ReadonlySet = new Set([ "skipped", "deferred", @@ -72,14 +92,14 @@ export function judgeJobRun( (c) => !NOT_ATTEMPTED.has(c.outcome), ).length; const failed = candidates.filter((c) => FAILED.has(c.outcome)).length; - const screened = candidates.filter((c) => c.outcome === "screened").length; + const withheld = candidates.filter((c) => WITHHELD.has(c.outcome)).length; const allFailed = attempted > 0 && failed === attempted; return { total, attempted, failed, allFailed, - partial: !allFailed && (failed > 0 || screened > 0), + partial: !allFailed && (failed > 0 || withheld > 0), }; } diff --git a/src/lib/jobs/reminder/insights-handlers.ts b/src/lib/jobs/reminder/insights-handlers.ts index 54fec9bdb..c34506f45 100644 --- a/src/lib/jobs/reminder/insights-handlers.ts +++ b/src/lib/jobs/reminder/insights-handlers.ts @@ -21,7 +21,7 @@ import { type InsightStatusGeneratePayload, runInsightStatusGenerate, } from "@/lib/jobs/insight-status-generate"; -import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { codeFact, jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { jobBudget } from "@/lib/jobs/job-budget"; import { withBackgroundEvent } from "@/lib/logging/background"; import { generateStatusBatchForUser } from "@/lib/insights/status-batch"; @@ -80,7 +80,7 @@ export async function runStatusBatchCron( locale: await resolveJobLocale(user.locale), force: false, }); - generated += result.batched + result.fellBack; + generated += result.batched + result.fellBackAnswered; served += result.served; } catch (error) { failed++; @@ -203,6 +203,10 @@ export async function handleInsightPregenerateJob( all_failed: summary.allFailed, assessments_warmed: summary.assessmentsWarmed, metric_assessments_warmed: summary.metricAssessmentsWarmed, + // #1384 — what failed, for the admin page and the alert ("none" + // when nothing did, or the generator could not say). + failure_cause: codeFact(summary.firstFailure?.reason) ?? "none", + failure_provider: codeFact(summary.firstFailure?.provider) ?? "none", }); } catch (err) { evt.addWarning( diff --git a/tests/integration/__golden__/status-parity/batch.json b/tests/integration/__golden__/status-parity/batch.json index 6091a6647..725ea250c 100644 --- a/tests/integration/__golden__/status-parity/batch.json +++ b/tests/integration/__golden__/status-parity/batch.json @@ -460,6 +460,7 @@ "batchCallMade": true, "batched": 6, "fellBack": 1, + "fellBackAnswered": 0, "served": 0 } }, @@ -1719,6 +1720,7 @@ "batchCallMade": true, "batched": 0, "fellBack": 7, + "fellBackAnswered": 0, "served": 0 } }, @@ -1831,6 +1833,7 @@ "batchCallMade": false, "batched": 0, "fellBack": 1, + "fellBackAnswered": 0, "served": 6 } } diff --git a/tests/integration/background-switch-freshness.test.ts b/tests/integration/background-switch-freshness.test.ts index 99f3ac81f..8ff0db47d 100644 --- a/tests/integration/background-switch-freshness.test.ts +++ b/tests/integration/background-switch-freshness.test.ts @@ -40,6 +40,7 @@ vi.mock("@/lib/insights/status-batch", async () => { served: 0, batched: 0, fellBack: 0, + fellBackAnswered: 0, batchCallMade: false, }; if (!(await aiCapabilityForJob(userId, "statusText")).available) { diff --git a/tests/integration/job-failures.test.ts b/tests/integration/job-failures.test.ts index f13180fd7..997c20f97 100644 --- a/tests/integration/job-failures.test.ts +++ b/tests/integration/job-failures.test.ts @@ -126,6 +126,41 @@ describe("readFailingQueues", () => { await expect(readFailingQueues()).resolves.toEqual([]); }); + // #1384 — the repeated-failure alert counts a nightly pass that completed + // with every account failed, and sent the operator to this page, which + // listed only pg-boss failures and said nothing had failed. + it("lists a run that completed with every account failed, in words, and not a good run", async () => { + const ALL_FAILED_QUEUE = "insight-pregenerate"; + await boss.createQueue(ALL_FAILED_QUEUE); + async function completeOne(did: Record) { + const jobId = await boss.send(ALL_FAILED_QUEUE, {}, { retryLimit: 0 }); + const [job] = await boss.fetch(ALL_FAILED_QUEUE); + if (!job || job.id !== jobId) throw new Error("nothing to fetch"); + await boss.complete(ALL_FAILED_QUEUE, job.id, { ok: true, did }); + } + // A good night, and the night from the report: one account, one failure. + await completeOne({ total: 1, generated: 1, failed: 0, all_failed: false }); + await completeOne({ + total: 1, + generated: 0, + failed: 1, + all_failed: true, + assessments_warmed: 0, + failure_cause: "all_providers_failed", + failure_provider: "openai_compatible", + }); + + const failing = await readFailingQueues(); + const entry = failing?.find((e) => e.queue === ALL_FAILED_QUEUE); + expect(entry?.failures).toBe(1); + expect(entry?.lastError).toBe( + "The run completed, but every account it attempted failed (1 of 1). " + + "First cause: all_providers_failed from provider openai_compatible. " + + "Accounts without a usable provider are skipped and not counted; " + + "job.run.failed in the app log lists each account's outcome.", + ); + }); + it("ignores a failure older than the window", async () => { await failOneJob(QUEUE, { userId: "user-a" }, "old failure"); await getPrismaClient().$executeRawUnsafe( From 42c8b11fcac3d9dabfa52554e1e45cfc404b5afb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 15 Nov 2026 10:46:35 +0100 Subject: [PATCH 2/7] chore(release): v1.47.6 --- CHANGELOG.md | 37 +++++++++++++++++-------------------- docs/api/openapi.yaml | 2 +- package.json | 2 +- public/sw.js | 2 +- 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de935e7cb..fecec54e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,26 +42,23 @@ links the help page. Brave keeps push messaging off by default, so in Brave the message names the setting to turn on: "Use Google services for push messaging" under brave://settings/privacy. (Discussion #1378) -- **Illness episode onset dates keep the intended calendar day.** - Creating a new illness episode in the morning was refused because its noon - default was rejected as in the future, and onset dates could display as the - wrong calendar day in non-UTC time zones. Saving now anchors to the current - time when before local noon and preserves stored timestamps on edit. -- **No more nightly "job still failing" alert over a provider that cannot - answer.** When the only account on an instance still had an AI provider - saved whose key the provider no longer accepted (removing the key from a - Local or OpenAI-compatible provider keeps its address, so it is still - called), or whose provider HealthLog had paused after repeated errors, the - nightly briefing pass counted that account as a failure, and every night - the admins were told the job was broken. A key the provider refuses is now - that account's matter: the run notes it but does not fail over it, unless - it is the instance's own shared key. An account whose providers are all - paused is skipped, since nothing was sent. The Background jobs card on the - admin status page now lists a run that finished with every account - failing, the same runs the alert counts, and the alert and the card say in - words what failed and which provider, instead of quoting the run's - counters. Those counters also stop counting a fallback text as a warmed - assessment. (#1384) +- **Illness episodes can be added in the morning.** The form sent the + start date as noon, which before noon is still in the future, so the + save was refused. Start and end dates now also stay on the day you + picked when your time zone is far from UTC, and editing an episode + keeps the times you saved. (#1376) +- **No nightly "job still failing" alert over an AI provider that cannot + answer.** An account that still had a provider saved whose key the + provider refused (removing the key from a Local or OpenAI-compatible + provider keeps its address, so it is still called), or whose providers + HealthLog had paused after repeated errors, counted as a failed + briefing every night. On an instance with one account that read as a + broken job. A refused key of the account's own is now noted but no + longer fails the run; a refused shared server key still does. An + account whose providers are all paused is skipped, since nothing was + sent. The Background jobs card on the admin status page now lists the + same runs the alert counts, and both say in words what failed and + which provider. (#1384) ### Security diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index fbb976f6c..aede67992 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: HealthLog API - version: 1.47.5 + version: 1.47.6 description: >- Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest. diff --git a/package.json b/package.json index 87525da8e..d7bfeb12c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "healthlog", - "version": "1.47.5", + "version": "1.47.6", "description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.", "license": "PolyForm-Noncommercial-1.0.0", "homepage": "https://healthlog.dev", diff --git a/public/sw.js b/public/sw.js index fbb56e5c2..5ca6eb509 100644 --- a/public/sw.js +++ b/public/sw.js @@ -16,7 +16,7 @@ try { } const CACHE_VERSION = (typeof self !== "undefined" && self.__APP_VERSION__) || - /* @sw-version-fallback */ "v1.47.5"; + /* @sw-version-fallback */ "v1.47.6"; const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`; const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`; // Every HealthLog cache, current and legacy, and nothing another app on the From e19b9299b72c80eea2723bc866ac61dc254e00d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 15 Nov 2026 10:46:41 +0100 Subject: [PATCH 3/7] chore(release): date the 1.47.6 changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fecec54e0..840291047 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Unreleased] +## [1.47.6] — 2026-11-15 ### Added From 883df7b5697fe725e59c48c705185c16be2fe116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 15 Nov 2026 10:46:54 +0100 Subject: [PATCH 4/7] chore(release): keep an empty Unreleased heading --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 840291047..5b9ac6ba7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## [Unreleased] + ## [1.47.6] — 2026-11-15 ### Added From 23a6f85b0f718e8213575e4c1eee7086472f7468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 15 Nov 2026 10:58:06 +0100 Subject: [PATCH 5/7] fix(sharing): drop an account read answered across a record switch Raising the record-session hold clears the query cache, which remounts the account query, so the switching tab has a GET /api/auth/me in flight beside its own POST /api/account/switch. When the server answered that read from the record being left and the answer was applied after the commit, fetchMe rewrote the scope mirror back to the old record. A peer tab that had already settled read that write as a foreign switch and published a hold expecting the old record, which no /me would ever confirm, so every tab sat behind the hydration gate until the stored transition aged out. The transition store now counts the changes a tab applies, and fetchMe repeats a read whose request straddled one instead of applying it. --- CHANGELOG.md | 6 + .../__tests__/use-auth-superseded-me.test.ts | 149 ++++++++++++++++++ src/hooks/use-auth.ts | 50 +++++- .../query-keys/record-session-transition.ts | 23 +++ 4 files changed, 220 insertions(+), 8 deletions(-) create mode 100644 src/hooks/__tests__/use-auth-superseded-me.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d4156079..69ca20e46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,12 @@ default was rejected as in the future, and onset dates could display as the wrong calendar day in non-UTC time zones. Saving now anchors to the current time when before local noon and preserves stored timestamps on edit. +- **Opening a record with the app open in a second tab.** Switching into or + out of a shared record could leave both tabs on the loading spinner, and a + reload within the next half minute did not help. An account check the + switching tab had started a moment before the switch could answer with the + record being left and overwrite the switch the other tabs were waiting on. + Such an answer is now discarded and asked again. ### Security diff --git a/src/hooks/__tests__/use-auth-superseded-me.test.ts b/src/hooks/__tests__/use-auth-superseded-me.test.ts new file mode 100644 index 000000000..86f1b154c --- /dev/null +++ b/src/hooks/__tests__/use-auth-superseded-me.test.ts @@ -0,0 +1,149 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { + __resetRecordFenceForTests, + currentRecordFenceState, +} from "@/lib/api/record-fence"; +import { + __resetRecordScopeForTests, + getRecordScope, + setRecordScope, +} from "@/lib/query-keys/record-scope"; +import { + __resetRecordSessionTransitionForTests, + beginRecordSessionTransition, + commitRecordSessionTransition, + getRecordSessionTransition, +} from "@/lib/query-keys/record-session-transition"; + +const apiGet = vi.fn(); + +vi.mock("@/lib/api/api-fetch", () => ({ + apiGet: (...args: unknown[]) => apiGet(...args), + apiFetchRaw: vi.fn(), + ApiError: class ApiError extends Error {}, +})); + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ push: vi.fn(), replace: vi.fn() }), +})); + +import { fetchMe } from "../use-auth"; + +/** + * A `/api/auth/me` answered across a record-session transition. + * + * The browser journey that found this: the switcher clears the query cache + * when it raises the hold, which remounts the account query, so the initiating + * tab has a `/me` in flight beside its own switch POST. The server answers it + * with the record being left. Applied after the commit, that answer rewrote the + * scope mirror to the old record, a settled peer read the write as a foreign + * switch, and every tab then waited for a `/me` naming a record the server had + * already left — behind the hydration gate, for good. + */ + +const TARGET = "record-b"; + +const entry = { + accountId: TARGET, + username: "record-b", + displayName: null, + fullName: null, + access: "read", + level: "read", + recordKind: "shared", + sections: null, + canWrite: false, + writableDomains: [], + manageableDomains: [], +}; + +const base = { + id: "delegate", + username: "delegate", + role: "USER", + timezone: "Europe/Berlin", +}; + +/** What the server says before the switch lands: the caller's own record. */ +const beforeSwitch = { + ...base, + accountAccess: { + accounts: [entry], + active: null, + recordKind: "self", + canSwitch: true, + }, + recordSession: { epoch: 4, scope: null }, +}; + +/** And after it. */ +const afterSwitch = { + ...base, + accountAccess: { + accounts: [entry], + active: entry, + recordKind: "shared", + canSwitch: true, + }, + recordSession: { epoch: 5, scope: TARGET }, +}; + +beforeEach(() => { + apiGet.mockReset(); + __resetRecordScopeForTests(); + __resetRecordSessionTransitionForTests(); + __resetRecordFenceForTests(); +}); + +describe("fetchMe across a record-session transition", () => { + it("drops an answer that straddled the commit and reads again", async () => { + setRecordScope(null); + const id = beginRecordSessionTransition(TARGET); + + apiGet + .mockImplementationOnce(async () => { + // The switch commits while this read is on the wire, and the server + // had already answered it from the record being left. + commitRecordSessionTransition(id, TARGET); + return beforeSwitch; + }) + .mockResolvedValueOnce(afterSwitch); + + const user = await fetchMe(); + + expect(apiGet).toHaveBeenCalledTimes(2); + expect(user.accountAccess?.active?.accountId).toBe(TARGET); + // The committed scope survives; the pre-switch answer never reached the + // mirror or the fence. + expect(getRecordScope()).toBe(TARGET); + expect(currentRecordFenceState()).toEqual({ epoch: 5, scope: TARGET }); + // And the hold releases on the read that describes the switched session. + expect(getRecordSessionTransition().phase).toBe("ready"); + }); + + it("applies an answer when nothing moved while it was in flight", async () => { + const id = beginRecordSessionTransition(TARGET); + commitRecordSessionTransition(id, TARGET); + apiGet.mockResolvedValueOnce(afterSwitch); + + await fetchMe(); + + expect(apiGet).toHaveBeenCalledTimes(1); + expect(getRecordScope()).toBe(TARGET); + expect(getRecordSessionTransition().phase).toBe("ready"); + }); + + it("gives up without applying anything when the session never holds still", async () => { + setRecordScope(null); + apiGet.mockImplementation(async () => { + // Every read straddles a fresh transition. + beginRecordSessionTransition(TARGET); + return beforeSwitch; + }); + + await expect(fetchMe()).rejects.toThrow(/kept moving/); + expect(apiGet).toHaveBeenCalledTimes(4); + expect(currentRecordFenceState()).toBeNull(); + }); +}); diff --git a/src/hooks/use-auth.ts b/src/hooks/use-auth.ts index 0c84e4924..580d4a7bf 100644 --- a/src/hooks/use-auth.ts +++ b/src/hooks/use-auth.ts @@ -36,6 +36,7 @@ import { setRefusedRecordScope, } from "@/lib/query-keys/record-scope"; import { + recordSessionRevision, settleRecordSessionTransition, settleRefusedRecordSessionTransition, } from "@/lib/query-keys/record-session-transition"; @@ -361,6 +362,46 @@ export function isAuthVerdictUnknown(error: unknown): boolean { return true; } +/** + * How many times `fetchMe` asks again when a record-session transition moved + * while its request was in flight. Each repeat needs another transition inside + * one round trip, which only a browser switching records over and over + * produces. + */ +const SUPERSEDED_ME_RETRIES = 3; + +type MePayload = Partial & { + id: string; + username: string; + role: string; + timezone: string; +}; + +/** + * `GET /api/auth/me`, never answered across a record-session transition. + * + * Everything `fetchMe` does with the payload is a statement about which record + * this browser is in: it adopts the fence context, rewrites the cache-scope + * mirror every other tab watches, and may release the hold. A request issued + * before a transition and answered after it may have been read on the far side + * of a switch, and applying it is how a browser ended up wedged behind the + * hydration gate (see `recordSessionRevision`). Such an answer is dropped and + * the read repeated; the repeat is issued after the change, so it describes the + * session as it now is. + */ +async function getCurrentMe(): Promise { + for (let attempt = 0; ; attempt++) { + const before = recordSessionRevision(); + const data = await apiGet("/api/auth/me"); + if (recordSessionRevision() === before) return data; + if (attempt >= SUPERSEDED_ME_RETRIES) { + // Not an `ApiError`: no verdict on the session, so nothing is wiped and + // the hold stays until a later read settles it. + throw new Error("record session kept moving during /api/auth/me"); + } + } +} + export async function fetchMe(): Promise { // v1.16.4 — routed through the typed wrapper; a non-OK /me (401) // throws `ApiError`, which `useAuth` treats as "not authenticated" @@ -369,14 +410,7 @@ export async function fetchMe(): Promise { // v1.4.47 — coerce `disableCoach` against `undefined` so a stale // /me payload from a partial-deploy rollback (older server image // without the field) keeps the Coach surface visible by default. - const data = await apiGet< - Partial & { - id: string; - username: string; - role: string; - timezone: string; - } - >("/api/auth/me"); + const data = await getCurrentMe(); // Hour-cycle preference: coerce against a stale /me payload, then mirror // into localStorage so `useFormatters()` and the legacy format helpers // (which cannot reach the query cache) render the same clock. diff --git a/src/lib/query-keys/record-session-transition.ts b/src/lib/query-keys/record-session-transition.ts index c971883c3..d44f8d5a9 100644 --- a/src/lib/query-keys/record-session-transition.ts +++ b/src/lib/query-keys/record-session-transition.ts @@ -51,6 +51,8 @@ const READY: RecordSessionTransition = { let transition = READY; let seeded = false; +/** Bumped on every change this tab applies; see `recordSessionRevision`. */ +let revision = 0; const listeners = new Set<() => void>(); let channel: BroadcastChannel | null = null; let storageListenerAttached = false; @@ -107,6 +109,7 @@ function applyWire(next: WireTransition): void { } transition = toSnapshot(next); seeded = true; + revision += 1; notify(); } @@ -228,6 +231,26 @@ export function getRecordSessionTransition(): RecordSessionTransition { return transition; } +/** + * A counter that moves whenever this tab applies a transition change — its own + * publish, or one received from another tab. + * + * `fetchMe` reads it on both sides of its request. A `/api/auth/me` answered + * across a change cannot say which side of that change the server read it on: + * the switcher's own `queryClient.clear()` remounts the account query the + * moment the hold goes up, so the initiator routinely has a `/me` in flight + * beside its own `POST /api/account/switch`, and that answer can describe the + * record the session has just left. Applied after the commit, it rewrites the + * scope mirror back to the old record; every peer that has already settled + * reads that write as a foreign switch and holds for a `/me` naming the old + * record, which no server answer will ever give. The counter is how `fetchMe` + * recognises such an answer and asks again instead of applying it. + */ +export function recordSessionRevision(): number { + seed(); + return revision; +} + export function subscribeToRecordSessionTransition( listener: () => void, ): () => void { From e84c358fdff5a68620d3979f756191ff3df8f31a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 15 Nov 2026 10:58:13 +0100 Subject: [PATCH 6/7] fix(charts): match chart loading shells to the charts they become The footer reserve under a day-linked chart's loading shell still held the caption line that was dropped from those charts, so every such shell stood 20 px taller than its chart (40 px on a phone). The dashboard's mood chart loaded behind the generic shell, which is about 30 px shorter than the mood card. When the shared chart chunk arrived late the dashboard's chart row changed height under a reader already scrolled past it; the tablet shell journey measured
right after such a swap and found it no longer at its end. The reserve now holds only the day rug, and the mood chart gets a shell built from the same card primitives and boxes. The tablet journey also waits for every chart cell to reveal and every loading shell to go before it scrolls to the end, instead of trusting two equal heights 250 ms apart. --- CHANGELOG.md | 5 ++ e2e/tablet-shell.spec.ts | 36 +++++++++++ src/app/insights/mood/page-client.tsx | 7 +- src/components/charts/chart-skeleton.tsx | 64 +++++++++++++++---- .../dashboard/tiles/chart-tiles.tsx | 7 +- 5 files changed, 103 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69ca20e46..50a6d2152 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,11 @@ switching tab had started a moment before the switch could answer with the record being left and overwrite the switch the other tabs were waiting on. Such an answer is now discarded and asked again. +- **Charts no longer jump while they load.** The placeholder of every chart + that links its days was 20 px taller than the chart it turned into (40 px + on a phone), and the mood chart's placeholder was about 30 px shorter, so + the dashboard and the metric pages shifted under the reader when the charts + arrived. ### Security diff --git a/e2e/tablet-shell.spec.ts b/e2e/tablet-shell.spec.ts index ddd77e8a7..6f67b9cbb 100644 --- a/e2e/tablet-shell.spec.ts +++ b/e2e/tablet-shell.spec.ts @@ -54,6 +54,42 @@ async function openDashboard(page: Page) { ).toBeVisible(); // The module-gated destinations join the rail once `/api/auth/me` lands. await expect(page.locator('aside nav a[href="/medications"]')).toHaveCount(1); + await expectChartRowSettled(page); +} + +/** + * The dashboard's chart row has swapped every loading shell for its chart. + * + * The charts are one lazily loaded chunk, and each cell holds a skeleton until + * it lands and the row reveals. The swap changes the row's height, so a page + * measured before it is not the page the reader ends up with: on a loaded + * runner the chunk arrived a second after the dashboard painted, after + * `scrollMainToEnd` had already seen two equal heights, and `
` was no + * longer at its end when it was measured. Waited for on the cells' own state + * rather than on time. + */ +async function expectChartRowSettled(page: Page) { + await expect + .poll( + () => + page.evaluate(() => { + const main = document.getElementById("main-content")!; + const cells = main.querySelectorAll( + '[data-slot="dashboard-chart-cell"]', + ); + const held = main.querySelectorAll( + '[data-slot="dashboard-chart-cell"]:not([data-revealed="true"])', + ); + const shells = main.querySelectorAll('[data-slot="chart-skeleton"]'); + return { + cells: cells.length > 0, + held: held.length, + shells: shells.length, + }; + }), + { timeout: 15_000 }, + ) + .toEqual({ cells: true, held: 0, shells: 0 }); } /** The shell fills the window exactly and the document has nothing to scroll. */ diff --git a/src/app/insights/mood/page-client.tsx b/src/app/insights/mood/page-client.tsx index 00c75e12d..fa0f048cc 100644 --- a/src/app/insights/mood/page-client.tsx +++ b/src/app/insights/mood/page-client.tsx @@ -12,7 +12,10 @@ import { queryKeys } from "@/lib/query-keys"; import { useTranslations } from "@/lib/i18n/context"; import { useInsightsLayoutPrefs } from "@/hooks/use-insights-layout-prefs"; import { Button } from "@/components/ui/button"; -import { ChartSkeleton } from "@/components/charts/chart-skeleton"; +import { + ChartSkeleton, + MoodChartSkeleton, +} from "@/components/charts/chart-skeleton"; import { MetricEmptyState } from "@/components/insights/metric-empty-state"; import { MetricTargetSummary } from "@/components/insights/metric-target-summary"; import { @@ -54,7 +57,7 @@ const MoodChart = dynamic( import("@/components/charts/chart-runtime").then((mod) => ({ default: mod.MoodChart, })), - { ssr: false, loading: () => }, + { ssr: false, loading: () => }, ); /** diff --git a/src/components/charts/chart-skeleton.tsx b/src/components/charts/chart-skeleton.tsx index f160a07b3..828c24630 100644 --- a/src/components/charts/chart-skeleton.tsx +++ b/src/components/charts/chart-skeleton.tsx @@ -2,6 +2,7 @@ import { useEffect, useState } from "react"; +import { Card, CardContent, CardHeader } from "@/components/ui/card"; import { Skeleton } from "@/components/ui/skeleton"; import { cn } from "@/lib/utils"; import { useTranslations } from "@/lib/i18n/context"; @@ -50,12 +51,16 @@ const SLOW_HINT_DELAY_MS = 3_000; /** * The footer a loaded `` paints under its plot: the row of day - * dots and its caption on a day-linked chart, and the collapsed "show as - * table" disclosure. A loading shell without it was ~100 px shorter than the - * card it turned into (CLS on every metric page, the BMI page most visibly), - * so the shells reserve the same boxes: `h-3` rug, `mt-1` caption (two lines - * on a phone, one from `sm`), and the disclosure's `mt-3 border-t pt-3` frame - * around its `min-h-11` button. + * dots on a day-linked chart, and the collapsed "show as table" disclosure. A + * loading shell without it was ~100 px shorter than the card it turned into + * (CLS on every metric page, the BMI page most visibly), so the shells reserve + * the same boxes: the `h-3` rug, and the disclosure's `mt-3 border-t pt-3` + * frame around its `min-h-11` button. + * + * The rug is all a day-linked chart paints under its plot. It used to carry a + * one-line caption as well, and the reserve kept that line after the caption + * went: every day-linked shell stood 20 px taller than its chart (40 px on a + * phone) and the page shrank under the reader when the chart arrived. */ export function ChartFooterReserve({ dayLinks = false, @@ -67,12 +72,7 @@ export function ChartFooterReserve({ if (!dayLinks && !dataTable) return null; return (