Merge pull request #1385 from MBombeck/release/v1.47.6

release: v1.47.6
This commit is contained in:
Marc-André Bombeck authored and GitHub committed 2026-11-15 11:34:48 +01:00
commit 93f68915db
45 files changed
+1068 -133

No files matched your search

+31 -5
View File
@@ -2,6 +2,8 @@
## [Unreleased]
## [1.47.6] — 2026-11-15
### Added
- **A note on a medication.** Directions such as "30 to 60 minutes before
@@ -42,11 +44,35 @@
links the help page. Brave keeps push messaging off by default, so in
Brave the message names the setting to turn on: "Use Google services
for push messaging" under brave://settings/privacy. (Discussion #1378)
- **Illness episode onset dates keep the intended calendar day.**
Creating a new illness episode in the morning was refused because its noon
default was rejected as in the future, and onset dates could display as the
wrong calendar day in non-UTC time zones. Saving now anchors to the current
time when before local noon and preserves stored timestamps on edit.
- **Illness episodes can be added in the morning.** The form sent the
start date as noon, which before noon is still in the future, so the
save was refused. Start and end dates now also stay on the day you
picked when your time zone is far from UTC, and editing an episode
keeps the times you saved. (#1376)
- **No nightly "job still failing" alert over an AI provider that cannot
answer.** An account that still had a provider saved whose key the
provider refused (removing the key from a Local or OpenAI-compatible
provider keeps its address, so it is still called), or whose providers
HealthLog had paused after repeated errors, counted as a failed
briefing every night. On an instance with one account that read as a
broken job. A refused key of the account's own is now noted but no
longer fails the run; a refused shared server key still does. An
account whose providers are all paused is skipped, since nothing was
sent. The Background jobs card on the admin status page now lists the
same runs the alert counts, and both say in words what failed and
which provider. (#1384)
- **Switching records with HealthLog open in a second tab.** Switching
into or out of a shared record could leave both tabs on the loading
spinner, and a reload within the next half minute did not help. An
account check the switching tab had started just before the switch
could answer for the record being left and undo the switch the other
tabs were waiting for. Such an answer is now thrown away and asked
again.
- **Charts no longer jump while they load.** The placeholder of every
chart that links its days was 20 px taller than the chart it turned
into (40 px on a phone), and the mood chart's placeholder was about
30 px shorter, so the dashboard and the metric pages shifted under you
when the charts arrived.
### Security
+1 -1
View File
@@ -1,7 +1,7 @@
openapi: 3.1.0
info:
title: HealthLog API
version: 1.47.5
version: 1.47.6
description: >-
Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest.
+36
View File
@@ -54,6 +54,42 @@ async function openDashboard(page: Page) {
).toBeVisible();
// The module-gated destinations join the rail once `/api/auth/me` lands.
await expect(page.locator('aside nav a[href="/medications"]')).toHaveCount(1);
await expectChartRowSettled(page);
}
/**
* The dashboard's chart row has swapped every loading shell for its chart.
*
* The charts are one lazily loaded chunk, and each cell holds a skeleton until
* it lands and the row reveals. The swap changes the row's height, so a page
* measured before it is not the page the reader ends up with: on a loaded
* runner the chunk arrived a second after the dashboard painted, after
* `scrollMainToEnd` had already seen two equal heights, and `<main>` was no
* longer at its end when it was measured. Waited for on the cells' own state
* rather than on time.
*/
async function expectChartRowSettled(page: Page) {
await expect
.poll(
() =>
page.evaluate(() => {
const main = document.getElementById("main-content")!;
const cells = main.querySelectorAll(
'[data-slot="dashboard-chart-cell"]',
);
const held = main.querySelectorAll(
'[data-slot="dashboard-chart-cell"]:not([data-revealed="true"])',
);
const shells = main.querySelectorAll('[data-slot="chart-skeleton"]');
return {
cells: cells.length > 0,
held: held.length,
shells: shells.length,
};
}),
{ timeout: 15_000 },
)
.toEqual({ cells: true, held: 0, shells: 0 });
}
/** The shell fills the window exactly and the document has nothing to scroll. */
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Hintergrundjobs",
"description": "Queues, deren Jobs nach allen Wiederholungen aufgegeben haben.",
"description": "Queues, deren Jobs nach allen Wiederholungen aufgaben oder deren Lauf bei jedem versuchten Konto scheiterte.",
"none": "In den letzten {hours} Stunden ist kein Job fehlgeschlagen.",
"unavailable": "Keine Job-Queue lesbar. Dieses Deployment läuft ohne Hintergrund-Worker.",
"countOne": "{count} Fehlschlag",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Background jobs",
"description": "Queues whose jobs gave up after exhausting their retries.",
"description": "Queues whose jobs gave up after their retries, or whose run ended with every account it tried failing.",
"none": "No job has failed in the last {hours} hours.",
"unavailable": "No job queue to read. This deployment runs without the background worker.",
"countOne": "{count} failure",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Trabajos en segundo plano",
"description": "Colas cuyos trabajos se rindieron tras agotar sus reintentos.",
"description": "Colas cuyos trabajos se rindieron tras sus reintentos o cuya ejecución falló en cada cuenta intentada.",
"none": "Ningún trabajo ha fallado en las últimas {hours} horas.",
"unavailable": "No hay ninguna cola que consultar: este despliegue funciona sin el worker en segundo plano.",
"countOne": "{count} fallo",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Tâches en arrière-plan",
"description": "Files dont les tâches ont abandonné après avoir épuisé leurs tentatives.",
"description": "Files dont les tâches ont abandonné après leurs tentatives, ou dont l’exécution a échoué pour chaque compte.",
"none": "Aucune tâche n’a échoué au cours des {hours} dernières heures.",
"unavailable": "Aucune file à consulter : ce déploiement fonctionne sans le worker en arrière-plan.",
"countOne": "{count} échec",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Processi in background",
"description": "Code i cui processi hanno rinunciato dopo aver esaurito i tentativi.",
"description": "Code i cui processi hanno rinunciato dopo i tentativi o la cui esecuzione è fallita per ogni account.",
"none": "Nessun processo è fallito nelle ultime {hours} ore.",
"unavailable": "Nessuna coda da leggere: questo deployment funziona senza il worker in background.",
"countOne": "{count} errore",
+1 -1
View File
@@ -8743,7 +8743,7 @@
},
"failingJobs": {
"title": "백그라운드 작업",
"description": "재시도를 다 쓰고 포기한 작업이 있는 큐예요.",
"description": "재시도를 다 쓰고 포기했거나, 실행에서 시도한 모든 계정이 실패한 큐예요.",
"none": "최근 {hours}시간 동안 실패한 작업이 없어요.",
"unavailable": "읽을 작업 큐가 없어요. 이 배포본은 백그라운드 워커 없이 돌고 있어요.",
"countOne": "실패 {count}건",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Achtergrondtaken",
"description": "Wachtrijen waarvan taken het hebben opgegeven nadat alle nieuwe pogingen waren verbruikt.",
"description": "Wachtrijen waarvan taken na alle pogingen opgaven, of waarvan een run voor elk geprobeerd account mislukte.",
"none": "Er is de afgelopen {hours} uur geen taak mislukt.",
"unavailable": "Geen takenwachtrij om te lezen. Deze installatie draait zonder de achtergrondworker.",
"countOne": "{count} fout",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Zadania w tle",
"description": "Kolejki, których zadania poddały się po wyczerpaniu ponowień.",
"description": "Kolejki, których zadania poddały się po ponowieniach albo których przebieg zawiódł dla każdego konta.",
"none": "W ciągu ostatnich {hours} godzin żadne zadanie nie zawiodło.",
"unavailable": "Brak kolejki zadań do odczytu: to wdrożenie działa bez workera w tle.",
"countOne": "{count} niepowodzenie",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Tarefas em segundo plano",
"description": "Filas cujas tarefas desistiram depois de esgotar as tentativas.",
"description": "Filas cujas tarefas desistiram após as tentativas, ou cuja execução falhou em todas as contas tentadas.",
"none": "Nenhuma tarefa falhou nas últimas {hours} horas.",
"unavailable": "Nenhuma fila de tarefas para ler. Esta implantação roda sem o worker em segundo plano.",
"countOne": "{count} falha",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Фоновые задания",
"description": "Очереди, задания которых сдались после исчерпания повторных попыток.",
"description": "Очереди, задания которых сдались после всех попыток или чей запуск не удался ни для одного аккаунта.",
"none": "За последние {hours} часов ни одно задание не завершилось неудачей.",
"unavailable": "Нет очереди заданий для чтения. Это развёртывание работает без фонового воркера.",
"countOne": "{count} неудача",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "healthlog",
"version": "1.47.5",
"version": "1.47.6",
"description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.",
"license": "PolyForm-Noncommercial-1.0.0",
"homepage": "https://healthlog.dev",
+1 -1
View File
@@ -16,7 +16,7 @@ try {
}
const CACHE_VERSION =
(typeof self !== "undefined" && self.__APP_VERSION__) ||
/* @sw-version-fallback */ "v1.47.5";
/* @sw-version-fallback */ "v1.47.6";
const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`;
const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`;
// Every HealthLog cache, current and legacy, and nothing another app on the
@@ -1,7 +1,7 @@
{
"$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
"count": 84,
"total": 93816,
"total": 93819,
"entries": {
"app/medications/page-client.tsx": 842,
"components/charts/mood-chart.tsx": 1166,
@@ -64,7 +64,7 @@
"lib/insights/mood-aggregates.ts": 1053,
"lib/integrations/status.ts": 1448,
"lib/jobs/coach-nudge.ts": 1084,
"lib/jobs/insight-pregenerate.ts": 1189,
"lib/jobs/insight-pregenerate.ts": 1192,
"lib/jobs/medication-intake-import.ts": 819,
"lib/jobs/offhost-backup.ts": 1094,
"lib/jobs/reaction-line.ts": 803,
+1 -1
View File
@@ -435,7 +435,7 @@
"lib/jobs/__tests__/document-summary-reaper.test.ts": 1,
"lib/jobs/__tests__/document-summary.test.ts": 19,
"lib/jobs/__tests__/geo-backfill.test.ts": 1,
"lib/jobs/__tests__/insight-pregenerate.test.ts": 45,
"lib/jobs/__tests__/insight-pregenerate.test.ts": 43,
"lib/jobs/__tests__/insight-status-generate.test.ts": 2,
"lib/jobs/__tests__/measurement-maintenance-schedule.test.ts": 2,
"lib/jobs/__tests__/measurement-reminder-byhour.test.ts": 2,
@@ -1,7 +1,7 @@
{
"$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.",
"count": 2863,
"total": 9122,
"total": 9118,
"entries": {
"__tests__/account-payload-consumer-guard.test.ts": 2,
"__tests__/acting-account-boundary-guard.test.ts": 7,
@@ -2134,7 +2134,7 @@
"lib/insights/chart-tokens.ts": 24,
"lib/insights/coach-launch-context.tsx": 10,
"lib/insights/comprehensive-aggregator.ts": 27,
"lib/insights/comprehensive-generate.ts": 45,
"lib/insights/comprehensive-generate.ts": 44,
"lib/insights/correlation-channel-series.ts": 16,
"lib/insights/correlation-discovery.ts": 30,
"lib/insights/correlation-series-builders.ts": 1,
@@ -2299,7 +2299,7 @@
"lib/jobs/google-health-oauth-state-cleanup.ts": 1,
"lib/jobs/google-health-sleep-repair.ts": 1,
"lib/jobs/insight-pregenerate-shared.ts": 1,
"lib/jobs/insight-pregenerate.ts": 36,
"lib/jobs/insight-pregenerate.ts": 33,
"lib/jobs/insight-status-generate-shared.ts": 3,
"lib/jobs/insight-status-generate.ts": 3,
"lib/jobs/intake-auto-skip.ts": 8,
@@ -168,6 +168,11 @@ const ALLOWED: Readonly<Record<string, { count: number; reason: string }>> = {
reason:
"No pgboss schema on a web-only deployment; null means no queue to ask, and the evaluation then judges nothing.",
},
"lib/jobs/job-failures.ts::explainAllFailedRun": {
count: 1,
reason:
"A pg-boss output that does not parse as a run's outcome is not an all-failed run; presentError then shows the raw text as it always did.",
},
"lib/jobs/job-failures.ts::readActiveJobsStartedBefore": {
count: 1,
reason:
+5 -2
View File
@@ -12,7 +12,10 @@ import { queryKeys } from "@/lib/query-keys";
import { useTranslations } from "@/lib/i18n/context";
import { useInsightsLayoutPrefs } from "@/hooks/use-insights-layout-prefs";
import { Button } from "@/components/ui/button";
import { ChartSkeleton } from "@/components/charts/chart-skeleton";
import {
ChartSkeleton,
MoodChartSkeleton,
} from "@/components/charts/chart-skeleton";
import { MetricEmptyState } from "@/components/insights/metric-empty-state";
import { MetricTargetSummary } from "@/components/insights/metric-target-summary";
import {
@@ -54,7 +57,7 @@ const MoodChart = dynamic(
import("@/components/charts/chart-runtime").then((mod) => ({
default: mod.MoodChart,
})),
{ ssr: false, loading: () => <ChartSkeleton /> },
{ ssr: false, loading: () => <MoodChartSkeleton /> },
);
/**
+52 -12
View File
@@ -2,6 +2,7 @@
import { useEffect, useState } from "react";
import { Card, CardContent, CardHeader } from "@/components/ui/card";
import { Skeleton } from "@/components/ui/skeleton";
import { cn } from "@/lib/utils";
import { useTranslations } from "@/lib/i18n/context";
@@ -50,12 +51,16 @@ const SLOW_HINT_DELAY_MS = 3_000;
/**
* The footer a loaded `<HealthChart>` paints under its plot: the row of day
* dots and its caption on a day-linked chart, and the collapsed "show as
* table" disclosure. A loading shell without it was ~100 px shorter than the
* card it turned into (CLS on every metric page, the BMI page most visibly),
* so the shells reserve the same boxes: `h-3` rug, `mt-1` caption (two lines
* on a phone, one from `sm`), and the disclosure's `mt-3 border-t pt-3` frame
* around its `min-h-11` button.
* dots on a day-linked chart, and the collapsed "show as table" disclosure. A
* loading shell without it was ~100 px shorter than the card it turned into
* (CLS on every metric page, the BMI page most visibly), so the shells reserve
* the same boxes: the `h-3` rug, and the disclosure's `mt-3 border-t pt-3`
* frame around its `min-h-11` button.
*
* The rug is all a day-linked chart paints under its plot. It used to carry a
* one-line caption as well, and the reserve kept that line after the caption
* went: every day-linked shell stood 20 px taller than its chart (40 px on a
* phone) and the page shrank under the reader when the chart arrived.
*/
export function ChartFooterReserve({
dayLinks = false,
@@ -67,12 +72,7 @@ export function ChartFooterReserve({
if (!dayLinks && !dataTable) return null;
return (
<div aria-hidden="true" data-slot="chart-footer-reserve">
{dayLinks ? (
<>
<div className="h-3" />
<div className="mt-1 h-9 sm:h-4" />
</>
) : null}
{dayLinks ? <div className="h-3" /> : null}
{dataTable ? (
<div className="border-border mt-3 border-t pt-3">
<div className="h-11" />
@@ -172,3 +172,43 @@ export function ChartSkeleton({
</div>
);
}
/**
* The loading shell for a full-size, day-linked `<MoodChart>`.
*
* The mood chart is drawn in a `Card` (vertical padding and a gap between
* header and plot, where `ChartSkeleton` has padding and a margin), its plot
* band is 200 / 220 px rather than 240 / 280, and it paints the day rug and
* the collapsed data-table disclosure under the plot. Shown in the generic
* shell, the card grew by about 30 px when the chart chunk arrived, under a
* reader already scrolled past it. So it gets the same primitives and the
* same boxes.
*/
export function MoodChartSkeleton() {
const { t } = useTranslations();
return (
<Card
data-slot="chart-skeleton"
role="status"
aria-busy="true"
aria-live="polite"
>
<span className="sr-only">{t("charts.loadingLabel")}</span>
<CardHeader>
<div className="flex flex-col gap-2 sm:flex-row sm:items-center sm:justify-between">
<Skeleton className="h-5 w-32" />
<div className="flex h-11 items-center gap-2">
<Skeleton className="h-6 w-12" />
<Skeleton className="h-6 w-12" />
<Skeleton className="h-6 w-12" />
<Skeleton className="h-6 w-12" />
</div>
</div>
</CardHeader>
<CardContent>
<Skeleton className="h-[var(--chart-height,200px)] w-full md:h-[var(--chart-height-md,220px)]" />
<ChartFooterReserve dayLinks dataTable />
</CardContent>
</Card>
);
}
@@ -4,7 +4,10 @@ import type { ComponentProps, ReactNode } from "react";
import dynamic from "next/dynamic";
import { ChartErrorBoundary } from "@/components/charts/chart-error-state";
import { ChartSkeleton } from "@/components/charts/chart-skeleton";
import {
ChartSkeleton,
MoodChartSkeleton,
} from "@/components/charts/chart-skeleton";
import { HealthChartDynamic } from "@/components/charts/health-chart-dynamic";
import { RecentWorkoutsTile } from "@/components/dashboard/recent-workouts-tile";
import { RecentAchievementsCard } from "@/components/gamification/recent-achievements-card";
@@ -26,7 +29,7 @@ const MoodChartLazy = dynamic(
importWithRetry(() => import("@/components/charts/chart-runtime")).then(
(mod) => ({ default: mod.MoodChart }),
),
{ ssr: false, loading: () => <ChartSkeleton /> },
{ ssr: false, loading: () => <MoodChartSkeleton /> },
);
const MoodChart = (props: ComponentProps<typeof MoodChartLazy>) => (
<ChartErrorBoundary>
@@ -0,0 +1,149 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
__resetRecordFenceForTests,
currentRecordFenceState,
} from "@/lib/api/record-fence";
import {
__resetRecordScopeForTests,
getRecordScope,
setRecordScope,
} from "@/lib/query-keys/record-scope";
import {
__resetRecordSessionTransitionForTests,
beginRecordSessionTransition,
commitRecordSessionTransition,
getRecordSessionTransition,
} from "@/lib/query-keys/record-session-transition";
const apiGet = vi.fn();
vi.mock("@/lib/api/api-fetch", () => ({
apiGet: (...args: unknown[]) => apiGet(...args),
apiFetchRaw: vi.fn(),
ApiError: class ApiError extends Error {},
}));
vi.mock("next/navigation", () => ({
useRouter: () => ({ push: vi.fn(), replace: vi.fn() }),
}));
import { fetchMe } from "../use-auth";
/**
* A `/api/auth/me` answered across a record-session transition.
*
* The browser journey that found this: the switcher clears the query cache
* when it raises the hold, which remounts the account query, so the initiating
* tab has a `/me` in flight beside its own switch POST. The server answers it
* with the record being left. Applied after the commit, that answer rewrote the
* scope mirror to the old record, a settled peer read the write as a foreign
* switch, and every tab then waited for a `/me` naming a record the server had
* already left — behind the hydration gate, for good.
*/
const TARGET = "record-b";
const entry = {
accountId: TARGET,
username: "record-b",
displayName: null,
fullName: null,
access: "read",
level: "read",
recordKind: "shared",
sections: null,
canWrite: false,
writableDomains: [],
manageableDomains: [],
};
const base = {
id: "delegate",
username: "delegate",
role: "USER",
timezone: "Europe/Berlin",
};
/** What the server says before the switch lands: the caller's own record. */
const beforeSwitch = {
...base,
accountAccess: {
accounts: [entry],
active: null,
recordKind: "self",
canSwitch: true,
},
recordSession: { epoch: 4, scope: null },
};
/** And after it. */
const afterSwitch = {
...base,
accountAccess: {
accounts: [entry],
active: entry,
recordKind: "shared",
canSwitch: true,
},
recordSession: { epoch: 5, scope: TARGET },
};
beforeEach(() => {
apiGet.mockReset();
__resetRecordScopeForTests();
__resetRecordSessionTransitionForTests();
__resetRecordFenceForTests();
});
describe("fetchMe across a record-session transition", () => {
it("drops an answer that straddled the commit and reads again", async () => {
setRecordScope(null);
const id = beginRecordSessionTransition(TARGET);
apiGet
.mockImplementationOnce(async () => {
// The switch commits while this read is on the wire, and the server
// had already answered it from the record being left.
commitRecordSessionTransition(id, TARGET);
return beforeSwitch;
})
.mockResolvedValueOnce(afterSwitch);
const user = await fetchMe();
expect(apiGet).toHaveBeenCalledTimes(2);
expect(user.accountAccess?.active?.accountId).toBe(TARGET);
// The committed scope survives; the pre-switch answer never reached the
// mirror or the fence.
expect(getRecordScope()).toBe(TARGET);
expect(currentRecordFenceState()).toEqual({ epoch: 5, scope: TARGET });
// And the hold releases on the read that describes the switched session.
expect(getRecordSessionTransition().phase).toBe("ready");
});
it("applies an answer when nothing moved while it was in flight", async () => {
const id = beginRecordSessionTransition(TARGET);
commitRecordSessionTransition(id, TARGET);
apiGet.mockResolvedValueOnce(afterSwitch);
await fetchMe();
expect(apiGet).toHaveBeenCalledTimes(1);
expect(getRecordScope()).toBe(TARGET);
expect(getRecordSessionTransition().phase).toBe("ready");
});
it("gives up without applying anything when the session never holds still", async () => {
setRecordScope(null);
apiGet.mockImplementation(async () => {
// Every read straddles a fresh transition.
beginRecordSessionTransition(TARGET);
return beforeSwitch;
});
await expect(fetchMe()).rejects.toThrow(/kept moving/);
expect(apiGet).toHaveBeenCalledTimes(4);
expect(currentRecordFenceState()).toBeNull();
});
});
+42 -8
View File
@@ -36,6 +36,7 @@ import {
setRefusedRecordScope,
} from "@/lib/query-keys/record-scope";
import {
recordSessionRevision,
settleRecordSessionTransition,
settleRefusedRecordSessionTransition,
} from "@/lib/query-keys/record-session-transition";
@@ -361,6 +362,46 @@ export function isAuthVerdictUnknown(error: unknown): boolean {
return true;
}
/**
* How many times `fetchMe` asks again when a record-session transition moved
* while its request was in flight. Each repeat needs another transition inside
* one round trip, which only a browser switching records over and over
* produces.
*/
const SUPERSEDED_ME_RETRIES = 3;
type MePayload = Partial<AuthUser> & {
id: string;
username: string;
role: string;
timezone: string;
};
/**
* `GET /api/auth/me`, never answered across a record-session transition.
*
* Everything `fetchMe` does with the payload is a statement about which record
* this browser is in: it adopts the fence context, rewrites the cache-scope
* mirror every other tab watches, and may release the hold. A request issued
* before a transition and answered after it may have been read on the far side
* of a switch, and applying it is how a browser ended up wedged behind the
* hydration gate (see `recordSessionRevision`). Such an answer is dropped and
* the read repeated; the repeat is issued after the change, so it describes the
* session as it now is.
*/
async function getCurrentMe(): Promise<MePayload> {
for (let attempt = 0; ; attempt++) {
const before = recordSessionRevision();
const data = await apiGet<MePayload>("/api/auth/me");
if (recordSessionRevision() === before) return data;
if (attempt >= SUPERSEDED_ME_RETRIES) {
// Not an `ApiError`: no verdict on the session, so nothing is wiped and
// the hold stays until a later read settles it.
throw new Error("record session kept moving during /api/auth/me");
}
}
}
export async function fetchMe(): Promise<AuthUser> {
// v1.16.4 — routed through the typed wrapper; a non-OK /me (401)
// throws `ApiError`, which `useAuth` treats as "not authenticated"
@@ -369,14 +410,7 @@ export async function fetchMe(): Promise<AuthUser> {
// v1.4.47 — coerce `disableCoach` against `undefined` so a stale
// /me payload from a partial-deploy rollback (older server image
// without the field) keeps the Coach surface visible by default.
const data = await apiGet<
Partial<AuthUser> & {
id: string;
username: string;
role: string;
timezone: string;
}
>("/api/auth/me");
const data = await getCurrentMe();
// Hour-cycle preference: coerce against a stale /me payload, then mirror
// into localStorage so `useFormatters()` and the legacy format helpers
// (which cannot reach the query cache) render the same clock.
@@ -209,4 +209,59 @@ describe("comprehensive warm-path timeout + failure observability", () => {
(failAnnotation![0] as { meta?: { reason?: string } }).meta?.reason,
).toBe("all-providers-failed");
});
// #1384 — the nightly pass needs to know which configured provider failed,
// whose credential a refusal was, and whether anything was sent at all.
it("names the provider that failed first and whether its credential was refused", async () => {
runRawCompletionWithFallback.mockRejectedValue(
Object.assign(new AllProvidersFailedError([]), {
attempts: [
{
providerType: "openai-compatible",
attempt: 1,
failureReason: "401",
httpStatus: 401,
},
],
primaryCredentialExpired: true,
sentNothing: false,
}),
);
const outcome = await generateComprehensiveInsight("u1", { locale: "de" });
expect(outcome).toEqual({
status: "failed",
reason: "all-providers-failed",
credentialExpired: true,
provider: "openai-compatible",
});
});
it("says so when every provider was benched and nothing was sent (#1279)", async () => {
runRawCompletionWithFallback.mockRejectedValue(
Object.assign(new AllProvidersFailedError([]), {
attempts: [
{
providerType: "local",
attempt: 1,
failureReason: "benched-for-background-work",
httpStatus: null,
failureKind: "paused",
},
],
primaryCredentialExpired: false,
sentNothing: true,
}),
);
const outcome = await generateComprehensiveInsight("u1", { locale: "de" });
expect(outcome).toEqual({
status: "failed",
reason: "all-providers-failed",
provider: "local",
sentNothing: true,
});
});
});
@@ -134,6 +134,7 @@ describe("generateStatusBatchForUser — the statusText capability comes first",
served: 0,
batched: 0,
fellBack: 0,
fellBackAnswered: 0,
batchCallMade: false,
});
});
@@ -340,6 +341,15 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => {
vi.mocked(runStatusCompletion).mockResolvedValue({
kind: "error",
} as never);
// #1384 — one card's own call came back with a model note; the other six
// served their fallback. A nightly warm over a provider that only failed
// reported `assessments_warmed: 6`, because every fallback counted.
vi.mocked(runPreparedStatusCard).mockResolvedValueOnce({
hasProvider: true,
text: "note",
cached: false,
updatedAt: "t",
});
const result = await generateStatusBatchForUser("u1", { locale: "en" });
@@ -347,6 +357,7 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => {
for (const f of finalizers) expect(f).not.toHaveBeenCalled();
expect(runPreparedStatusCard).toHaveBeenCalledTimes(7);
expect(result.fellBack).toBe(7);
expect(result.fellBackAnswered).toBe(1);
expect(result.batched).toBe(0);
});
@@ -411,6 +422,7 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => {
for (const f of finalizers) expect(f).not.toHaveBeenCalled();
expect(runPreparedStatusCard).not.toHaveBeenCalled();
expect(result.fellBack).toBe(7);
expect(result.fellBackAnswered).toBe(0);
expect(result.batchCallMade).toBe(false);
});
});
+37
View File
@@ -0,0 +1,37 @@
/**
* What a failed briefing generation knows about its provider chain beyond
* the reason (#1384): which configured provider failed first, whether that
* provider refused the credential, and whether anything was sent at all.
*
* The nightly pass needs all three to settle an account honestly. A refusal
* of the account's own credential is the account's matter, a refusal of the
* operator's shared key is the operator's, and a chain whose every entry the
* provider health ledger benched for background work (#1279) sent no request,
* so the account was not attempted. Without them every such account read as
* a failure, night after night.
*/
import { AllProvidersFailedError } from "@/lib/ai/provider-runner";
export interface ChainFailureFacts {
/**
* The chain's first provider refused the credential (a dead Codex
* sign-in, a revoked key). No retry can succeed until someone reconnects
* or replaces it.
*/
credentialExpired?: boolean;
/** The chain entry that failed first, e.g. `openai-compatible`. */
provider?: string;
/** Every entry was benched for background work; no request left. */
sentNothing?: boolean;
}
/** The facts of a thrown chain failure; empty for any other error. */
export function chainFailureFacts(error: unknown): ChainFailureFacts {
if (!(error instanceof AllProvidersFailedError)) return {};
const provider = error.attempts?.[0]?.providerType;
return {
...(error.primaryCredentialExpired ? { credentialExpired: true } : {}),
...(provider ? { provider } : {}),
...(error.sentNothing ? { sentNothing: true } : {}),
};
}
+6 -16
View File
@@ -93,6 +93,10 @@ import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic";
import { insightResultSchema, type InsightResult } from "@/lib/ai/types";
import { resolveProviderChain } from "@/lib/ai/provider";
import { AllProvidersFailedError } from "@/lib/ai/provider-runner";
import {
chainFailureFacts,
type ChainFailureFacts,
} from "@/lib/insights/chain-failure-facts";
import {
BriefingBudgetExceededError,
runBriefingCompletion,
@@ -178,17 +182,7 @@ export type GenerateOutcome =
| "scope-changed"
| "profile-scope-changed";
}
| {
status: "failed";
reason: string;
/**
* v1.42 — the chain's first provider refused the credential (a dead
* Codex sign-in, a revoked key). The nightly pass reports the account
* as `auth_failed` and does not queue a retry that cannot succeed until
* the person reconnects.
*/
credentialExpired?: boolean;
};
| ({ status: "failed"; reason: string } & ChainFailureFacts);
/** Higher, seedless temperature for the daily-briefing phrasing re-roll. */
const BRIEFING_REROLL_TEMPERATURE = 0.6;
@@ -1290,11 +1284,7 @@ export async function generateComprehensiveInsight(
? (e.attempts?.[0]?.failureKind ?? null)
: providerFailureKind(e).failure,
});
const credentialExpired =
e instanceof AllProvidersFailedError && e.primaryCredentialExpired;
return credentialExpired
? { status: "failed", reason, credentialExpired }
: { status: "failed", reason };
return { status: "failed", reason, ...chainFailureFacts(e) };
}
// Anthropic + local have no native JSON mode, so a ```json-fenced or
+39 -6
View File
@@ -37,6 +37,7 @@ import {
runPreparedStatusCard,
type PendingStatusCard,
type PreparedStatusCard,
type StatusCardResult,
} from "@/lib/insights/status-card-generation";
import { runStatusCompletion } from "@/lib/insights/status-provider";
import {
@@ -77,10 +78,27 @@ export interface StatusBatchResult {
/** Metrics that fell back to a single-card call (omitted by the batch, or
* the batch failed). */
fellBack: number;
/**
* Of `fellBack`, the metrics whose own single-card call came back with a
* model-written assessment. The rest served their deterministic fallback:
* no provider, a failed or cooling-down provider, or a screened reply.
* `batched + fellBackAnswered` is what a warm pass may call warmed (#1384);
* `fellBack` alone counted a night of six failed calls as six assessments.
*/
fellBackAnswered: number;
/** True when the one batched provider call was issued. */
batchCallMade: boolean;
}
/**
* Whether a single-card run ended in a model-written assessment: the persisted
* shape is the only one with a provider and no cache flag. The no-key, the
* timeout / error stub and the screened fallback all carry `cached: true`.
*/
function modelAnswered(result: StatusCardResult): boolean {
return result.hasProvider === true && result.cached === false;
}
/** Pull `perMetric` out of a (possibly fenced) model completion. */
function parsePerMetric(content: string): Record<string, string> | null {
const tryParse = (raw: string): Record<string, string> | null => {
@@ -119,6 +137,7 @@ export async function generateStatusBatchForUser(
served: 0,
batched: 0,
fellBack: 0,
fellBackAnswered: 0,
batchCallMade: false,
};
@@ -171,11 +190,19 @@ export async function generateStatusBatchForUser(
// One metric still needing the LLM is not worth a batch envelope — run it
// through its own single-card path (identical persisted result).
if (pending.length === 1) {
await runPreparedStatusCard(pending[0]);
if (modelAnswered(await runPreparedStatusCard(pending[0]))) {
result.fellBackAnswered++;
}
result.fellBack++;
annotate({
action: { name: "insights.status.batch" },
meta: { served: result.served, batched: 0, fell_back: 1, single: true },
meta: {
served: result.served,
batched: 0,
fell_back: 1,
fell_back_answered: result.fellBackAnswered,
single: true,
},
});
return result;
}
@@ -272,14 +299,19 @@ export async function generateStatusBatchForUser(
// Batch missed this metric (omitted key, or timeout/error/invalid-json):
// fall back to the single-card path. On a batch timeout/error this writes
// the per-card negative stub via the card's own `timeout`.
await runPreparedStatusCard(card);
return "fellBack" as const;
return modelAnswered(await runPreparedStatusCard(card))
? ("fellBackAnswered" as const)
: ("fellBack" as const);
});
const outcomes = await Promise.all(fanOuts);
for (const o of outcomes) {
if (o === "batched") result.batched++;
else result.fellBack++;
if (o === "batched") {
result.batched++;
continue;
}
result.fellBack++;
if (o === "fellBackAnswered") result.fellBackAnswered++;
}
annotate({
@@ -288,6 +320,7 @@ export async function generateStatusBatchForUser(
served: result.served,
batched: result.batched,
fell_back: result.fellBack,
fell_back_answered: result.fellBackAnswered,
pending: pending.length,
},
});
@@ -103,6 +103,7 @@ vi.mock("@/lib/insights/status-batch", () => ({
served: 0,
batched: 0,
fellBack: 0,
fellBackAnswered: 0,
batchCallMade: false,
})),
}));
@@ -132,6 +133,7 @@ import {
INSIGHT_PREGENERATE_EXPIRE_SECONDS,
} from "../insight-pregenerate-shared";
import { handleInsightPregenerateJob } from "../reminder/insights-handlers";
import { judgeJobRun, type reportJobRun } from "../job-run-report";
import { insightRetryOptions } from "../reminder/registrar-shared";
import {
groupOf,
@@ -1556,15 +1558,9 @@ describe("runInsightPregenerate — warm budget honours the response timeout (v1
// withheld (screened) briefing and a dead sign-in queue no provider retry,
// and a time-budget cut-off reports the unreached accounts as deferred.
describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
type Reported = {
queue: string;
runId: string;
candidates: Array<{ key: string; outcome: string; cause?: string }>;
};
function harness(users: Array<{ id: string; locale: string }>) {
const { prisma } = makePrisma(users);
const reportRun = vi.fn(async (input: Reported) => ({
const reportRun = vi.fn<typeof reportJobRun>(async (input) => ({
total: input.candidates.length,
attempted: 0,
failed: 0,
@@ -1606,7 +1602,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
statusGenerators: h.statusGenerators,
warmGenericMetrics: h.warmGenericMetrics,
enqueueRetry: h.enqueueRetry,
reportRun: h.reportRun as never,
reportRun: h.reportRun,
});
expect(h.reportRun).toHaveBeenCalledTimes(1);
@@ -1639,7 +1635,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
statusGenerators: h.statusGenerators,
warmGenericMetrics: h.warmGenericMetrics,
enqueueRetry: h.enqueueRetry,
reportRun: h.reportRun as never,
reportRun: h.reportRun,
});
expect(h.enqueueRetry).toHaveBeenCalledTimes(1);
@@ -1668,7 +1664,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
statusGenerators: h.statusGenerators,
warmGenericMetrics: h.warmGenericMetrics,
enqueueRetry: h.enqueueRetry,
reportRun: h.reportRun as never,
reportRun: h.reportRun,
shouldStop: () => ++calls > 1,
});
@@ -1677,3 +1673,121 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
).toEqual(["ok", "deferred", "deferred"]);
});
});
// #1384 — a self-hoster whose only account kept a provider that no longer
// works was told every night that the job was failing. The run used the real
// verdict here, so what the alert reads (`all_failed`) is what is asserted.
describe("runInsightPregenerate — a provider that cannot serve is not a failing run (#1384)", () => {
async function runOne(outcome: Record<string, unknown>) {
const { prisma } = makePrisma([{ id: "only-account", locale: "en" }]);
const reportRun = vi.fn<typeof reportJobRun>(async (input) =>
judgeJobRun(input.candidates),
);
const enqueueRetry = vi.fn().mockResolvedValue(undefined);
const result = await runInsightPregenerate(prisma as never, {
generate: vi.fn().mockResolvedValue(outcome),
statusGenerators: Array.from({ length: 7 }, () =>
vi.fn().mockResolvedValue({ hasProvider: true, cached: true }),
),
warmGenericMetrics: vi.fn().mockResolvedValue(0),
enqueueRetry,
reportRun,
});
return {
result,
enqueueRetry,
candidate: reportRun.mock.calls[0][0].candidates[0],
};
}
it("skips an account whose every provider was benched, so nothing was sent", async () => {
const { result, enqueueRetry, candidate } = await runOne({
status: "failed",
reason: "all-providers-failed",
provider: "local",
sentNothing: true,
});
expect(candidate).toEqual(
expect.objectContaining({
outcome: "skipped",
cause: "provider_benched",
}),
);
expect(result.skipped).toBe(1);
expect(result.failed).toBe(0);
expect(result.allFailed).toBe(false);
// Nobody attempted: the handler then writes no `all_failed`, so a benched
// night neither extends nor ends the alert's streak.
expect(result.attempted).toBe(0);
expect(result.firstFailure).toBeNull();
// The retry would find the same bench.
expect(enqueueRetry).not.toHaveBeenCalled();
expect(annotateSpy).toHaveBeenCalledWith(
expect.objectContaining({
action: { name: "insights.pregenerate.provider_benched" },
}),
);
});
it("reports a refused credential of the account's own as auth_failed, naming the provider, without failing the run", async () => {
const { result, enqueueRetry, candidate } = await runOne({
status: "failed",
reason: "all-providers-failed",
provider: "openai-compatible",
credentialExpired: true,
});
expect(candidate).toEqual(
expect.objectContaining({
outcome: "auth_failed",
cause: "all-providers-failed:openai-compatible",
}),
);
expect(result.allFailed).toBe(false);
expect(result.attempted).toBe(1);
expect(enqueueRetry).not.toHaveBeenCalled();
expect(annotateSpy).toHaveBeenCalledWith(
expect.objectContaining({
action: { name: "insights.pregenerate.comprehensive_failed" },
meta: expect.objectContaining({
provider: "openai-compatible",
credential_refused: true,
}),
}),
);
});
it("still fails the run when the operator's own key was refused, and says which", async () => {
const { result, enqueueRetry, candidate } = await runOne({
status: "failed",
reason: "all-providers-failed",
provider: "admin-openai",
credentialExpired: true,
});
expect(candidate.outcome).toBe("failed");
expect(result.allFailed).toBe(true);
expect(result.firstFailure).toEqual({
reason: "all-providers-failed",
provider: "admin-openai",
});
// A refused key does not heal in 45 minutes.
expect(enqueueRetry).not.toHaveBeenCalled();
});
it("carries the cause and provider of an ordinary failure into the run's facts", async () => {
const { result, enqueueRetry } = await runOne({
status: "failed",
reason: "provider-error",
provider: "local",
});
expect(result.allFailed).toBe(true);
expect(result.firstFailure).toEqual({
reason: "provider-error",
provider: "local",
});
expect(enqueueRetry).toHaveBeenCalledTimes(1);
});
});
@@ -193,3 +193,36 @@ describe("bounded generic JobOutcome serialization", () => {
});
});
});
// #1384 — a generator reason and a provider type become facts the admin page
// and the repeated-failure alert can show.
describe("codeFact", () => {
it("turns our own hyphenated codes into stable fact codes", async () => {
const { codeFact, serializeJobOutcome } = await vi.importActual<
typeof import("@/lib/jobs/job-outcome")
>("@/lib/jobs/job-outcome");
expect(codeFact("all-providers-failed")).toBe("all_providers_failed");
expect(codeFact("openai-compatible")).toBe("openai_compatible");
expect(codeFact("generator:invalid-json")).toBe("generator_invalid_json");
expect(codeFact("timeout")).toBe("timeout");
// Nothing usable: the caller leaves the fact out.
expect(codeFact("")).toBeNull();
expect(codeFact("---")).toBeNull();
expect(codeFact(null)).toBeNull();
// What it produces is a fact the serializer accepts.
expect(
serializeJobOutcome(
jobDone({
failure_cause: codeFact("all-providers-failed")!,
failure_provider: codeFact("admin-openai")!,
}),
),
).toEqual({
ok: true,
did: {
failure_cause: "all_providers_failed",
failure_provider: "admin_openai",
},
});
});
});
+47 -9
View File
@@ -47,13 +47,27 @@ describe("judgeJobRun", () => {
});
});
it("counts a dead credential as a failure", () => {
expect(
judgeJobRun([
{ key: "a", outcome: "auth_failed" },
{ key: "b", outcome: "failed" },
]).allFailed,
).toBe(true);
// #1384 — a key the issuer revoked stays saved and is refused every night.
// Counted as a failure, it made a single-account instance's nightly pass
// read as all-failed forever and alerted the operator about the job.
it("does not fail a run over a refused credential of the account's own", () => {
expect(judgeJobRun([{ key: "a", outcome: "auth_failed" }])).toEqual({
total: 1,
attempted: 1,
failed: 0,
allFailed: false,
partial: true,
});
});
it("reads a refused credential beside a real failure as partial", () => {
const verdict = judgeJobRun([
{ key: "a", outcome: "auth_failed" },
{ key: "b", outcome: "failed" },
]);
expect(verdict.allFailed).toBe(false);
expect(verdict.partial).toBe(true);
expect(verdict.failed).toBe(1);
});
it("reads one success among failures as partial, not failed", () => {
@@ -84,7 +98,7 @@ describe("reportJobRun", () => {
queue: "insight-pregenerate",
runId: "r1",
candidates: [
{ key: "a", outcome: "auth_failed", cause: "all-providers-failed" },
{ key: "a", outcome: "failed", cause: "all-providers-failed" },
{ key: "b", outcome: "skipped", cause: "no-consent" },
],
});
@@ -101,7 +115,31 @@ describe("reportJobRun", () => {
consecutiveFailures: 3,
failed: 1,
total: 1,
causes: ["auth_failed:all-providers-failed"],
causes: ["failed:all-providers-failed"],
},
});
});
it("emits job.run.partial, never job.run.failed, for a run whose only attempt was a refused credential (#1384)", async () => {
const verdict = await reportJobRun({
queue: "insight-pregenerate",
runId: "r1",
candidates: [
{
key: "a",
outcome: "auth_failed",
cause: "all-providers-failed:openai-compatible",
},
],
});
expect(verdict.allFailed).toBe(false);
expect(readConsecutiveRunFailures).not.toHaveBeenCalled();
expect(emitSignal).toHaveBeenCalledTimes(1);
expect(emitSignal.mock.calls[0][0]).toMatchObject({
action: "job.run.partial",
level: "warn",
meta: {
causes: ["auth_failed:all-providers-failed:openai-compatible"],
},
});
});
@@ -87,6 +87,34 @@ describe("judgeQueueRuns", () => {
).toBe("failing");
});
// #1384 — the alert quoted the completed run's own outcome JSON, and the
// operator it sent to the admin page found nothing there.
it("says in words what an all-failed night was, with its cause and provider", () => {
const output = JSON.stringify({
ok: true,
did: {
total: 1,
failed: 1,
all_failed: true,
assessments_warmed: 0,
failure_cause: "all_providers_failed",
failure_provider: "admin_openai",
},
});
const verdict = judgeQueueRuns(
[0, 1, 2].map((ago) =>
run(ago, { state: "completed", allFailed: true, error: output }),
),
);
expect(verdict.state).toBe("failing");
expect(verdict.lastError).toBe(
"The run completed, but every account it attempted failed (1 of 1). " +
"First cause: all_providers_failed from provider admin_openai. " +
"Accounts without a usable provider are skipped and not counted; " +
"job.run.failed in the app log lists each account's outcome.",
);
});
it("explains a bare timeout the way the admin card does", () => {
const verdict = judgeQueueRuns(
[0, 1, 2].map((ago) =>
@@ -0,0 +1,93 @@
/**
* How the nightly briefing pass settles an account whose generation came back
* `failed` (#1384).
*
* A failure used to be one of three things: screened, a dead sign-in, or a
* provider failure worth a retry in 45 minutes. Two cases were missing, and
* both made a self-hoster's only account read as a failing run every night,
* which the repeated-failure alert then reported as a broken job:
*
* - Every provider of the chain was benched for background work by the
* provider health ledger (#1279), so no request was sent. The account was
* not attempted tonight; its earlier failures were counted by the runs
* that made them, and a retry would find the same bench. It is `skipped`.
* - The provider refused the credential. A refused credential does not heal
* in 45 minutes, so no retry is queued, and whose credential it was
* decides who has to act. The operator's shared key or central sign-in is
* the operator's to fix, and a run where it was refused for everyone is a
* failing run: `failed`. The account's own key, gateway or sign-in is its
* owner's to fix: `auth_failed`, which the run reports but does not fail
* over (`job-run-report.ts`). An API key revoked at its issuer stays saved
* and is refused again every night, unlike a ChatGPT sign-in, which the
* refusal parks.
*
* The model answered and the safety screen withheld it (`outbound-screened`)
* stays `screened` with no retry: asking the same model again in 45 minutes
* is the same question, and it was screened eight times a night.
*/
import { isOperatorHeldProvider } from "@/lib/ai/provider-egress";
import type { GenerateOutcome } from "@/lib/insights/comprehensive-generate";
import type { JobRunCandidate, JobRunOutcome } from "@/lib/jobs/job-run-report";
type FailedGeneration = Extract<GenerateOutcome, { status: "failed" }>;
export interface FailedGenerationVerdict {
outcome: JobRunOutcome;
/** The run line's cause: the reason, and the provider that failed first. */
cause: string;
/** False when nothing was sent: the account is tallied as skipped. */
attempted: boolean;
/** Queue the bounded 45-minute retry for this account. */
retry: boolean;
}
export function settleFailedGeneration(
generated: FailedGeneration,
): FailedGenerationVerdict {
if (generated.sentNothing) {
return {
outcome: "skipped",
cause: "provider_benched",
attempted: false,
retry: false,
};
}
const cause = generated.provider
? `${generated.reason}:${generated.provider}`
: generated.reason;
if (generated.reason === "outbound-screened") {
return { outcome: "screened", cause, attempted: true, retry: false };
}
if (generated.credentialExpired) {
const operatorHeld =
generated.provider !== undefined &&
isOperatorHeldProvider(generated.provider);
return {
outcome: operatorHeld ? "failed" : "auth_failed",
cause,
attempted: true,
retry: false,
};
}
return { outcome: "failed", cause, attempted: true, retry: true };
}
/** The first failed account of a run: its reason and the provider, if known. */
export interface FirstFailure {
reason: string;
provider: string | null;
}
/**
* The first failed account's reason and provider, read back off the run's
* causes, for the job's facts: what the admin page and the repeated-failure
* alert show for a run that completed with every account failed.
*/
export function firstFailure(
candidates: readonly JobRunCandidate[],
): FirstFailure | null {
const cause = candidates.find((c) => c.outcome === "failed")?.cause;
if (!cause) return null;
const [reason, provider] = cause.split(":");
return { reason, provider: provider ?? null };
}
+35 -32
View File
@@ -81,6 +81,10 @@ import {
} from "@/lib/insights/metric-status-registry";
import { withTimeout } from "@/lib/insights/with-timeout";
import { comprehensiveWarmBudgetMs } from "@/lib/jobs/model-time";
import {
firstFailure,
settleFailedGeneration,
} from "@/lib/jobs/insight-pregenerate-failure";
import {
INSIGHT_PREGENERATE_QUEUE,
enqueueForceWarm,
@@ -212,12 +216,13 @@ export interface PregenerateRunResult {
/** The job's time budget ran out before every candidate was reached. */
stoppedEarly: boolean;
/**
* v1.42 — every candidate the run attempted failed (see `reportJobRun`).
* Written into the job's facts as `all_failed`, which is how a completed
* night with nothing generated still counts toward the repeated-failure
* streak.
* v1.42 — every candidate the run attempted failed (see `reportJobRun`),
* written into the job's facts as `all_failed` so a completed night with
* nothing generated counts toward the repeated-failure streak. `attempted`
* (#1384) goes out as `run_attempted`; at 0 the streak steps over the night.
*/
allFailed: boolean;
attempted: number;
/**
* Count of per-metric assessment caches written warm across the whole
* run. The comprehensive generator evicts the seven `*-status`
@@ -235,6 +240,8 @@ export interface PregenerateRunResult {
* seven specialised cards.
*/
metricAssessmentsWarmed: number;
/** #1384 — the first failed account's reason and provider, as facts. */
firstFailure: ReturnType<typeof firstFailure>;
}
/**
@@ -302,9 +309,9 @@ async function warmPerStatusCaches(
* once and issues ONE provider call for the metrics still needing the LLM,
* fanning the response back into the same per-metric cache rows the
* standalone generators wrote. Replaces the prior seven independent warm
* calls. Returns the count of metrics that got a fresh (batched or
* fallback) assessment so the wide-event tally is comparable to the old
* per-card count.
* calls. Returns the count of metrics that got a fresh model-written
* assessment, batched or through a card's own fallback call, so the
* wide-event tally is comparable to the old per-card count.
*
* Refill-only (`force: false`): a card already generated today and a card
* whose content-hash is unchanged both resolve inside the batch as `served`
@@ -322,7 +329,7 @@ async function warmStatusBatch(
locale,
force: false,
});
warmed += result.batched + result.fellBack;
warmed += result.batched + result.fellBackAnswered;
} catch (err) {
// The batch must never throw the cron's user loop off course — but
// a swallowed batch failure left ~7 cards cold with zero signal.
@@ -553,8 +560,10 @@ export async function runInsightPregenerate(
budgetBlocked: 0,
stoppedEarly: false,
allFailed: false,
attempted: 0,
assessmentsWarmed: 0,
metricAssessmentsWarmed: 0,
firstFailure: null,
};
// Master kill-switch — when the operator disabled the briefing
@@ -619,6 +628,8 @@ export async function runInsightPregenerate(
candidates: runOutcomes,
});
result.allFailed = verdict.allFailed;
result.attempted = verdict.attempted;
result.firstFailure = firstFailure(runOutcomes);
return result;
@@ -783,42 +794,34 @@ export async function runInsightPregenerate(
result.skipped++;
settle("skipped", generated.reason);
break;
case "failed":
result.failed++;
// v1.28.30 — THE previously-silent failure path. A generator
// that resolves with `{ status: "failed" }` (features error,
// oversize payload, invalid JSON after retry, abort) bumped
// the tally with no annotation at all — the nightly summary
// showed `failed: 1` and nothing else was greppable. Reuse
// the `comprehensive_failed` shape with the generator's own
// reason as the cause.
case "failed": {
// Benched, screened, a refused credential and whose, or a
// retryable failure: `insight-pregenerate-failure.ts` (#1384).
const verdict = settleFailedGeneration(generated);
if (verdict.attempted) result.failed++;
else result.skipped++;
annotate({
action: { name: "insights.pregenerate.comprehensive_failed" },
action: {
name: verdict.attempted
? "insights.pregenerate.comprehensive_failed"
: "insights.pregenerate.provider_benched",
},
meta: {
locale,
stage: "nightly.generator",
cause: `generator:${generated.reason}`,
provider: generated.provider ?? null,
credential_refused: generated.credentialExpired === true,
budget_ms: warmBudgetMs,
message: null,
},
});
if (generated.reason === "outbound-screened") {
// v1.42 — the model answered and the safety screen withheld
// it, after the generator's own repair. Asking the same model
// again in 45 minutes is not a provider retry; it is the same
// question, and it was screened eight times a night.
settle("screened", generated.reason);
} else if (generated.credentialExpired) {
// v1.42 — a dead sign-in (a Codex refresh token the issuer
// revoked) does not heal in 45 minutes either; the person has
// to reconnect. The refusal parked the connection as expired,
// which Settings shows, and the next run no longer counts it.
settle("auth_failed", generated.reason);
} else {
settle("failed", generated.reason);
settle(verdict.outcome, verdict.cause);
if (verdict.retry) {
await enqueueRetry({ userId: candidate.id, locale });
}
break;
}
}
}
}
+57 -4
View File
@@ -92,7 +92,7 @@ const MAX_ERROR_CHARS = 300;
export interface FailingQueue {
/** The pg-boss queue name, e.g. `environment-fetch`. */
queue: string;
/** Jobs that exhausted their retries inside the window. */
/** Failed runs inside the window: exhausted retries, or all-failed passes. */
failures: number;
/** ISO instant of the newest failure. */
lastFailedAt: string;
@@ -140,10 +140,48 @@ function explainTimeout(message: string, expireSeconds: number | null): string {
);
}
/**
* Say what a completed run that failed every account means (#1384). Such a
* run's `output` is its outcome, `{"ok":true,"did":{…,"all_failed":true}}`,
* and quoting that JSON told the operator nothing: not that the job ran, not
* which provider failed. Null when `raw` is not such an outcome.
*/
function explainAllFailedRun(raw: string): string | null {
if (!raw.startsWith("{")) return null;
let did: Record<string, unknown>;
try {
const parsed = JSON.parse(raw) as { ok?: unknown; did?: unknown };
if (parsed?.ok !== true || typeof parsed.did !== "object") return null;
did = (parsed.did ?? {}) as Record<string, unknown>;
} catch {
return null;
}
if (did.all_failed !== true) return null;
// Every attempted account failed, so the count is both sides of "of".
const failed =
typeof did.failed === "number" && did.failed > 0
? ` (${did.failed} of ${did.failed})`
: "";
// "none" is how the run says it has no cause or provider to name.
const named = (value: unknown): string | null =>
typeof value === "string" && value !== "none" ? value : null;
const cause = named(did.failure_cause);
const provider = named(did.failure_provider);
const why = cause
? ` First cause: ${cause}${provider ? ` from provider ${provider}` : ""}.`
: "";
return (
`The run completed, but every account it attempted failed${failed}.${why} ` +
`Accounts without a usable provider are skipped and not counted; ` +
`job.run.failed in the app log lists each account's outcome.`
);
}
/**
* Turn whatever pg-boss stored in `output` into one line. A thrown `Error`
* serialises to `{ message, stack, … }`; a thrown non-Error can be any JSON at
* all, so the raw text is the fallback rather than an invented placeholder.
* A completed run that reported every account failed is said in words.
*/
export function presentError(
raw: string | null,
@@ -152,6 +190,8 @@ export function presentError(
if (raw === null) return "";
const trimmed = raw.trim();
if (trimmed.length === 0) return "";
const allFailed = explainAllFailedRun(trimmed);
if (allFailed !== null) return allFailed;
if (TIMEOUT_MESSAGE.test(trimmed))
return explainTimeout(trimmed, expireSeconds);
if (HEARTBEAT_MESSAGE.test(trimmed)) {
@@ -168,8 +208,13 @@ export function presentError(
}
/**
* Every queue with at least one exhausted-retry failure in the window, newest
* first. `null` when the queue schema is not present at all.
* Every queue with at least one failed run in the window, newest first:
* a job that exhausted its retries, or a fan-out pass that completed but
* reported that every account it attempted failed (`did.all_failed`). The
* second kind is what the repeated-failure alert counts too
* (`readConsecutiveRunFailures`, `system-job-alert.ts`); the page used to
* list only the first, so an operator the alert sent here read "no job has
* failed" (#1384). `null` when the queue schema is not present at all.
*/
export async function readFailingQueues(
withinHours: number = JOB_FAILURE_WINDOW_HOURS,
@@ -194,7 +239,10 @@ export async function readFailingQueues(
array_agg(expire_seconds ORDER BY completed_on DESC NULLS LAST)
)[1] AS last_expire_seconds
FROM pgboss.job
WHERE state = 'failed'
WHERE (
state = 'failed'
OR (state = 'completed' AND output->'did'->>'all_failed' = 'true')
)
AND completed_on > now() - make_interval(hours => ${withinHours}::int)
GROUP BY name
ORDER BY max(completed_on) DESC
@@ -449,6 +497,11 @@ export async function readConsecutiveRunFailures(
AND state IN ('completed', 'failed')
AND completed_on IS NOT NULL
AND (state = 'failed' OR output->'did'->>'all_failed' IS NOT NULL)
-- #1384: a run that attempted nobody neither fails nor recovers.
AND NOT (
state = 'completed'
AND COALESCE(output->'did'->>'run_attempted', '') = '0'
)
ORDER BY completed_on DESC
LIMIT ${CONSECUTIVE_LOOKBACK_ROWS}
`;
+24
View File
@@ -130,6 +130,11 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
"duration_ms",
"errored",
"failed",
// #1384 — a fan-out pass's first failed candidate: its cause and the
// provider that failed for it, as stable codes. What the admin page and
// the repeated-failure alert show for a run that completed all-failed.
"failure_cause",
"failure_provider",
"failures",
"feedback_buckets",
"feedback_total_rows",
@@ -239,6 +244,9 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
"removed",
"resting_corrected",
"rows_written",
// #1384 — how many accounts a fan-out pass attempted; 0 marks a run that
// neither failed nor recovered, which the failure streak steps over.
"run_attempted",
// v1.39.1 — the background restore of a stored backup, and the sweep that
// re-queues one a stopped worker left running. Counts and a claim flag.
"restore_claimed",
@@ -423,6 +431,22 @@ export function percentFact(ratio: number | null | undefined): number {
return wholeFact(ratio * 100);
}
/**
* A short code from our own vocabulary (a generator reason, a provider type)
* as a fact: `all-providers-failed` becomes `all_providers_failed`. Null when
* nothing of it survives, so the caller leaves the fact out.
*/
export function codeFact(value: string | null | undefined): string | null {
if (value == null) return null;
const code = value
.toLowerCase()
.replace(/[^a-z0-9]+/g, "_")
.replace(/^[^a-z]+/, "")
.slice(0, 64)
.replace(/_+$/, "");
return STABLE_CODE_PATTERN.test(code) ? code : null;
}
/** Why one fact is refused, or null when it is a valid fact. */
function invalidFact(key: string, value: unknown): string | null {
if (!JOB_FACT_ALLOWLIST.has(key))
+25 -5
View File
@@ -21,6 +21,18 @@
* attempted but is not a provider failure, so a run of only screened
* candidates is partial, not failed.
*
* `auth_failed` is attempted and is not a failure of the run either (#1384).
* It means the provider refused the account's own credential: a key revoked
* at its issuer, a gateway that now wants a key the account no longer
* stores. Only that account's owner can change it, and the run itself worked.
* It used to count as a failure on the assumption that a refusal is reported
* once and then parked; that holds for a ChatGPT sign-in and for nothing else,
* so an API key the issuer had revoked failed every night for as long as it
* stayed saved. On an instance where that account was the only candidate the
* run then read as all-failed, and the operator was told the job was broken
* night after night. A refusal of the operator's own credential is reported by
* the caller as `failed`, because there the operator is the one who can act.
*
* The verdict comes back to the caller, which writes `all_failed` into its
* `jobDone` facts. That fact is what lets `readConsecutiveRunFailures` count a
* completed-but-all-failed night as part of a streak.
@@ -46,18 +58,26 @@ export interface JobRunVerdict {
total: number;
/** Candidates it actually attempted (not skipped, not deferred). */
attempted: number;
/** `failed` + `auth_failed`. */
/** Candidates that failed (`failed`; a refused credential is not one). */
failed: number;
/** Every attempted candidate failed (and at least one was attempted). */
allFailed: boolean;
/** Something failed or was withheld, but not everything. */
/**
* Something failed, was withheld or had its credential refused, but not
* everything failed.
*/
partial: boolean;
}
/** Candidates listed on the run's line; the counts always cover all of them. */
const MAX_LISTED_CANDIDATES = 50;
const FAILED: ReadonlySet<JobRunOutcome> = new Set(["failed", "auth_failed"]);
const FAILED: ReadonlySet<JobRunOutcome> = new Set(["failed"]);
/** Attempted, not a failure of the run, but worth a `partial` line. */
const WITHHELD: ReadonlySet<JobRunOutcome> = new Set([
"screened",
"auth_failed",
]);
const NOT_ATTEMPTED: ReadonlySet<JobRunOutcome> = new Set([
"skipped",
"deferred",
@@ -72,14 +92,14 @@ export function judgeJobRun(
(c) => !NOT_ATTEMPTED.has(c.outcome),
).length;
const failed = candidates.filter((c) => FAILED.has(c.outcome)).length;
const screened = candidates.filter((c) => c.outcome === "screened").length;
const withheld = candidates.filter((c) => WITHHELD.has(c.outcome)).length;
const allFailed = attempted > 0 && failed === attempted;
return {
total,
attempted,
failed,
allFailed,
partial: !allFailed && (failed > 0 || screened > 0),
partial: !allFailed && (failed > 0 || withheld > 0),
};
}
+11 -2
View File
@@ -21,7 +21,7 @@ import {
type InsightStatusGeneratePayload,
runInsightStatusGenerate,
} from "@/lib/jobs/insight-status-generate";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { codeFact, jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { jobBudget } from "@/lib/jobs/job-budget";
import { withBackgroundEvent } from "@/lib/logging/background";
import { generateStatusBatchForUser } from "@/lib/insights/status-batch";
@@ -80,7 +80,7 @@ export async function runStatusBatchCron(
locale: await resolveJobLocale(user.locale),
force: false,
});
generated += result.batched + result.fellBack;
generated += result.batched + result.fellBackAnswered;
served += result.served;
} catch (error) {
failed++;
@@ -201,8 +201,17 @@ export async function handleInsightPregenerateJob(
// v1.42 — read back by `readConsecutiveRunFailures`, so a night that
// completed with nothing generated counts toward the failure streak.
all_failed: summary.allFailed,
// #1384 — a night that attempted no account (every provider paused)
// is neither a failure nor a recovery. `run_attempted: 0` is how the
// streak readers (`readConsecutiveRunFailures`, the repeated-failure
// alert) know to step over it instead of ending the streak.
run_attempted: summary.attempted,
assessments_warmed: summary.assessmentsWarmed,
metric_assessments_warmed: summary.metricAssessmentsWarmed,
// #1384 — what failed, for the admin page and the alert ("none"
// when nothing did, or the generator could not say).
failure_cause: codeFact(summary.firstFailure?.reason) ?? "none",
failure_provider: codeFact(summary.firstFailure?.provider) ?? "none",
});
} catch (err) {
evt.addWarning(
+6
View File
@@ -197,6 +197,12 @@ export async function readTerminalRuns(
WHERE name = q.name
AND completed_on IS NOT NULL
AND state IN ('completed', 'failed')
-- #1384: a fan-out run that attempted nobody neither fails nor
-- recovers; it must not end a streak.
AND NOT (
state = 'completed'
AND COALESCE(output->'did'->>'run_attempted', '') = '0'
)
ORDER BY completed_on DESC
LIMIT ${LOOKBACK_ROWS}
) j
@@ -51,6 +51,8 @@ const READY: RecordSessionTransition = {
let transition = READY;
let seeded = false;
/** Bumped on every change this tab applies; see `recordSessionRevision`. */
let revision = 0;
const listeners = new Set<() => void>();
let channel: BroadcastChannel | null = null;
let storageListenerAttached = false;
@@ -107,6 +109,7 @@ function applyWire(next: WireTransition): void {
}
transition = toSnapshot(next);
seeded = true;
revision += 1;
notify();
}
@@ -228,6 +231,26 @@ export function getRecordSessionTransition(): RecordSessionTransition {
return transition;
}
/**
* A counter that moves whenever this tab applies a transition change — its own
* publish, or one received from another tab.
*
* `fetchMe` reads it on both sides of its request. A `/api/auth/me` answered
* across a change cannot say which side of that change the server read it on:
* the switcher's own `queryClient.clear()` remounts the account query the
* moment the hold goes up, so the initiator routinely has a `/me` in flight
* beside its own `POST /api/account/switch`, and that answer can describe the
* record the session has just left. Applied after the commit, it rewrites the
* scope mirror back to the old record; every peer that has already settled
* reads that write as a foreign switch and holds for a `/me` naming the old
* record, which no server answer will ever give. The counter is how `fetchMe`
* recognises such an answer and asks again instead of applying it.
*/
export function recordSessionRevision(): number {
seed();
return revision;
}
export function subscribeToRecordSessionTransition(
listener: () => void,
): () => void {
@@ -460,6 +460,7 @@
"batchCallMade": true,
"batched": 6,
"fellBack": 1,
"fellBackAnswered": 0,
"served": 0
}
},
@@ -1719,6 +1720,7 @@
"batchCallMade": true,
"batched": 0,
"fellBack": 7,
"fellBackAnswered": 0,
"served": 0
}
},
@@ -1831,6 +1833,7 @@
"batchCallMade": false,
"batched": 0,
"fellBack": 1,
"fellBackAnswered": 0,
"served": 6
}
}
@@ -40,6 +40,7 @@ vi.mock("@/lib/insights/status-batch", async () => {
served: 0,
batched: 0,
fellBack: 0,
fellBackAnswered: 0,
batchCallMade: false,
};
if (!(await aiCapabilityForJob(userId, "statusText")).available) {
+48
View File
@@ -126,6 +126,41 @@ describe("readFailingQueues", () => {
await expect(readFailingQueues()).resolves.toEqual([]);
});
// #1384 — the repeated-failure alert counts a nightly pass that completed
// with every account failed, and sent the operator to this page, which
// listed only pg-boss failures and said nothing had failed.
it("lists a run that completed with every account failed, in words, and not a good run", async () => {
const ALL_FAILED_QUEUE = "insight-pregenerate";
await boss.createQueue(ALL_FAILED_QUEUE);
async function completeOne(did: Record<string, unknown>) {
const jobId = await boss.send(ALL_FAILED_QUEUE, {}, { retryLimit: 0 });
const [job] = await boss.fetch(ALL_FAILED_QUEUE);
if (!job || job.id !== jobId) throw new Error("nothing to fetch");
await boss.complete(ALL_FAILED_QUEUE, job.id, { ok: true, did });
}
// A good night, and the night from the report: one account, one failure.
await completeOne({ total: 1, generated: 1, failed: 0, all_failed: false });
await completeOne({
total: 1,
generated: 0,
failed: 1,
all_failed: true,
assessments_warmed: 0,
failure_cause: "all_providers_failed",
failure_provider: "openai_compatible",
});
const failing = await readFailingQueues();
const entry = failing?.find((e) => e.queue === ALL_FAILED_QUEUE);
expect(entry?.failures).toBe(1);
expect(entry?.lastError).toBe(
"The run completed, but every account it attempted failed (1 of 1). " +
"First cause: all_providers_failed from provider openai_compatible. " +
"Accounts without a usable provider are skipped and not counted; " +
"job.run.failed in the app log lists each account's outcome.",
);
});
it("ignores a failure older than the window", async () => {
await failOneJob(QUEUE, { userId: "user-a" }, "old failure");
await getPrismaClient().$executeRawUnsafe(
@@ -233,6 +268,19 @@ describe("readConsecutiveRunFailures (v1.42)", () => {
await expect(readConsecutiveRunFailures(STREAK_QUEUE)).resolves.toBe(3);
});
it("steps over a night that attempted nobody instead of ending the streak (#1384)", async () => {
await boss.createQueue(STREAK_QUEUE);
// Failed, then every provider paused (nothing attempted), then failed.
await finishedMinutesAgo(await completeOneJob({ all_failed: true }), 30);
await finishedMinutesAgo(
await completeOneJob({ all_failed: false, run_attempted: 0 }),
20,
);
await finishedMinutesAgo(await completeOneJob({ all_failed: true }), 10);
await expect(readConsecutiveRunFailures(STREAK_QUEUE)).resolves.toBe(3);
});
it("answers 1 for a queue with no history: the run failing now", async () => {
await expect(readConsecutiveRunFailures("never-ran")).resolves.toBe(1);
});
+17 -1
View File
@@ -62,6 +62,7 @@ async function runOnce(
outcome: "fail" | "complete",
userId = "u",
retryLimit = 0,
output: Record<string, unknown> = { ok: true },
) {
const id = await boss.send(
queue,
@@ -80,7 +81,7 @@ async function runOnce(
new Error(`canceling statement due to statement timeout (${userId})`),
);
} else {
await boss.complete(queue, job.id, { ok: true });
await boss.complete(queue, job.id, output);
return;
}
}
@@ -193,6 +194,21 @@ describe("notifySystemJobFailures", () => {
expect(outcome.sent).toEqual([]);
});
it("lets a run that attempted nobody neither end nor extend the streak (#1384)", async () => {
await makeUser({ role: "ADMIN" });
// What the briefing pass writes on a night every provider was paused.
const benched = {
ok: true,
did: { all_failed: false, run_attempted: 0, skipped: 1 },
};
await runOnce(SYSTEM_QUEUE, "fail");
await runOnce(SYSTEM_QUEUE, "fail");
await runOnce(SYSTEM_QUEUE, "complete", "u", 0, benched);
expect((await notifySystemJobFailures()).failing).toBe(0);
await runOnce(SYSTEM_QUEUE, "fail");
expect((await notifySystemJobFailures()).failing).toBe(1);
});
it("counts a job that used up its retries as one failure", async () => {
await makeUser({ role: "ADMIN" });
await runOnce(SYSTEM_QUEUE, "fail", "u", 2);