mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-16 15:23:29 +00:00
Merge pull request #1385 from MBombeck/release/v1.47.6
release: v1.47.6
This commit is contained in:
45 files changed
+1068
-133
No files matched your search
+31
-5
@@ -2,6 +2,8 @@
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [1.47.6] — 2026-11-15
|
||||
|
||||
### Added
|
||||
|
||||
- **A note on a medication.** Directions such as "30 to 60 minutes before
|
||||
@@ -42,11 +44,35 @@
|
||||
links the help page. Brave keeps push messaging off by default, so in
|
||||
Brave the message names the setting to turn on: "Use Google services
|
||||
for push messaging" under brave://settings/privacy. (Discussion #1378)
|
||||
- **Illness episode onset dates keep the intended calendar day.**
|
||||
Creating a new illness episode in the morning was refused because its noon
|
||||
default was rejected as in the future, and onset dates could display as the
|
||||
wrong calendar day in non-UTC time zones. Saving now anchors to the current
|
||||
time when before local noon and preserves stored timestamps on edit.
|
||||
- **Illness episodes can be added in the morning.** The form sent the
|
||||
start date as noon, which before noon is still in the future, so the
|
||||
save was refused. Start and end dates now also stay on the day you
|
||||
picked when your time zone is far from UTC, and editing an episode
|
||||
keeps the times you saved. (#1376)
|
||||
- **No nightly "job still failing" alert over an AI provider that cannot
|
||||
answer.** An account that still had a provider saved whose key the
|
||||
provider refused (removing the key from a Local or OpenAI-compatible
|
||||
provider keeps its address, so it is still called), or whose providers
|
||||
HealthLog had paused after repeated errors, counted as a failed
|
||||
briefing every night. On an instance with one account that read as a
|
||||
broken job. A refused key of the account's own is now noted but no
|
||||
longer fails the run; a refused shared server key still does. An
|
||||
account whose providers are all paused is skipped, since nothing was
|
||||
sent. The Background jobs card on the admin status page now lists the
|
||||
same runs the alert counts, and both say in words what failed and
|
||||
which provider. (#1384)
|
||||
- **Switching records with HealthLog open in a second tab.** Switching
|
||||
into or out of a shared record could leave both tabs on the loading
|
||||
spinner, and a reload within the next half minute did not help. An
|
||||
account check the switching tab had started just before the switch
|
||||
could answer for the record being left and undo the switch the other
|
||||
tabs were waiting for. Such an answer is now thrown away and asked
|
||||
again.
|
||||
- **Charts no longer jump while they load.** The placeholder of every
|
||||
chart that links its days was 20 px taller than the chart it turned
|
||||
into (40 px on a phone), and the mood chart's placeholder was about
|
||||
30 px shorter, so the dashboard and the metric pages shifted under you
|
||||
when the charts arrived.
|
||||
|
||||
### Security
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: 3.1.0
|
||||
info:
|
||||
title: HealthLog API
|
||||
version: 1.47.5
|
||||
version: 1.47.6
|
||||
description: >-
|
||||
Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest.
|
||||
|
||||
|
||||
@@ -54,6 +54,42 @@ async function openDashboard(page: Page) {
|
||||
).toBeVisible();
|
||||
// The module-gated destinations join the rail once `/api/auth/me` lands.
|
||||
await expect(page.locator('aside nav a[href="/medications"]')).toHaveCount(1);
|
||||
await expectChartRowSettled(page);
|
||||
}
|
||||
|
||||
/**
|
||||
* The dashboard's chart row has swapped every loading shell for its chart.
|
||||
*
|
||||
* The charts are one lazily loaded chunk, and each cell holds a skeleton until
|
||||
* it lands and the row reveals. The swap changes the row's height, so a page
|
||||
* measured before it is not the page the reader ends up with: on a loaded
|
||||
* runner the chunk arrived a second after the dashboard painted, after
|
||||
* `scrollMainToEnd` had already seen two equal heights, and `<main>` was no
|
||||
* longer at its end when it was measured. Waited for on the cells' own state
|
||||
* rather than on time.
|
||||
*/
|
||||
async function expectChartRowSettled(page: Page) {
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
page.evaluate(() => {
|
||||
const main = document.getElementById("main-content")!;
|
||||
const cells = main.querySelectorAll(
|
||||
'[data-slot="dashboard-chart-cell"]',
|
||||
);
|
||||
const held = main.querySelectorAll(
|
||||
'[data-slot="dashboard-chart-cell"]:not([data-revealed="true"])',
|
||||
);
|
||||
const shells = main.querySelectorAll('[data-slot="chart-skeleton"]');
|
||||
return {
|
||||
cells: cells.length > 0,
|
||||
held: held.length,
|
||||
shells: shells.length,
|
||||
};
|
||||
}),
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
.toEqual({ cells: true, held: 0, shells: 0 });
|
||||
}
|
||||
|
||||
/** The shell fills the window exactly and the document has nothing to scroll. */
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Hintergrundjobs",
|
||||
"description": "Queues, deren Jobs nach allen Wiederholungen aufgegeben haben.",
|
||||
"description": "Queues, deren Jobs nach allen Wiederholungen aufgaben oder deren Lauf bei jedem versuchten Konto scheiterte.",
|
||||
"none": "In den letzten {hours} Stunden ist kein Job fehlgeschlagen.",
|
||||
"unavailable": "Keine Job-Queue lesbar. Dieses Deployment läuft ohne Hintergrund-Worker.",
|
||||
"countOne": "{count} Fehlschlag",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Background jobs",
|
||||
"description": "Queues whose jobs gave up after exhausting their retries.",
|
||||
"description": "Queues whose jobs gave up after their retries, or whose run ended with every account it tried failing.",
|
||||
"none": "No job has failed in the last {hours} hours.",
|
||||
"unavailable": "No job queue to read. This deployment runs without the background worker.",
|
||||
"countOne": "{count} failure",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Trabajos en segundo plano",
|
||||
"description": "Colas cuyos trabajos se rindieron tras agotar sus reintentos.",
|
||||
"description": "Colas cuyos trabajos se rindieron tras sus reintentos o cuya ejecución falló en cada cuenta intentada.",
|
||||
"none": "Ningún trabajo ha fallado en las últimas {hours} horas.",
|
||||
"unavailable": "No hay ninguna cola que consultar: este despliegue funciona sin el worker en segundo plano.",
|
||||
"countOne": "{count} fallo",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Tâches en arrière-plan",
|
||||
"description": "Files dont les tâches ont abandonné après avoir épuisé leurs tentatives.",
|
||||
"description": "Files dont les tâches ont abandonné après leurs tentatives, ou dont l’exécution a échoué pour chaque compte.",
|
||||
"none": "Aucune tâche n’a échoué au cours des {hours} dernières heures.",
|
||||
"unavailable": "Aucune file à consulter : ce déploiement fonctionne sans le worker en arrière-plan.",
|
||||
"countOne": "{count} échec",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Processi in background",
|
||||
"description": "Code i cui processi hanno rinunciato dopo aver esaurito i tentativi.",
|
||||
"description": "Code i cui processi hanno rinunciato dopo i tentativi o la cui esecuzione è fallita per ogni account.",
|
||||
"none": "Nessun processo è fallito nelle ultime {hours} ore.",
|
||||
"unavailable": "Nessuna coda da leggere: questo deployment funziona senza il worker in background.",
|
||||
"countOne": "{count} errore",
|
||||
|
||||
+1
-1
@@ -8743,7 +8743,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "백그라운드 작업",
|
||||
"description": "재시도를 다 쓰고 포기한 작업이 있는 큐예요.",
|
||||
"description": "재시도를 다 쓰고 포기했거나, 실행에서 시도한 모든 계정이 실패한 큐예요.",
|
||||
"none": "최근 {hours}시간 동안 실패한 작업이 없어요.",
|
||||
"unavailable": "읽을 작업 큐가 없어요. 이 배포본은 백그라운드 워커 없이 돌고 있어요.",
|
||||
"countOne": "실패 {count}건",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Achtergrondtaken",
|
||||
"description": "Wachtrijen waarvan taken het hebben opgegeven nadat alle nieuwe pogingen waren verbruikt.",
|
||||
"description": "Wachtrijen waarvan taken na alle pogingen opgaven, of waarvan een run voor elk geprobeerd account mislukte.",
|
||||
"none": "Er is de afgelopen {hours} uur geen taak mislukt.",
|
||||
"unavailable": "Geen takenwachtrij om te lezen. Deze installatie draait zonder de achtergrondworker.",
|
||||
"countOne": "{count} fout",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Zadania w tle",
|
||||
"description": "Kolejki, których zadania poddały się po wyczerpaniu ponowień.",
|
||||
"description": "Kolejki, których zadania poddały się po ponowieniach albo których przebieg zawiódł dla każdego konta.",
|
||||
"none": "W ciągu ostatnich {hours} godzin żadne zadanie nie zawiodło.",
|
||||
"unavailable": "Brak kolejki zadań do odczytu: to wdrożenie działa bez workera w tle.",
|
||||
"countOne": "{count} niepowodzenie",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Tarefas em segundo plano",
|
||||
"description": "Filas cujas tarefas desistiram depois de esgotar as tentativas.",
|
||||
"description": "Filas cujas tarefas desistiram após as tentativas, ou cuja execução falhou em todas as contas tentadas.",
|
||||
"none": "Nenhuma tarefa falhou nas últimas {hours} horas.",
|
||||
"unavailable": "Nenhuma fila de tarefas para ler. Esta implantação roda sem o worker em segundo plano.",
|
||||
"countOne": "{count} falha",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Фоновые задания",
|
||||
"description": "Очереди, задания которых сдались после исчерпания повторных попыток.",
|
||||
"description": "Очереди, задания которых сдались после всех попыток или чей запуск не удался ни для одного аккаунта.",
|
||||
"none": "За последние {hours} часов ни одно задание не завершилось неудачей.",
|
||||
"unavailable": "Нет очереди заданий для чтения. Это развёртывание работает без фонового воркера.",
|
||||
"countOne": "{count} неудача",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "healthlog",
|
||||
"version": "1.47.5",
|
||||
"version": "1.47.6",
|
||||
"description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.",
|
||||
"license": "PolyForm-Noncommercial-1.0.0",
|
||||
"homepage": "https://healthlog.dev",
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ try {
|
||||
}
|
||||
const CACHE_VERSION =
|
||||
(typeof self !== "undefined" && self.__APP_VERSION__) ||
|
||||
/* @sw-version-fallback */ "v1.47.5";
|
||||
/* @sw-version-fallback */ "v1.47.6";
|
||||
const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`;
|
||||
const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`;
|
||||
// Every HealthLog cache, current and legacy, and nothing another app on the
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
|
||||
"count": 84,
|
||||
"total": 93816,
|
||||
"total": 93819,
|
||||
"entries": {
|
||||
"app/medications/page-client.tsx": 842,
|
||||
"components/charts/mood-chart.tsx": 1166,
|
||||
@@ -64,7 +64,7 @@
|
||||
"lib/insights/mood-aggregates.ts": 1053,
|
||||
"lib/integrations/status.ts": 1448,
|
||||
"lib/jobs/coach-nudge.ts": 1084,
|
||||
"lib/jobs/insight-pregenerate.ts": 1189,
|
||||
"lib/jobs/insight-pregenerate.ts": 1192,
|
||||
"lib/jobs/medication-intake-import.ts": 819,
|
||||
"lib/jobs/offhost-backup.ts": 1094,
|
||||
"lib/jobs/reaction-line.ts": 803,
|
||||
|
||||
@@ -435,7 +435,7 @@
|
||||
"lib/jobs/__tests__/document-summary-reaper.test.ts": 1,
|
||||
"lib/jobs/__tests__/document-summary.test.ts": 19,
|
||||
"lib/jobs/__tests__/geo-backfill.test.ts": 1,
|
||||
"lib/jobs/__tests__/insight-pregenerate.test.ts": 45,
|
||||
"lib/jobs/__tests__/insight-pregenerate.test.ts": 43,
|
||||
"lib/jobs/__tests__/insight-status-generate.test.ts": 2,
|
||||
"lib/jobs/__tests__/measurement-maintenance-schedule.test.ts": 2,
|
||||
"lib/jobs/__tests__/measurement-reminder-byhour.test.ts": 2,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.",
|
||||
"count": 2863,
|
||||
"total": 9122,
|
||||
"total": 9118,
|
||||
"entries": {
|
||||
"__tests__/account-payload-consumer-guard.test.ts": 2,
|
||||
"__tests__/acting-account-boundary-guard.test.ts": 7,
|
||||
@@ -2134,7 +2134,7 @@
|
||||
"lib/insights/chart-tokens.ts": 24,
|
||||
"lib/insights/coach-launch-context.tsx": 10,
|
||||
"lib/insights/comprehensive-aggregator.ts": 27,
|
||||
"lib/insights/comprehensive-generate.ts": 45,
|
||||
"lib/insights/comprehensive-generate.ts": 44,
|
||||
"lib/insights/correlation-channel-series.ts": 16,
|
||||
"lib/insights/correlation-discovery.ts": 30,
|
||||
"lib/insights/correlation-series-builders.ts": 1,
|
||||
@@ -2299,7 +2299,7 @@
|
||||
"lib/jobs/google-health-oauth-state-cleanup.ts": 1,
|
||||
"lib/jobs/google-health-sleep-repair.ts": 1,
|
||||
"lib/jobs/insight-pregenerate-shared.ts": 1,
|
||||
"lib/jobs/insight-pregenerate.ts": 36,
|
||||
"lib/jobs/insight-pregenerate.ts": 33,
|
||||
"lib/jobs/insight-status-generate-shared.ts": 3,
|
||||
"lib/jobs/insight-status-generate.ts": 3,
|
||||
"lib/jobs/intake-auto-skip.ts": 8,
|
||||
|
||||
@@ -168,6 +168,11 @@ const ALLOWED: Readonly<Record<string, { count: number; reason: string }>> = {
|
||||
reason:
|
||||
"No pgboss schema on a web-only deployment; null means no queue to ask, and the evaluation then judges nothing.",
|
||||
},
|
||||
"lib/jobs/job-failures.ts::explainAllFailedRun": {
|
||||
count: 1,
|
||||
reason:
|
||||
"A pg-boss output that does not parse as a run's outcome is not an all-failed run; presentError then shows the raw text as it always did.",
|
||||
},
|
||||
"lib/jobs/job-failures.ts::readActiveJobsStartedBefore": {
|
||||
count: 1,
|
||||
reason:
|
||||
|
||||
@@ -12,7 +12,10 @@ import { queryKeys } from "@/lib/query-keys";
|
||||
import { useTranslations } from "@/lib/i18n/context";
|
||||
import { useInsightsLayoutPrefs } from "@/hooks/use-insights-layout-prefs";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { ChartSkeleton } from "@/components/charts/chart-skeleton";
|
||||
import {
|
||||
ChartSkeleton,
|
||||
MoodChartSkeleton,
|
||||
} from "@/components/charts/chart-skeleton";
|
||||
import { MetricEmptyState } from "@/components/insights/metric-empty-state";
|
||||
import { MetricTargetSummary } from "@/components/insights/metric-target-summary";
|
||||
import {
|
||||
@@ -54,7 +57,7 @@ const MoodChart = dynamic(
|
||||
import("@/components/charts/chart-runtime").then((mod) => ({
|
||||
default: mod.MoodChart,
|
||||
})),
|
||||
{ ssr: false, loading: () => <ChartSkeleton /> },
|
||||
{ ssr: false, loading: () => <MoodChartSkeleton /> },
|
||||
);
|
||||
|
||||
/**
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import { useEffect, useState } from "react";
|
||||
|
||||
import { Card, CardContent, CardHeader } from "@/components/ui/card";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { useTranslations } from "@/lib/i18n/context";
|
||||
@@ -50,12 +51,16 @@ const SLOW_HINT_DELAY_MS = 3_000;
|
||||
|
||||
/**
|
||||
* The footer a loaded `<HealthChart>` paints under its plot: the row of day
|
||||
* dots and its caption on a day-linked chart, and the collapsed "show as
|
||||
* table" disclosure. A loading shell without it was ~100 px shorter than the
|
||||
* card it turned into (CLS on every metric page, the BMI page most visibly),
|
||||
* so the shells reserve the same boxes: `h-3` rug, `mt-1` caption (two lines
|
||||
* on a phone, one from `sm`), and the disclosure's `mt-3 border-t pt-3` frame
|
||||
* around its `min-h-11` button.
|
||||
* dots on a day-linked chart, and the collapsed "show as table" disclosure. A
|
||||
* loading shell without it was ~100 px shorter than the card it turned into
|
||||
* (CLS on every metric page, the BMI page most visibly), so the shells reserve
|
||||
* the same boxes: the `h-3` rug, and the disclosure's `mt-3 border-t pt-3`
|
||||
* frame around its `min-h-11` button.
|
||||
*
|
||||
* The rug is all a day-linked chart paints under its plot. It used to carry a
|
||||
* one-line caption as well, and the reserve kept that line after the caption
|
||||
* went: every day-linked shell stood 20 px taller than its chart (40 px on a
|
||||
* phone) and the page shrank under the reader when the chart arrived.
|
||||
*/
|
||||
export function ChartFooterReserve({
|
||||
dayLinks = false,
|
||||
@@ -67,12 +72,7 @@ export function ChartFooterReserve({
|
||||
if (!dayLinks && !dataTable) return null;
|
||||
return (
|
||||
<div aria-hidden="true" data-slot="chart-footer-reserve">
|
||||
{dayLinks ? (
|
||||
<>
|
||||
<div className="h-3" />
|
||||
<div className="mt-1 h-9 sm:h-4" />
|
||||
</>
|
||||
) : null}
|
||||
{dayLinks ? <div className="h-3" /> : null}
|
||||
{dataTable ? (
|
||||
<div className="border-border mt-3 border-t pt-3">
|
||||
<div className="h-11" />
|
||||
@@ -172,3 +172,43 @@ export function ChartSkeleton({
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The loading shell for a full-size, day-linked `<MoodChart>`.
|
||||
*
|
||||
* The mood chart is drawn in a `Card` (vertical padding and a gap between
|
||||
* header and plot, where `ChartSkeleton` has padding and a margin), its plot
|
||||
* band is 200 / 220 px rather than 240 / 280, and it paints the day rug and
|
||||
* the collapsed data-table disclosure under the plot. Shown in the generic
|
||||
* shell, the card grew by about 30 px when the chart chunk arrived, under a
|
||||
* reader already scrolled past it. So it gets the same primitives and the
|
||||
* same boxes.
|
||||
*/
|
||||
export function MoodChartSkeleton() {
|
||||
const { t } = useTranslations();
|
||||
return (
|
||||
<Card
|
||||
data-slot="chart-skeleton"
|
||||
role="status"
|
||||
aria-busy="true"
|
||||
aria-live="polite"
|
||||
>
|
||||
<span className="sr-only">{t("charts.loadingLabel")}</span>
|
||||
<CardHeader>
|
||||
<div className="flex flex-col gap-2 sm:flex-row sm:items-center sm:justify-between">
|
||||
<Skeleton className="h-5 w-32" />
|
||||
<div className="flex h-11 items-center gap-2">
|
||||
<Skeleton className="h-6 w-12" />
|
||||
<Skeleton className="h-6 w-12" />
|
||||
<Skeleton className="h-6 w-12" />
|
||||
<Skeleton className="h-6 w-12" />
|
||||
</div>
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<Skeleton className="h-[var(--chart-height,200px)] w-full md:h-[var(--chart-height-md,220px)]" />
|
||||
<ChartFooterReserve dayLinks dataTable />
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@@ -4,7 +4,10 @@ import type { ComponentProps, ReactNode } from "react";
|
||||
import dynamic from "next/dynamic";
|
||||
|
||||
import { ChartErrorBoundary } from "@/components/charts/chart-error-state";
|
||||
import { ChartSkeleton } from "@/components/charts/chart-skeleton";
|
||||
import {
|
||||
ChartSkeleton,
|
||||
MoodChartSkeleton,
|
||||
} from "@/components/charts/chart-skeleton";
|
||||
import { HealthChartDynamic } from "@/components/charts/health-chart-dynamic";
|
||||
import { RecentWorkoutsTile } from "@/components/dashboard/recent-workouts-tile";
|
||||
import { RecentAchievementsCard } from "@/components/gamification/recent-achievements-card";
|
||||
@@ -26,7 +29,7 @@ const MoodChartLazy = dynamic(
|
||||
importWithRetry(() => import("@/components/charts/chart-runtime")).then(
|
||||
(mod) => ({ default: mod.MoodChart }),
|
||||
),
|
||||
{ ssr: false, loading: () => <ChartSkeleton /> },
|
||||
{ ssr: false, loading: () => <MoodChartSkeleton /> },
|
||||
);
|
||||
const MoodChart = (props: ComponentProps<typeof MoodChartLazy>) => (
|
||||
<ChartErrorBoundary>
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
__resetRecordFenceForTests,
|
||||
currentRecordFenceState,
|
||||
} from "@/lib/api/record-fence";
|
||||
import {
|
||||
__resetRecordScopeForTests,
|
||||
getRecordScope,
|
||||
setRecordScope,
|
||||
} from "@/lib/query-keys/record-scope";
|
||||
import {
|
||||
__resetRecordSessionTransitionForTests,
|
||||
beginRecordSessionTransition,
|
||||
commitRecordSessionTransition,
|
||||
getRecordSessionTransition,
|
||||
} from "@/lib/query-keys/record-session-transition";
|
||||
|
||||
const apiGet = vi.fn();
|
||||
|
||||
vi.mock("@/lib/api/api-fetch", () => ({
|
||||
apiGet: (...args: unknown[]) => apiGet(...args),
|
||||
apiFetchRaw: vi.fn(),
|
||||
ApiError: class ApiError extends Error {},
|
||||
}));
|
||||
|
||||
vi.mock("next/navigation", () => ({
|
||||
useRouter: () => ({ push: vi.fn(), replace: vi.fn() }),
|
||||
}));
|
||||
|
||||
import { fetchMe } from "../use-auth";
|
||||
|
||||
/**
|
||||
* A `/api/auth/me` answered across a record-session transition.
|
||||
*
|
||||
* The browser journey that found this: the switcher clears the query cache
|
||||
* when it raises the hold, which remounts the account query, so the initiating
|
||||
* tab has a `/me` in flight beside its own switch POST. The server answers it
|
||||
* with the record being left. Applied after the commit, that answer rewrote the
|
||||
* scope mirror to the old record, a settled peer read the write as a foreign
|
||||
* switch, and every tab then waited for a `/me` naming a record the server had
|
||||
* already left — behind the hydration gate, for good.
|
||||
*/
|
||||
|
||||
const TARGET = "record-b";
|
||||
|
||||
const entry = {
|
||||
accountId: TARGET,
|
||||
username: "record-b",
|
||||
displayName: null,
|
||||
fullName: null,
|
||||
access: "read",
|
||||
level: "read",
|
||||
recordKind: "shared",
|
||||
sections: null,
|
||||
canWrite: false,
|
||||
writableDomains: [],
|
||||
manageableDomains: [],
|
||||
};
|
||||
|
||||
const base = {
|
||||
id: "delegate",
|
||||
username: "delegate",
|
||||
role: "USER",
|
||||
timezone: "Europe/Berlin",
|
||||
};
|
||||
|
||||
/** What the server says before the switch lands: the caller's own record. */
|
||||
const beforeSwitch = {
|
||||
...base,
|
||||
accountAccess: {
|
||||
accounts: [entry],
|
||||
active: null,
|
||||
recordKind: "self",
|
||||
canSwitch: true,
|
||||
},
|
||||
recordSession: { epoch: 4, scope: null },
|
||||
};
|
||||
|
||||
/** And after it. */
|
||||
const afterSwitch = {
|
||||
...base,
|
||||
accountAccess: {
|
||||
accounts: [entry],
|
||||
active: entry,
|
||||
recordKind: "shared",
|
||||
canSwitch: true,
|
||||
},
|
||||
recordSession: { epoch: 5, scope: TARGET },
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
apiGet.mockReset();
|
||||
__resetRecordScopeForTests();
|
||||
__resetRecordSessionTransitionForTests();
|
||||
__resetRecordFenceForTests();
|
||||
});
|
||||
|
||||
describe("fetchMe across a record-session transition", () => {
|
||||
it("drops an answer that straddled the commit and reads again", async () => {
|
||||
setRecordScope(null);
|
||||
const id = beginRecordSessionTransition(TARGET);
|
||||
|
||||
apiGet
|
||||
.mockImplementationOnce(async () => {
|
||||
// The switch commits while this read is on the wire, and the server
|
||||
// had already answered it from the record being left.
|
||||
commitRecordSessionTransition(id, TARGET);
|
||||
return beforeSwitch;
|
||||
})
|
||||
.mockResolvedValueOnce(afterSwitch);
|
||||
|
||||
const user = await fetchMe();
|
||||
|
||||
expect(apiGet).toHaveBeenCalledTimes(2);
|
||||
expect(user.accountAccess?.active?.accountId).toBe(TARGET);
|
||||
// The committed scope survives; the pre-switch answer never reached the
|
||||
// mirror or the fence.
|
||||
expect(getRecordScope()).toBe(TARGET);
|
||||
expect(currentRecordFenceState()).toEqual({ epoch: 5, scope: TARGET });
|
||||
// And the hold releases on the read that describes the switched session.
|
||||
expect(getRecordSessionTransition().phase).toBe("ready");
|
||||
});
|
||||
|
||||
it("applies an answer when nothing moved while it was in flight", async () => {
|
||||
const id = beginRecordSessionTransition(TARGET);
|
||||
commitRecordSessionTransition(id, TARGET);
|
||||
apiGet.mockResolvedValueOnce(afterSwitch);
|
||||
|
||||
await fetchMe();
|
||||
|
||||
expect(apiGet).toHaveBeenCalledTimes(1);
|
||||
expect(getRecordScope()).toBe(TARGET);
|
||||
expect(getRecordSessionTransition().phase).toBe("ready");
|
||||
});
|
||||
|
||||
it("gives up without applying anything when the session never holds still", async () => {
|
||||
setRecordScope(null);
|
||||
apiGet.mockImplementation(async () => {
|
||||
// Every read straddles a fresh transition.
|
||||
beginRecordSessionTransition(TARGET);
|
||||
return beforeSwitch;
|
||||
});
|
||||
|
||||
await expect(fetchMe()).rejects.toThrow(/kept moving/);
|
||||
expect(apiGet).toHaveBeenCalledTimes(4);
|
||||
expect(currentRecordFenceState()).toBeNull();
|
||||
});
|
||||
});
|
||||
+42
-8
@@ -36,6 +36,7 @@ import {
|
||||
setRefusedRecordScope,
|
||||
} from "@/lib/query-keys/record-scope";
|
||||
import {
|
||||
recordSessionRevision,
|
||||
settleRecordSessionTransition,
|
||||
settleRefusedRecordSessionTransition,
|
||||
} from "@/lib/query-keys/record-session-transition";
|
||||
@@ -361,6 +362,46 @@ export function isAuthVerdictUnknown(error: unknown): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* How many times `fetchMe` asks again when a record-session transition moved
|
||||
* while its request was in flight. Each repeat needs another transition inside
|
||||
* one round trip, which only a browser switching records over and over
|
||||
* produces.
|
||||
*/
|
||||
const SUPERSEDED_ME_RETRIES = 3;
|
||||
|
||||
type MePayload = Partial<AuthUser> & {
|
||||
id: string;
|
||||
username: string;
|
||||
role: string;
|
||||
timezone: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* `GET /api/auth/me`, never answered across a record-session transition.
|
||||
*
|
||||
* Everything `fetchMe` does with the payload is a statement about which record
|
||||
* this browser is in: it adopts the fence context, rewrites the cache-scope
|
||||
* mirror every other tab watches, and may release the hold. A request issued
|
||||
* before a transition and answered after it may have been read on the far side
|
||||
* of a switch, and applying it is how a browser ended up wedged behind the
|
||||
* hydration gate (see `recordSessionRevision`). Such an answer is dropped and
|
||||
* the read repeated; the repeat is issued after the change, so it describes the
|
||||
* session as it now is.
|
||||
*/
|
||||
async function getCurrentMe(): Promise<MePayload> {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
const before = recordSessionRevision();
|
||||
const data = await apiGet<MePayload>("/api/auth/me");
|
||||
if (recordSessionRevision() === before) return data;
|
||||
if (attempt >= SUPERSEDED_ME_RETRIES) {
|
||||
// Not an `ApiError`: no verdict on the session, so nothing is wiped and
|
||||
// the hold stays until a later read settles it.
|
||||
throw new Error("record session kept moving during /api/auth/me");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchMe(): Promise<AuthUser> {
|
||||
// v1.16.4 — routed through the typed wrapper; a non-OK /me (401)
|
||||
// throws `ApiError`, which `useAuth` treats as "not authenticated"
|
||||
@@ -369,14 +410,7 @@ export async function fetchMe(): Promise<AuthUser> {
|
||||
// v1.4.47 — coerce `disableCoach` against `undefined` so a stale
|
||||
// /me payload from a partial-deploy rollback (older server image
|
||||
// without the field) keeps the Coach surface visible by default.
|
||||
const data = await apiGet<
|
||||
Partial<AuthUser> & {
|
||||
id: string;
|
||||
username: string;
|
||||
role: string;
|
||||
timezone: string;
|
||||
}
|
||||
>("/api/auth/me");
|
||||
const data = await getCurrentMe();
|
||||
// Hour-cycle preference: coerce against a stale /me payload, then mirror
|
||||
// into localStorage so `useFormatters()` and the legacy format helpers
|
||||
// (which cannot reach the query cache) render the same clock.
|
||||
|
||||
@@ -209,4 +209,59 @@ describe("comprehensive warm-path timeout + failure observability", () => {
|
||||
(failAnnotation![0] as { meta?: { reason?: string } }).meta?.reason,
|
||||
).toBe("all-providers-failed");
|
||||
});
|
||||
|
||||
// #1384 — the nightly pass needs to know which configured provider failed,
|
||||
// whose credential a refusal was, and whether anything was sent at all.
|
||||
it("names the provider that failed first and whether its credential was refused", async () => {
|
||||
runRawCompletionWithFallback.mockRejectedValue(
|
||||
Object.assign(new AllProvidersFailedError([]), {
|
||||
attempts: [
|
||||
{
|
||||
providerType: "openai-compatible",
|
||||
attempt: 1,
|
||||
failureReason: "401",
|
||||
httpStatus: 401,
|
||||
},
|
||||
],
|
||||
primaryCredentialExpired: true,
|
||||
sentNothing: false,
|
||||
}),
|
||||
);
|
||||
|
||||
const outcome = await generateComprehensiveInsight("u1", { locale: "de" });
|
||||
|
||||
expect(outcome).toEqual({
|
||||
status: "failed",
|
||||
reason: "all-providers-failed",
|
||||
credentialExpired: true,
|
||||
provider: "openai-compatible",
|
||||
});
|
||||
});
|
||||
|
||||
it("says so when every provider was benched and nothing was sent (#1279)", async () => {
|
||||
runRawCompletionWithFallback.mockRejectedValue(
|
||||
Object.assign(new AllProvidersFailedError([]), {
|
||||
attempts: [
|
||||
{
|
||||
providerType: "local",
|
||||
attempt: 1,
|
||||
failureReason: "benched-for-background-work",
|
||||
httpStatus: null,
|
||||
failureKind: "paused",
|
||||
},
|
||||
],
|
||||
primaryCredentialExpired: false,
|
||||
sentNothing: true,
|
||||
}),
|
||||
);
|
||||
|
||||
const outcome = await generateComprehensiveInsight("u1", { locale: "de" });
|
||||
|
||||
expect(outcome).toEqual({
|
||||
status: "failed",
|
||||
reason: "all-providers-failed",
|
||||
provider: "local",
|
||||
sentNothing: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -134,6 +134,7 @@ describe("generateStatusBatchForUser — the statusText capability comes first",
|
||||
served: 0,
|
||||
batched: 0,
|
||||
fellBack: 0,
|
||||
fellBackAnswered: 0,
|
||||
batchCallMade: false,
|
||||
});
|
||||
});
|
||||
@@ -340,6 +341,15 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => {
|
||||
vi.mocked(runStatusCompletion).mockResolvedValue({
|
||||
kind: "error",
|
||||
} as never);
|
||||
// #1384 — one card's own call came back with a model note; the other six
|
||||
// served their fallback. A nightly warm over a provider that only failed
|
||||
// reported `assessments_warmed: 6`, because every fallback counted.
|
||||
vi.mocked(runPreparedStatusCard).mockResolvedValueOnce({
|
||||
hasProvider: true,
|
||||
text: "note",
|
||||
cached: false,
|
||||
updatedAt: "t",
|
||||
});
|
||||
|
||||
const result = await generateStatusBatchForUser("u1", { locale: "en" });
|
||||
|
||||
@@ -347,6 +357,7 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => {
|
||||
for (const f of finalizers) expect(f).not.toHaveBeenCalled();
|
||||
expect(runPreparedStatusCard).toHaveBeenCalledTimes(7);
|
||||
expect(result.fellBack).toBe(7);
|
||||
expect(result.fellBackAnswered).toBe(1);
|
||||
expect(result.batched).toBe(0);
|
||||
});
|
||||
|
||||
@@ -411,6 +422,7 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => {
|
||||
for (const f of finalizers) expect(f).not.toHaveBeenCalled();
|
||||
expect(runPreparedStatusCard).not.toHaveBeenCalled();
|
||||
expect(result.fellBack).toBe(7);
|
||||
expect(result.fellBackAnswered).toBe(0);
|
||||
expect(result.batchCallMade).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,37 @@
|
||||
/**
|
||||
* What a failed briefing generation knows about its provider chain beyond
|
||||
* the reason (#1384): which configured provider failed first, whether that
|
||||
* provider refused the credential, and whether anything was sent at all.
|
||||
*
|
||||
* The nightly pass needs all three to settle an account honestly. A refusal
|
||||
* of the account's own credential is the account's matter, a refusal of the
|
||||
* operator's shared key is the operator's, and a chain whose every entry the
|
||||
* provider health ledger benched for background work (#1279) sent no request,
|
||||
* so the account was not attempted. Without them every such account read as
|
||||
* a failure, night after night.
|
||||
*/
|
||||
import { AllProvidersFailedError } from "@/lib/ai/provider-runner";
|
||||
|
||||
export interface ChainFailureFacts {
|
||||
/**
|
||||
* The chain's first provider refused the credential (a dead Codex
|
||||
* sign-in, a revoked key). No retry can succeed until someone reconnects
|
||||
* or replaces it.
|
||||
*/
|
||||
credentialExpired?: boolean;
|
||||
/** The chain entry that failed first, e.g. `openai-compatible`. */
|
||||
provider?: string;
|
||||
/** Every entry was benched for background work; no request left. */
|
||||
sentNothing?: boolean;
|
||||
}
|
||||
|
||||
/** The facts of a thrown chain failure; empty for any other error. */
|
||||
export function chainFailureFacts(error: unknown): ChainFailureFacts {
|
||||
if (!(error instanceof AllProvidersFailedError)) return {};
|
||||
const provider = error.attempts?.[0]?.providerType;
|
||||
return {
|
||||
...(error.primaryCredentialExpired ? { credentialExpired: true } : {}),
|
||||
...(provider ? { provider } : {}),
|
||||
...(error.sentNothing ? { sentNothing: true } : {}),
|
||||
};
|
||||
}
|
||||
@@ -93,6 +93,10 @@ import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic";
|
||||
import { insightResultSchema, type InsightResult } from "@/lib/ai/types";
|
||||
import { resolveProviderChain } from "@/lib/ai/provider";
|
||||
import { AllProvidersFailedError } from "@/lib/ai/provider-runner";
|
||||
import {
|
||||
chainFailureFacts,
|
||||
type ChainFailureFacts,
|
||||
} from "@/lib/insights/chain-failure-facts";
|
||||
import {
|
||||
BriefingBudgetExceededError,
|
||||
runBriefingCompletion,
|
||||
@@ -178,17 +182,7 @@ export type GenerateOutcome =
|
||||
| "scope-changed"
|
||||
| "profile-scope-changed";
|
||||
}
|
||||
| {
|
||||
status: "failed";
|
||||
reason: string;
|
||||
/**
|
||||
* v1.42 — the chain's first provider refused the credential (a dead
|
||||
* Codex sign-in, a revoked key). The nightly pass reports the account
|
||||
* as `auth_failed` and does not queue a retry that cannot succeed until
|
||||
* the person reconnects.
|
||||
*/
|
||||
credentialExpired?: boolean;
|
||||
};
|
||||
| ({ status: "failed"; reason: string } & ChainFailureFacts);
|
||||
|
||||
/** Higher, seedless temperature for the daily-briefing phrasing re-roll. */
|
||||
const BRIEFING_REROLL_TEMPERATURE = 0.6;
|
||||
@@ -1290,11 +1284,7 @@ export async function generateComprehensiveInsight(
|
||||
? (e.attempts?.[0]?.failureKind ?? null)
|
||||
: providerFailureKind(e).failure,
|
||||
});
|
||||
const credentialExpired =
|
||||
e instanceof AllProvidersFailedError && e.primaryCredentialExpired;
|
||||
return credentialExpired
|
||||
? { status: "failed", reason, credentialExpired }
|
||||
: { status: "failed", reason };
|
||||
return { status: "failed", reason, ...chainFailureFacts(e) };
|
||||
}
|
||||
|
||||
// Anthropic + local have no native JSON mode, so a ```json-fenced or
|
||||
|
||||
@@ -37,6 +37,7 @@ import {
|
||||
runPreparedStatusCard,
|
||||
type PendingStatusCard,
|
||||
type PreparedStatusCard,
|
||||
type StatusCardResult,
|
||||
} from "@/lib/insights/status-card-generation";
|
||||
import { runStatusCompletion } from "@/lib/insights/status-provider";
|
||||
import {
|
||||
@@ -77,10 +78,27 @@ export interface StatusBatchResult {
|
||||
/** Metrics that fell back to a single-card call (omitted by the batch, or
|
||||
* the batch failed). */
|
||||
fellBack: number;
|
||||
/**
|
||||
* Of `fellBack`, the metrics whose own single-card call came back with a
|
||||
* model-written assessment. The rest served their deterministic fallback:
|
||||
* no provider, a failed or cooling-down provider, or a screened reply.
|
||||
* `batched + fellBackAnswered` is what a warm pass may call warmed (#1384);
|
||||
* `fellBack` alone counted a night of six failed calls as six assessments.
|
||||
*/
|
||||
fellBackAnswered: number;
|
||||
/** True when the one batched provider call was issued. */
|
||||
batchCallMade: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a single-card run ended in a model-written assessment: the persisted
|
||||
* shape is the only one with a provider and no cache flag. The no-key, the
|
||||
* timeout / error stub and the screened fallback all carry `cached: true`.
|
||||
*/
|
||||
function modelAnswered(result: StatusCardResult): boolean {
|
||||
return result.hasProvider === true && result.cached === false;
|
||||
}
|
||||
|
||||
/** Pull `perMetric` out of a (possibly fenced) model completion. */
|
||||
function parsePerMetric(content: string): Record<string, string> | null {
|
||||
const tryParse = (raw: string): Record<string, string> | null => {
|
||||
@@ -119,6 +137,7 @@ export async function generateStatusBatchForUser(
|
||||
served: 0,
|
||||
batched: 0,
|
||||
fellBack: 0,
|
||||
fellBackAnswered: 0,
|
||||
batchCallMade: false,
|
||||
};
|
||||
|
||||
@@ -171,11 +190,19 @@ export async function generateStatusBatchForUser(
|
||||
// One metric still needing the LLM is not worth a batch envelope — run it
|
||||
// through its own single-card path (identical persisted result).
|
||||
if (pending.length === 1) {
|
||||
await runPreparedStatusCard(pending[0]);
|
||||
if (modelAnswered(await runPreparedStatusCard(pending[0]))) {
|
||||
result.fellBackAnswered++;
|
||||
}
|
||||
result.fellBack++;
|
||||
annotate({
|
||||
action: { name: "insights.status.batch" },
|
||||
meta: { served: result.served, batched: 0, fell_back: 1, single: true },
|
||||
meta: {
|
||||
served: result.served,
|
||||
batched: 0,
|
||||
fell_back: 1,
|
||||
fell_back_answered: result.fellBackAnswered,
|
||||
single: true,
|
||||
},
|
||||
});
|
||||
return result;
|
||||
}
|
||||
@@ -272,14 +299,19 @@ export async function generateStatusBatchForUser(
|
||||
// Batch missed this metric (omitted key, or timeout/error/invalid-json):
|
||||
// fall back to the single-card path. On a batch timeout/error this writes
|
||||
// the per-card negative stub via the card's own `timeout`.
|
||||
await runPreparedStatusCard(card);
|
||||
return "fellBack" as const;
|
||||
return modelAnswered(await runPreparedStatusCard(card))
|
||||
? ("fellBackAnswered" as const)
|
||||
: ("fellBack" as const);
|
||||
});
|
||||
|
||||
const outcomes = await Promise.all(fanOuts);
|
||||
for (const o of outcomes) {
|
||||
if (o === "batched") result.batched++;
|
||||
else result.fellBack++;
|
||||
if (o === "batched") {
|
||||
result.batched++;
|
||||
continue;
|
||||
}
|
||||
result.fellBack++;
|
||||
if (o === "fellBackAnswered") result.fellBackAnswered++;
|
||||
}
|
||||
|
||||
annotate({
|
||||
@@ -288,6 +320,7 @@ export async function generateStatusBatchForUser(
|
||||
served: result.served,
|
||||
batched: result.batched,
|
||||
fell_back: result.fellBack,
|
||||
fell_back_answered: result.fellBackAnswered,
|
||||
pending: pending.length,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -103,6 +103,7 @@ vi.mock("@/lib/insights/status-batch", () => ({
|
||||
served: 0,
|
||||
batched: 0,
|
||||
fellBack: 0,
|
||||
fellBackAnswered: 0,
|
||||
batchCallMade: false,
|
||||
})),
|
||||
}));
|
||||
@@ -132,6 +133,7 @@ import {
|
||||
INSIGHT_PREGENERATE_EXPIRE_SECONDS,
|
||||
} from "../insight-pregenerate-shared";
|
||||
import { handleInsightPregenerateJob } from "../reminder/insights-handlers";
|
||||
import { judgeJobRun, type reportJobRun } from "../job-run-report";
|
||||
import { insightRetryOptions } from "../reminder/registrar-shared";
|
||||
import {
|
||||
groupOf,
|
||||
@@ -1556,15 +1558,9 @@ describe("runInsightPregenerate — warm budget honours the response timeout (v1
|
||||
// withheld (screened) briefing and a dead sign-in queue no provider retry,
|
||||
// and a time-budget cut-off reports the unreached accounts as deferred.
|
||||
describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
|
||||
type Reported = {
|
||||
queue: string;
|
||||
runId: string;
|
||||
candidates: Array<{ key: string; outcome: string; cause?: string }>;
|
||||
};
|
||||
|
||||
function harness(users: Array<{ id: string; locale: string }>) {
|
||||
const { prisma } = makePrisma(users);
|
||||
const reportRun = vi.fn(async (input: Reported) => ({
|
||||
const reportRun = vi.fn<typeof reportJobRun>(async (input) => ({
|
||||
total: input.candidates.length,
|
||||
attempted: 0,
|
||||
failed: 0,
|
||||
@@ -1606,7 +1602,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
|
||||
statusGenerators: h.statusGenerators,
|
||||
warmGenericMetrics: h.warmGenericMetrics,
|
||||
enqueueRetry: h.enqueueRetry,
|
||||
reportRun: h.reportRun as never,
|
||||
reportRun: h.reportRun,
|
||||
});
|
||||
|
||||
expect(h.reportRun).toHaveBeenCalledTimes(1);
|
||||
@@ -1639,7 +1635,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
|
||||
statusGenerators: h.statusGenerators,
|
||||
warmGenericMetrics: h.warmGenericMetrics,
|
||||
enqueueRetry: h.enqueueRetry,
|
||||
reportRun: h.reportRun as never,
|
||||
reportRun: h.reportRun,
|
||||
});
|
||||
|
||||
expect(h.enqueueRetry).toHaveBeenCalledTimes(1);
|
||||
@@ -1668,7 +1664,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
|
||||
statusGenerators: h.statusGenerators,
|
||||
warmGenericMetrics: h.warmGenericMetrics,
|
||||
enqueueRetry: h.enqueueRetry,
|
||||
reportRun: h.reportRun as never,
|
||||
reportRun: h.reportRun,
|
||||
shouldStop: () => ++calls > 1,
|
||||
});
|
||||
|
||||
@@ -1677,3 +1673,121 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
|
||||
).toEqual(["ok", "deferred", "deferred"]);
|
||||
});
|
||||
});
|
||||
|
||||
// #1384 — a self-hoster whose only account kept a provider that no longer
|
||||
// works was told every night that the job was failing. The run used the real
|
||||
// verdict here, so what the alert reads (`all_failed`) is what is asserted.
|
||||
describe("runInsightPregenerate — a provider that cannot serve is not a failing run (#1384)", () => {
|
||||
async function runOne(outcome: Record<string, unknown>) {
|
||||
const { prisma } = makePrisma([{ id: "only-account", locale: "en" }]);
|
||||
const reportRun = vi.fn<typeof reportJobRun>(async (input) =>
|
||||
judgeJobRun(input.candidates),
|
||||
);
|
||||
const enqueueRetry = vi.fn().mockResolvedValue(undefined);
|
||||
const result = await runInsightPregenerate(prisma as never, {
|
||||
generate: vi.fn().mockResolvedValue(outcome),
|
||||
statusGenerators: Array.from({ length: 7 }, () =>
|
||||
vi.fn().mockResolvedValue({ hasProvider: true, cached: true }),
|
||||
),
|
||||
warmGenericMetrics: vi.fn().mockResolvedValue(0),
|
||||
enqueueRetry,
|
||||
reportRun,
|
||||
});
|
||||
return {
|
||||
result,
|
||||
enqueueRetry,
|
||||
candidate: reportRun.mock.calls[0][0].candidates[0],
|
||||
};
|
||||
}
|
||||
|
||||
it("skips an account whose every provider was benched, so nothing was sent", async () => {
|
||||
const { result, enqueueRetry, candidate } = await runOne({
|
||||
status: "failed",
|
||||
reason: "all-providers-failed",
|
||||
provider: "local",
|
||||
sentNothing: true,
|
||||
});
|
||||
|
||||
expect(candidate).toEqual(
|
||||
expect.objectContaining({
|
||||
outcome: "skipped",
|
||||
cause: "provider_benched",
|
||||
}),
|
||||
);
|
||||
expect(result.skipped).toBe(1);
|
||||
expect(result.failed).toBe(0);
|
||||
expect(result.allFailed).toBe(false);
|
||||
// Nobody attempted: the handler then writes no `all_failed`, so a benched
|
||||
// night neither extends nor ends the alert's streak.
|
||||
expect(result.attempted).toBe(0);
|
||||
expect(result.firstFailure).toBeNull();
|
||||
// The retry would find the same bench.
|
||||
expect(enqueueRetry).not.toHaveBeenCalled();
|
||||
expect(annotateSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
action: { name: "insights.pregenerate.provider_benched" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("reports a refused credential of the account's own as auth_failed, naming the provider, without failing the run", async () => {
|
||||
const { result, enqueueRetry, candidate } = await runOne({
|
||||
status: "failed",
|
||||
reason: "all-providers-failed",
|
||||
provider: "openai-compatible",
|
||||
credentialExpired: true,
|
||||
});
|
||||
|
||||
expect(candidate).toEqual(
|
||||
expect.objectContaining({
|
||||
outcome: "auth_failed",
|
||||
cause: "all-providers-failed:openai-compatible",
|
||||
}),
|
||||
);
|
||||
expect(result.allFailed).toBe(false);
|
||||
expect(result.attempted).toBe(1);
|
||||
expect(enqueueRetry).not.toHaveBeenCalled();
|
||||
expect(annotateSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
action: { name: "insights.pregenerate.comprehensive_failed" },
|
||||
meta: expect.objectContaining({
|
||||
provider: "openai-compatible",
|
||||
credential_refused: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("still fails the run when the operator's own key was refused, and says which", async () => {
|
||||
const { result, enqueueRetry, candidate } = await runOne({
|
||||
status: "failed",
|
||||
reason: "all-providers-failed",
|
||||
provider: "admin-openai",
|
||||
credentialExpired: true,
|
||||
});
|
||||
|
||||
expect(candidate.outcome).toBe("failed");
|
||||
expect(result.allFailed).toBe(true);
|
||||
expect(result.firstFailure).toEqual({
|
||||
reason: "all-providers-failed",
|
||||
provider: "admin-openai",
|
||||
});
|
||||
// A refused key does not heal in 45 minutes.
|
||||
expect(enqueueRetry).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("carries the cause and provider of an ordinary failure into the run's facts", async () => {
|
||||
const { result, enqueueRetry } = await runOne({
|
||||
status: "failed",
|
||||
reason: "provider-error",
|
||||
provider: "local",
|
||||
});
|
||||
|
||||
expect(result.allFailed).toBe(true);
|
||||
expect(result.firstFailure).toEqual({
|
||||
reason: "provider-error",
|
||||
provider: "local",
|
||||
});
|
||||
expect(enqueueRetry).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -193,3 +193,36 @@ describe("bounded generic JobOutcome serialization", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// #1384 — a generator reason and a provider type become facts the admin page
|
||||
// and the repeated-failure alert can show.
|
||||
describe("codeFact", () => {
|
||||
it("turns our own hyphenated codes into stable fact codes", async () => {
|
||||
const { codeFact, serializeJobOutcome } = await vi.importActual<
|
||||
typeof import("@/lib/jobs/job-outcome")
|
||||
>("@/lib/jobs/job-outcome");
|
||||
expect(codeFact("all-providers-failed")).toBe("all_providers_failed");
|
||||
expect(codeFact("openai-compatible")).toBe("openai_compatible");
|
||||
expect(codeFact("generator:invalid-json")).toBe("generator_invalid_json");
|
||||
expect(codeFact("timeout")).toBe("timeout");
|
||||
// Nothing usable: the caller leaves the fact out.
|
||||
expect(codeFact("")).toBeNull();
|
||||
expect(codeFact("---")).toBeNull();
|
||||
expect(codeFact(null)).toBeNull();
|
||||
// What it produces is a fact the serializer accepts.
|
||||
expect(
|
||||
serializeJobOutcome(
|
||||
jobDone({
|
||||
failure_cause: codeFact("all-providers-failed")!,
|
||||
failure_provider: codeFact("admin-openai")!,
|
||||
}),
|
||||
),
|
||||
).toEqual({
|
||||
ok: true,
|
||||
did: {
|
||||
failure_cause: "all_providers_failed",
|
||||
failure_provider: "admin_openai",
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -47,13 +47,27 @@ describe("judgeJobRun", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("counts a dead credential as a failure", () => {
|
||||
expect(
|
||||
judgeJobRun([
|
||||
{ key: "a", outcome: "auth_failed" },
|
||||
{ key: "b", outcome: "failed" },
|
||||
]).allFailed,
|
||||
).toBe(true);
|
||||
// #1384 — a key the issuer revoked stays saved and is refused every night.
|
||||
// Counted as a failure, it made a single-account instance's nightly pass
|
||||
// read as all-failed forever and alerted the operator about the job.
|
||||
it("does not fail a run over a refused credential of the account's own", () => {
|
||||
expect(judgeJobRun([{ key: "a", outcome: "auth_failed" }])).toEqual({
|
||||
total: 1,
|
||||
attempted: 1,
|
||||
failed: 0,
|
||||
allFailed: false,
|
||||
partial: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("reads a refused credential beside a real failure as partial", () => {
|
||||
const verdict = judgeJobRun([
|
||||
{ key: "a", outcome: "auth_failed" },
|
||||
{ key: "b", outcome: "failed" },
|
||||
]);
|
||||
expect(verdict.allFailed).toBe(false);
|
||||
expect(verdict.partial).toBe(true);
|
||||
expect(verdict.failed).toBe(1);
|
||||
});
|
||||
|
||||
it("reads one success among failures as partial, not failed", () => {
|
||||
@@ -84,7 +98,7 @@ describe("reportJobRun", () => {
|
||||
queue: "insight-pregenerate",
|
||||
runId: "r1",
|
||||
candidates: [
|
||||
{ key: "a", outcome: "auth_failed", cause: "all-providers-failed" },
|
||||
{ key: "a", outcome: "failed", cause: "all-providers-failed" },
|
||||
{ key: "b", outcome: "skipped", cause: "no-consent" },
|
||||
],
|
||||
});
|
||||
@@ -101,7 +115,31 @@ describe("reportJobRun", () => {
|
||||
consecutiveFailures: 3,
|
||||
failed: 1,
|
||||
total: 1,
|
||||
causes: ["auth_failed:all-providers-failed"],
|
||||
causes: ["failed:all-providers-failed"],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("emits job.run.partial, never job.run.failed, for a run whose only attempt was a refused credential (#1384)", async () => {
|
||||
const verdict = await reportJobRun({
|
||||
queue: "insight-pregenerate",
|
||||
runId: "r1",
|
||||
candidates: [
|
||||
{
|
||||
key: "a",
|
||||
outcome: "auth_failed",
|
||||
cause: "all-providers-failed:openai-compatible",
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(verdict.allFailed).toBe(false);
|
||||
expect(readConsecutiveRunFailures).not.toHaveBeenCalled();
|
||||
expect(emitSignal).toHaveBeenCalledTimes(1);
|
||||
expect(emitSignal.mock.calls[0][0]).toMatchObject({
|
||||
action: "job.run.partial",
|
||||
level: "warn",
|
||||
meta: {
|
||||
causes: ["auth_failed:all-providers-failed:openai-compatible"],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
@@ -87,6 +87,34 @@ describe("judgeQueueRuns", () => {
|
||||
).toBe("failing");
|
||||
});
|
||||
|
||||
// #1384 — the alert quoted the completed run's own outcome JSON, and the
|
||||
// operator it sent to the admin page found nothing there.
|
||||
it("says in words what an all-failed night was, with its cause and provider", () => {
|
||||
const output = JSON.stringify({
|
||||
ok: true,
|
||||
did: {
|
||||
total: 1,
|
||||
failed: 1,
|
||||
all_failed: true,
|
||||
assessments_warmed: 0,
|
||||
failure_cause: "all_providers_failed",
|
||||
failure_provider: "admin_openai",
|
||||
},
|
||||
});
|
||||
const verdict = judgeQueueRuns(
|
||||
[0, 1, 2].map((ago) =>
|
||||
run(ago, { state: "completed", allFailed: true, error: output }),
|
||||
),
|
||||
);
|
||||
expect(verdict.state).toBe("failing");
|
||||
expect(verdict.lastError).toBe(
|
||||
"The run completed, but every account it attempted failed (1 of 1). " +
|
||||
"First cause: all_providers_failed from provider admin_openai. " +
|
||||
"Accounts without a usable provider are skipped and not counted; " +
|
||||
"job.run.failed in the app log lists each account's outcome.",
|
||||
);
|
||||
});
|
||||
|
||||
it("explains a bare timeout the way the admin card does", () => {
|
||||
const verdict = judgeQueueRuns(
|
||||
[0, 1, 2].map((ago) =>
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* How the nightly briefing pass settles an account whose generation came back
|
||||
* `failed` (#1384).
|
||||
*
|
||||
* A failure used to be one of three things: screened, a dead sign-in, or a
|
||||
* provider failure worth a retry in 45 minutes. Two cases were missing, and
|
||||
* both made a self-hoster's only account read as a failing run every night,
|
||||
* which the repeated-failure alert then reported as a broken job:
|
||||
*
|
||||
* - Every provider of the chain was benched for background work by the
|
||||
* provider health ledger (#1279), so no request was sent. The account was
|
||||
* not attempted tonight; its earlier failures were counted by the runs
|
||||
* that made them, and a retry would find the same bench. It is `skipped`.
|
||||
* - The provider refused the credential. A refused credential does not heal
|
||||
* in 45 minutes, so no retry is queued, and whose credential it was
|
||||
* decides who has to act. The operator's shared key or central sign-in is
|
||||
* the operator's to fix, and a run where it was refused for everyone is a
|
||||
* failing run: `failed`. The account's own key, gateway or sign-in is its
|
||||
* owner's to fix: `auth_failed`, which the run reports but does not fail
|
||||
* over (`job-run-report.ts`). An API key revoked at its issuer stays saved
|
||||
* and is refused again every night, unlike a ChatGPT sign-in, which the
|
||||
* refusal parks.
|
||||
*
|
||||
* The model answered and the safety screen withheld it (`outbound-screened`)
|
||||
* stays `screened` with no retry: asking the same model again in 45 minutes
|
||||
* is the same question, and it was screened eight times a night.
|
||||
*/
|
||||
import { isOperatorHeldProvider } from "@/lib/ai/provider-egress";
|
||||
import type { GenerateOutcome } from "@/lib/insights/comprehensive-generate";
|
||||
import type { JobRunCandidate, JobRunOutcome } from "@/lib/jobs/job-run-report";
|
||||
|
||||
type FailedGeneration = Extract<GenerateOutcome, { status: "failed" }>;
|
||||
|
||||
export interface FailedGenerationVerdict {
|
||||
outcome: JobRunOutcome;
|
||||
/** The run line's cause: the reason, and the provider that failed first. */
|
||||
cause: string;
|
||||
/** False when nothing was sent: the account is tallied as skipped. */
|
||||
attempted: boolean;
|
||||
/** Queue the bounded 45-minute retry for this account. */
|
||||
retry: boolean;
|
||||
}
|
||||
|
||||
export function settleFailedGeneration(
|
||||
generated: FailedGeneration,
|
||||
): FailedGenerationVerdict {
|
||||
if (generated.sentNothing) {
|
||||
return {
|
||||
outcome: "skipped",
|
||||
cause: "provider_benched",
|
||||
attempted: false,
|
||||
retry: false,
|
||||
};
|
||||
}
|
||||
const cause = generated.provider
|
||||
? `${generated.reason}:${generated.provider}`
|
||||
: generated.reason;
|
||||
if (generated.reason === "outbound-screened") {
|
||||
return { outcome: "screened", cause, attempted: true, retry: false };
|
||||
}
|
||||
if (generated.credentialExpired) {
|
||||
const operatorHeld =
|
||||
generated.provider !== undefined &&
|
||||
isOperatorHeldProvider(generated.provider);
|
||||
return {
|
||||
outcome: operatorHeld ? "failed" : "auth_failed",
|
||||
cause,
|
||||
attempted: true,
|
||||
retry: false,
|
||||
};
|
||||
}
|
||||
return { outcome: "failed", cause, attempted: true, retry: true };
|
||||
}
|
||||
|
||||
/** The first failed account of a run: its reason and the provider, if known. */
|
||||
export interface FirstFailure {
|
||||
reason: string;
|
||||
provider: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The first failed account's reason and provider, read back off the run's
|
||||
* causes, for the job's facts: what the admin page and the repeated-failure
|
||||
* alert show for a run that completed with every account failed.
|
||||
*/
|
||||
export function firstFailure(
|
||||
candidates: readonly JobRunCandidate[],
|
||||
): FirstFailure | null {
|
||||
const cause = candidates.find((c) => c.outcome === "failed")?.cause;
|
||||
if (!cause) return null;
|
||||
const [reason, provider] = cause.split(":");
|
||||
return { reason, provider: provider ?? null };
|
||||
}
|
||||
@@ -81,6 +81,10 @@ import {
|
||||
} from "@/lib/insights/metric-status-registry";
|
||||
import { withTimeout } from "@/lib/insights/with-timeout";
|
||||
import { comprehensiveWarmBudgetMs } from "@/lib/jobs/model-time";
|
||||
import {
|
||||
firstFailure,
|
||||
settleFailedGeneration,
|
||||
} from "@/lib/jobs/insight-pregenerate-failure";
|
||||
import {
|
||||
INSIGHT_PREGENERATE_QUEUE,
|
||||
enqueueForceWarm,
|
||||
@@ -212,12 +216,13 @@ export interface PregenerateRunResult {
|
||||
/** The job's time budget ran out before every candidate was reached. */
|
||||
stoppedEarly: boolean;
|
||||
/**
|
||||
* v1.42 — every candidate the run attempted failed (see `reportJobRun`).
|
||||
* Written into the job's facts as `all_failed`, which is how a completed
|
||||
* night with nothing generated still counts toward the repeated-failure
|
||||
* streak.
|
||||
* v1.42 — every candidate the run attempted failed (see `reportJobRun`),
|
||||
* written into the job's facts as `all_failed` so a completed night with
|
||||
* nothing generated counts toward the repeated-failure streak. `attempted`
|
||||
* (#1384) goes out as `run_attempted`; at 0 the streak steps over the night.
|
||||
*/
|
||||
allFailed: boolean;
|
||||
attempted: number;
|
||||
/**
|
||||
* Count of per-metric assessment caches written warm across the whole
|
||||
* run. The comprehensive generator evicts the seven `*-status`
|
||||
@@ -235,6 +240,8 @@ export interface PregenerateRunResult {
|
||||
* seven specialised cards.
|
||||
*/
|
||||
metricAssessmentsWarmed: number;
|
||||
/** #1384 — the first failed account's reason and provider, as facts. */
|
||||
firstFailure: ReturnType<typeof firstFailure>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -302,9 +309,9 @@ async function warmPerStatusCaches(
|
||||
* once and issues ONE provider call for the metrics still needing the LLM,
|
||||
* fanning the response back into the same per-metric cache rows the
|
||||
* standalone generators wrote. Replaces the prior seven independent warm
|
||||
* calls. Returns the count of metrics that got a fresh (batched or
|
||||
* fallback) assessment so the wide-event tally is comparable to the old
|
||||
* per-card count.
|
||||
* calls. Returns the count of metrics that got a fresh model-written
|
||||
* assessment, batched or through a card's own fallback call, so the
|
||||
* wide-event tally is comparable to the old per-card count.
|
||||
*
|
||||
* Refill-only (`force: false`): a card already generated today and a card
|
||||
* whose content-hash is unchanged both resolve inside the batch as `served`
|
||||
@@ -322,7 +329,7 @@ async function warmStatusBatch(
|
||||
locale,
|
||||
force: false,
|
||||
});
|
||||
warmed += result.batched + result.fellBack;
|
||||
warmed += result.batched + result.fellBackAnswered;
|
||||
} catch (err) {
|
||||
// The batch must never throw the cron's user loop off course — but
|
||||
// a swallowed batch failure left ~7 cards cold with zero signal.
|
||||
@@ -553,8 +560,10 @@ export async function runInsightPregenerate(
|
||||
budgetBlocked: 0,
|
||||
stoppedEarly: false,
|
||||
allFailed: false,
|
||||
attempted: 0,
|
||||
assessmentsWarmed: 0,
|
||||
metricAssessmentsWarmed: 0,
|
||||
firstFailure: null,
|
||||
};
|
||||
|
||||
// Master kill-switch — when the operator disabled the briefing
|
||||
@@ -619,6 +628,8 @@ export async function runInsightPregenerate(
|
||||
candidates: runOutcomes,
|
||||
});
|
||||
result.allFailed = verdict.allFailed;
|
||||
result.attempted = verdict.attempted;
|
||||
result.firstFailure = firstFailure(runOutcomes);
|
||||
|
||||
return result;
|
||||
|
||||
@@ -783,42 +794,34 @@ export async function runInsightPregenerate(
|
||||
result.skipped++;
|
||||
settle("skipped", generated.reason);
|
||||
break;
|
||||
case "failed":
|
||||
result.failed++;
|
||||
// v1.28.30 — THE previously-silent failure path. A generator
|
||||
// that resolves with `{ status: "failed" }` (features error,
|
||||
// oversize payload, invalid JSON after retry, abort) bumped
|
||||
// the tally with no annotation at all — the nightly summary
|
||||
// showed `failed: 1` and nothing else was greppable. Reuse
|
||||
// the `comprehensive_failed` shape with the generator's own
|
||||
// reason as the cause.
|
||||
case "failed": {
|
||||
// Benched, screened, a refused credential and whose, or a
|
||||
// retryable failure: `insight-pregenerate-failure.ts` (#1384).
|
||||
const verdict = settleFailedGeneration(generated);
|
||||
if (verdict.attempted) result.failed++;
|
||||
else result.skipped++;
|
||||
annotate({
|
||||
action: { name: "insights.pregenerate.comprehensive_failed" },
|
||||
action: {
|
||||
name: verdict.attempted
|
||||
? "insights.pregenerate.comprehensive_failed"
|
||||
: "insights.pregenerate.provider_benched",
|
||||
},
|
||||
meta: {
|
||||
locale,
|
||||
stage: "nightly.generator",
|
||||
cause: `generator:${generated.reason}`,
|
||||
provider: generated.provider ?? null,
|
||||
credential_refused: generated.credentialExpired === true,
|
||||
budget_ms: warmBudgetMs,
|
||||
message: null,
|
||||
},
|
||||
});
|
||||
if (generated.reason === "outbound-screened") {
|
||||
// v1.42 — the model answered and the safety screen withheld
|
||||
// it, after the generator's own repair. Asking the same model
|
||||
// again in 45 minutes is not a provider retry; it is the same
|
||||
// question, and it was screened eight times a night.
|
||||
settle("screened", generated.reason);
|
||||
} else if (generated.credentialExpired) {
|
||||
// v1.42 — a dead sign-in (a Codex refresh token the issuer
|
||||
// revoked) does not heal in 45 minutes either; the person has
|
||||
// to reconnect. The refusal parked the connection as expired,
|
||||
// which Settings shows, and the next run no longer counts it.
|
||||
settle("auth_failed", generated.reason);
|
||||
} else {
|
||||
settle("failed", generated.reason);
|
||||
settle(verdict.outcome, verdict.cause);
|
||||
if (verdict.retry) {
|
||||
await enqueueRetry({ userId: candidate.id, locale });
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,7 +92,7 @@ const MAX_ERROR_CHARS = 300;
|
||||
export interface FailingQueue {
|
||||
/** The pg-boss queue name, e.g. `environment-fetch`. */
|
||||
queue: string;
|
||||
/** Jobs that exhausted their retries inside the window. */
|
||||
/** Failed runs inside the window: exhausted retries, or all-failed passes. */
|
||||
failures: number;
|
||||
/** ISO instant of the newest failure. */
|
||||
lastFailedAt: string;
|
||||
@@ -140,10 +140,48 @@ function explainTimeout(message: string, expireSeconds: number | null): string {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Say what a completed run that failed every account means (#1384). Such a
|
||||
* run's `output` is its outcome, `{"ok":true,"did":{…,"all_failed":true}}`,
|
||||
* and quoting that JSON told the operator nothing: not that the job ran, not
|
||||
* which provider failed. Null when `raw` is not such an outcome.
|
||||
*/
|
||||
function explainAllFailedRun(raw: string): string | null {
|
||||
if (!raw.startsWith("{")) return null;
|
||||
let did: Record<string, unknown>;
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as { ok?: unknown; did?: unknown };
|
||||
if (parsed?.ok !== true || typeof parsed.did !== "object") return null;
|
||||
did = (parsed.did ?? {}) as Record<string, unknown>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (did.all_failed !== true) return null;
|
||||
// Every attempted account failed, so the count is both sides of "of".
|
||||
const failed =
|
||||
typeof did.failed === "number" && did.failed > 0
|
||||
? ` (${did.failed} of ${did.failed})`
|
||||
: "";
|
||||
// "none" is how the run says it has no cause or provider to name.
|
||||
const named = (value: unknown): string | null =>
|
||||
typeof value === "string" && value !== "none" ? value : null;
|
||||
const cause = named(did.failure_cause);
|
||||
const provider = named(did.failure_provider);
|
||||
const why = cause
|
||||
? ` First cause: ${cause}${provider ? ` from provider ${provider}` : ""}.`
|
||||
: "";
|
||||
return (
|
||||
`The run completed, but every account it attempted failed${failed}.${why} ` +
|
||||
`Accounts without a usable provider are skipped and not counted; ` +
|
||||
`job.run.failed in the app log lists each account's outcome.`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn whatever pg-boss stored in `output` into one line. A thrown `Error`
|
||||
* serialises to `{ message, stack, … }`; a thrown non-Error can be any JSON at
|
||||
* all, so the raw text is the fallback rather than an invented placeholder.
|
||||
* A completed run that reported every account failed is said in words.
|
||||
*/
|
||||
export function presentError(
|
||||
raw: string | null,
|
||||
@@ -152,6 +190,8 @@ export function presentError(
|
||||
if (raw === null) return "";
|
||||
const trimmed = raw.trim();
|
||||
if (trimmed.length === 0) return "";
|
||||
const allFailed = explainAllFailedRun(trimmed);
|
||||
if (allFailed !== null) return allFailed;
|
||||
if (TIMEOUT_MESSAGE.test(trimmed))
|
||||
return explainTimeout(trimmed, expireSeconds);
|
||||
if (HEARTBEAT_MESSAGE.test(trimmed)) {
|
||||
@@ -168,8 +208,13 @@ export function presentError(
|
||||
}
|
||||
|
||||
/**
|
||||
* Every queue with at least one exhausted-retry failure in the window, newest
|
||||
* first. `null` when the queue schema is not present at all.
|
||||
* Every queue with at least one failed run in the window, newest first:
|
||||
* a job that exhausted its retries, or a fan-out pass that completed but
|
||||
* reported that every account it attempted failed (`did.all_failed`). The
|
||||
* second kind is what the repeated-failure alert counts too
|
||||
* (`readConsecutiveRunFailures`, `system-job-alert.ts`); the page used to
|
||||
* list only the first, so an operator the alert sent here read "no job has
|
||||
* failed" (#1384). `null` when the queue schema is not present at all.
|
||||
*/
|
||||
export async function readFailingQueues(
|
||||
withinHours: number = JOB_FAILURE_WINDOW_HOURS,
|
||||
@@ -194,7 +239,10 @@ export async function readFailingQueues(
|
||||
array_agg(expire_seconds ORDER BY completed_on DESC NULLS LAST)
|
||||
)[1] AS last_expire_seconds
|
||||
FROM pgboss.job
|
||||
WHERE state = 'failed'
|
||||
WHERE (
|
||||
state = 'failed'
|
||||
OR (state = 'completed' AND output->'did'->>'all_failed' = 'true')
|
||||
)
|
||||
AND completed_on > now() - make_interval(hours => ${withinHours}::int)
|
||||
GROUP BY name
|
||||
ORDER BY max(completed_on) DESC
|
||||
@@ -449,6 +497,11 @@ export async function readConsecutiveRunFailures(
|
||||
AND state IN ('completed', 'failed')
|
||||
AND completed_on IS NOT NULL
|
||||
AND (state = 'failed' OR output->'did'->>'all_failed' IS NOT NULL)
|
||||
-- #1384: a run that attempted nobody neither fails nor recovers.
|
||||
AND NOT (
|
||||
state = 'completed'
|
||||
AND COALESCE(output->'did'->>'run_attempted', '') = '0'
|
||||
)
|
||||
ORDER BY completed_on DESC
|
||||
LIMIT ${CONSECUTIVE_LOOKBACK_ROWS}
|
||||
`;
|
||||
|
||||
@@ -130,6 +130,11 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
|
||||
"duration_ms",
|
||||
"errored",
|
||||
"failed",
|
||||
// #1384 — a fan-out pass's first failed candidate: its cause and the
|
||||
// provider that failed for it, as stable codes. What the admin page and
|
||||
// the repeated-failure alert show for a run that completed all-failed.
|
||||
"failure_cause",
|
||||
"failure_provider",
|
||||
"failures",
|
||||
"feedback_buckets",
|
||||
"feedback_total_rows",
|
||||
@@ -239,6 +244,9 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
|
||||
"removed",
|
||||
"resting_corrected",
|
||||
"rows_written",
|
||||
// #1384 — how many accounts a fan-out pass attempted; 0 marks a run that
|
||||
// neither failed nor recovered, which the failure streak steps over.
|
||||
"run_attempted",
|
||||
// v1.39.1 — the background restore of a stored backup, and the sweep that
|
||||
// re-queues one a stopped worker left running. Counts and a claim flag.
|
||||
"restore_claimed",
|
||||
@@ -423,6 +431,22 @@ export function percentFact(ratio: number | null | undefined): number {
|
||||
return wholeFact(ratio * 100);
|
||||
}
|
||||
|
||||
/**
|
||||
* A short code from our own vocabulary (a generator reason, a provider type)
|
||||
* as a fact: `all-providers-failed` becomes `all_providers_failed`. Null when
|
||||
* nothing of it survives, so the caller leaves the fact out.
|
||||
*/
|
||||
export function codeFact(value: string | null | undefined): string | null {
|
||||
if (value == null) return null;
|
||||
const code = value
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "_")
|
||||
.replace(/^[^a-z]+/, "")
|
||||
.slice(0, 64)
|
||||
.replace(/_+$/, "");
|
||||
return STABLE_CODE_PATTERN.test(code) ? code : null;
|
||||
}
|
||||
|
||||
/** Why one fact is refused, or null when it is a valid fact. */
|
||||
function invalidFact(key: string, value: unknown): string | null {
|
||||
if (!JOB_FACT_ALLOWLIST.has(key))
|
||||
|
||||
@@ -21,6 +21,18 @@
|
||||
* attempted but is not a provider failure, so a run of only screened
|
||||
* candidates is partial, not failed.
|
||||
*
|
||||
* `auth_failed` is attempted and is not a failure of the run either (#1384).
|
||||
* It means the provider refused the account's own credential: a key revoked
|
||||
* at its issuer, a gateway that now wants a key the account no longer
|
||||
* stores. Only that account's owner can change it, and the run itself worked.
|
||||
* It used to count as a failure on the assumption that a refusal is reported
|
||||
* once and then parked; that holds for a ChatGPT sign-in and for nothing else,
|
||||
* so an API key the issuer had revoked failed every night for as long as it
|
||||
* stayed saved. On an instance where that account was the only candidate the
|
||||
* run then read as all-failed, and the operator was told the job was broken
|
||||
* night after night. A refusal of the operator's own credential is reported by
|
||||
* the caller as `failed`, because there the operator is the one who can act.
|
||||
*
|
||||
* The verdict comes back to the caller, which writes `all_failed` into its
|
||||
* `jobDone` facts. That fact is what lets `readConsecutiveRunFailures` count a
|
||||
* completed-but-all-failed night as part of a streak.
|
||||
@@ -46,18 +58,26 @@ export interface JobRunVerdict {
|
||||
total: number;
|
||||
/** Candidates it actually attempted (not skipped, not deferred). */
|
||||
attempted: number;
|
||||
/** `failed` + `auth_failed`. */
|
||||
/** Candidates that failed (`failed`; a refused credential is not one). */
|
||||
failed: number;
|
||||
/** Every attempted candidate failed (and at least one was attempted). */
|
||||
allFailed: boolean;
|
||||
/** Something failed or was withheld, but not everything. */
|
||||
/**
|
||||
* Something failed, was withheld or had its credential refused, but not
|
||||
* everything failed.
|
||||
*/
|
||||
partial: boolean;
|
||||
}
|
||||
|
||||
/** Candidates listed on the run's line; the counts always cover all of them. */
|
||||
const MAX_LISTED_CANDIDATES = 50;
|
||||
|
||||
const FAILED: ReadonlySet<JobRunOutcome> = new Set(["failed", "auth_failed"]);
|
||||
const FAILED: ReadonlySet<JobRunOutcome> = new Set(["failed"]);
|
||||
/** Attempted, not a failure of the run, but worth a `partial` line. */
|
||||
const WITHHELD: ReadonlySet<JobRunOutcome> = new Set([
|
||||
"screened",
|
||||
"auth_failed",
|
||||
]);
|
||||
const NOT_ATTEMPTED: ReadonlySet<JobRunOutcome> = new Set([
|
||||
"skipped",
|
||||
"deferred",
|
||||
@@ -72,14 +92,14 @@ export function judgeJobRun(
|
||||
(c) => !NOT_ATTEMPTED.has(c.outcome),
|
||||
).length;
|
||||
const failed = candidates.filter((c) => FAILED.has(c.outcome)).length;
|
||||
const screened = candidates.filter((c) => c.outcome === "screened").length;
|
||||
const withheld = candidates.filter((c) => WITHHELD.has(c.outcome)).length;
|
||||
const allFailed = attempted > 0 && failed === attempted;
|
||||
return {
|
||||
total,
|
||||
attempted,
|
||||
failed,
|
||||
allFailed,
|
||||
partial: !allFailed && (failed > 0 || screened > 0),
|
||||
partial: !allFailed && (failed > 0 || withheld > 0),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ import {
|
||||
type InsightStatusGeneratePayload,
|
||||
runInsightStatusGenerate,
|
||||
} from "@/lib/jobs/insight-status-generate";
|
||||
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
|
||||
import { codeFact, jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
|
||||
import { jobBudget } from "@/lib/jobs/job-budget";
|
||||
import { withBackgroundEvent } from "@/lib/logging/background";
|
||||
import { generateStatusBatchForUser } from "@/lib/insights/status-batch";
|
||||
@@ -80,7 +80,7 @@ export async function runStatusBatchCron(
|
||||
locale: await resolveJobLocale(user.locale),
|
||||
force: false,
|
||||
});
|
||||
generated += result.batched + result.fellBack;
|
||||
generated += result.batched + result.fellBackAnswered;
|
||||
served += result.served;
|
||||
} catch (error) {
|
||||
failed++;
|
||||
@@ -201,8 +201,17 @@ export async function handleInsightPregenerateJob(
|
||||
// v1.42 — read back by `readConsecutiveRunFailures`, so a night that
|
||||
// completed with nothing generated counts toward the failure streak.
|
||||
all_failed: summary.allFailed,
|
||||
// #1384 — a night that attempted no account (every provider paused)
|
||||
// is neither a failure nor a recovery. `run_attempted: 0` is how the
|
||||
// streak readers (`readConsecutiveRunFailures`, the repeated-failure
|
||||
// alert) know to step over it instead of ending the streak.
|
||||
run_attempted: summary.attempted,
|
||||
assessments_warmed: summary.assessmentsWarmed,
|
||||
metric_assessments_warmed: summary.metricAssessmentsWarmed,
|
||||
// #1384 — what failed, for the admin page and the alert ("none"
|
||||
// when nothing did, or the generator could not say).
|
||||
failure_cause: codeFact(summary.firstFailure?.reason) ?? "none",
|
||||
failure_provider: codeFact(summary.firstFailure?.provider) ?? "none",
|
||||
});
|
||||
} catch (err) {
|
||||
evt.addWarning(
|
||||
|
||||
@@ -197,6 +197,12 @@ export async function readTerminalRuns(
|
||||
WHERE name = q.name
|
||||
AND completed_on IS NOT NULL
|
||||
AND state IN ('completed', 'failed')
|
||||
-- #1384: a fan-out run that attempted nobody neither fails nor
|
||||
-- recovers; it must not end a streak.
|
||||
AND NOT (
|
||||
state = 'completed'
|
||||
AND COALESCE(output->'did'->>'run_attempted', '') = '0'
|
||||
)
|
||||
ORDER BY completed_on DESC
|
||||
LIMIT ${LOOKBACK_ROWS}
|
||||
) j
|
||||
|
||||
@@ -51,6 +51,8 @@ const READY: RecordSessionTransition = {
|
||||
|
||||
let transition = READY;
|
||||
let seeded = false;
|
||||
/** Bumped on every change this tab applies; see `recordSessionRevision`. */
|
||||
let revision = 0;
|
||||
const listeners = new Set<() => void>();
|
||||
let channel: BroadcastChannel | null = null;
|
||||
let storageListenerAttached = false;
|
||||
@@ -107,6 +109,7 @@ function applyWire(next: WireTransition): void {
|
||||
}
|
||||
transition = toSnapshot(next);
|
||||
seeded = true;
|
||||
revision += 1;
|
||||
notify();
|
||||
}
|
||||
|
||||
@@ -228,6 +231,26 @@ export function getRecordSessionTransition(): RecordSessionTransition {
|
||||
return transition;
|
||||
}
|
||||
|
||||
/**
|
||||
* A counter that moves whenever this tab applies a transition change — its own
|
||||
* publish, or one received from another tab.
|
||||
*
|
||||
* `fetchMe` reads it on both sides of its request. A `/api/auth/me` answered
|
||||
* across a change cannot say which side of that change the server read it on:
|
||||
* the switcher's own `queryClient.clear()` remounts the account query the
|
||||
* moment the hold goes up, so the initiator routinely has a `/me` in flight
|
||||
* beside its own `POST /api/account/switch`, and that answer can describe the
|
||||
* record the session has just left. Applied after the commit, it rewrites the
|
||||
* scope mirror back to the old record; every peer that has already settled
|
||||
* reads that write as a foreign switch and holds for a `/me` naming the old
|
||||
* record, which no server answer will ever give. The counter is how `fetchMe`
|
||||
* recognises such an answer and asks again instead of applying it.
|
||||
*/
|
||||
export function recordSessionRevision(): number {
|
||||
seed();
|
||||
return revision;
|
||||
}
|
||||
|
||||
export function subscribeToRecordSessionTransition(
|
||||
listener: () => void,
|
||||
): () => void {
|
||||
|
||||
@@ -460,6 +460,7 @@
|
||||
"batchCallMade": true,
|
||||
"batched": 6,
|
||||
"fellBack": 1,
|
||||
"fellBackAnswered": 0,
|
||||
"served": 0
|
||||
}
|
||||
},
|
||||
@@ -1719,6 +1720,7 @@
|
||||
"batchCallMade": true,
|
||||
"batched": 0,
|
||||
"fellBack": 7,
|
||||
"fellBackAnswered": 0,
|
||||
"served": 0
|
||||
}
|
||||
},
|
||||
@@ -1831,6 +1833,7 @@
|
||||
"batchCallMade": false,
|
||||
"batched": 0,
|
||||
"fellBack": 1,
|
||||
"fellBackAnswered": 0,
|
||||
"served": 6
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,6 +40,7 @@ vi.mock("@/lib/insights/status-batch", async () => {
|
||||
served: 0,
|
||||
batched: 0,
|
||||
fellBack: 0,
|
||||
fellBackAnswered: 0,
|
||||
batchCallMade: false,
|
||||
};
|
||||
if (!(await aiCapabilityForJob(userId, "statusText")).available) {
|
||||
|
||||
@@ -126,6 +126,41 @@ describe("readFailingQueues", () => {
|
||||
await expect(readFailingQueues()).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
// #1384 — the repeated-failure alert counts a nightly pass that completed
|
||||
// with every account failed, and sent the operator to this page, which
|
||||
// listed only pg-boss failures and said nothing had failed.
|
||||
it("lists a run that completed with every account failed, in words, and not a good run", async () => {
|
||||
const ALL_FAILED_QUEUE = "insight-pregenerate";
|
||||
await boss.createQueue(ALL_FAILED_QUEUE);
|
||||
async function completeOne(did: Record<string, unknown>) {
|
||||
const jobId = await boss.send(ALL_FAILED_QUEUE, {}, { retryLimit: 0 });
|
||||
const [job] = await boss.fetch(ALL_FAILED_QUEUE);
|
||||
if (!job || job.id !== jobId) throw new Error("nothing to fetch");
|
||||
await boss.complete(ALL_FAILED_QUEUE, job.id, { ok: true, did });
|
||||
}
|
||||
// A good night, and the night from the report: one account, one failure.
|
||||
await completeOne({ total: 1, generated: 1, failed: 0, all_failed: false });
|
||||
await completeOne({
|
||||
total: 1,
|
||||
generated: 0,
|
||||
failed: 1,
|
||||
all_failed: true,
|
||||
assessments_warmed: 0,
|
||||
failure_cause: "all_providers_failed",
|
||||
failure_provider: "openai_compatible",
|
||||
});
|
||||
|
||||
const failing = await readFailingQueues();
|
||||
const entry = failing?.find((e) => e.queue === ALL_FAILED_QUEUE);
|
||||
expect(entry?.failures).toBe(1);
|
||||
expect(entry?.lastError).toBe(
|
||||
"The run completed, but every account it attempted failed (1 of 1). " +
|
||||
"First cause: all_providers_failed from provider openai_compatible. " +
|
||||
"Accounts without a usable provider are skipped and not counted; " +
|
||||
"job.run.failed in the app log lists each account's outcome.",
|
||||
);
|
||||
});
|
||||
|
||||
it("ignores a failure older than the window", async () => {
|
||||
await failOneJob(QUEUE, { userId: "user-a" }, "old failure");
|
||||
await getPrismaClient().$executeRawUnsafe(
|
||||
@@ -233,6 +268,19 @@ describe("readConsecutiveRunFailures (v1.42)", () => {
|
||||
await expect(readConsecutiveRunFailures(STREAK_QUEUE)).resolves.toBe(3);
|
||||
});
|
||||
|
||||
it("steps over a night that attempted nobody instead of ending the streak (#1384)", async () => {
|
||||
await boss.createQueue(STREAK_QUEUE);
|
||||
// Failed, then every provider paused (nothing attempted), then failed.
|
||||
await finishedMinutesAgo(await completeOneJob({ all_failed: true }), 30);
|
||||
await finishedMinutesAgo(
|
||||
await completeOneJob({ all_failed: false, run_attempted: 0 }),
|
||||
20,
|
||||
);
|
||||
await finishedMinutesAgo(await completeOneJob({ all_failed: true }), 10);
|
||||
|
||||
await expect(readConsecutiveRunFailures(STREAK_QUEUE)).resolves.toBe(3);
|
||||
});
|
||||
|
||||
it("answers 1 for a queue with no history: the run failing now", async () => {
|
||||
await expect(readConsecutiveRunFailures("never-ran")).resolves.toBe(1);
|
||||
});
|
||||
|
||||
@@ -62,6 +62,7 @@ async function runOnce(
|
||||
outcome: "fail" | "complete",
|
||||
userId = "u",
|
||||
retryLimit = 0,
|
||||
output: Record<string, unknown> = { ok: true },
|
||||
) {
|
||||
const id = await boss.send(
|
||||
queue,
|
||||
@@ -80,7 +81,7 @@ async function runOnce(
|
||||
new Error(`canceling statement due to statement timeout (${userId})`),
|
||||
);
|
||||
} else {
|
||||
await boss.complete(queue, job.id, { ok: true });
|
||||
await boss.complete(queue, job.id, output);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -193,6 +194,21 @@ describe("notifySystemJobFailures", () => {
|
||||
expect(outcome.sent).toEqual([]);
|
||||
});
|
||||
|
||||
it("lets a run that attempted nobody neither end nor extend the streak (#1384)", async () => {
|
||||
await makeUser({ role: "ADMIN" });
|
||||
// What the briefing pass writes on a night every provider was paused.
|
||||
const benched = {
|
||||
ok: true,
|
||||
did: { all_failed: false, run_attempted: 0, skipped: 1 },
|
||||
};
|
||||
await runOnce(SYSTEM_QUEUE, "fail");
|
||||
await runOnce(SYSTEM_QUEUE, "fail");
|
||||
await runOnce(SYSTEM_QUEUE, "complete", "u", 0, benched);
|
||||
expect((await notifySystemJobFailures()).failing).toBe(0);
|
||||
await runOnce(SYSTEM_QUEUE, "fail");
|
||||
expect((await notifySystemJobFailures()).failing).toBe(1);
|
||||
});
|
||||
|
||||
it("counts a job that used up its retries as one failure", async () => {
|
||||
await makeUser({ role: "ADMIN" });
|
||||
await runOnce(SYSTEM_QUEUE, "fail", "u", 2);
|
||||
|
||||
Reference in new issue
Block a user