mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-16 15:23:29 +00:00
Merge branch 'fix/1384-pregenerate-without-provider' into release/v1.47.6
This commit is contained in:
commit
f2cf316ad8
32 files changed
+675
-94
No files matched your search
@@ -47,6 +47,21 @@
|
||||
default was rejected as in the future, and onset dates could display as the
|
||||
wrong calendar day in non-UTC time zones. Saving now anchors to the current
|
||||
time when before local noon and preserves stored timestamps on edit.
|
||||
- **No more nightly "job still failing" alert over a provider that cannot
|
||||
answer.** When the only account on an instance still had an AI provider
|
||||
saved whose key the provider no longer accepted (removing the key from a
|
||||
Local or OpenAI-compatible provider keeps its address, so it is still
|
||||
called), or whose provider HealthLog had paused after repeated errors, the
|
||||
nightly briefing pass counted that account as a failure, and every night
|
||||
the admins were told the job was broken. A key the provider refuses is now
|
||||
that account's matter: the run notes it but does not fail over it, unless
|
||||
it is the instance's own shared key. An account whose providers are all
|
||||
paused is skipped, since nothing was sent. The Background jobs card on the
|
||||
admin status page now lists a run that finished with every account
|
||||
failing, the same runs the alert counts, and the alert and the card say in
|
||||
words what failed and which provider, instead of quoting the run's
|
||||
counters. Those counters also stop counting a fallback text as a warmed
|
||||
assessment. (#1384)
|
||||
|
||||
### Security
|
||||
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Hintergrundjobs",
|
||||
"description": "Queues, deren Jobs nach allen Wiederholungen aufgegeben haben.",
|
||||
"description": "Queues, deren Jobs nach allen Wiederholungen aufgaben oder deren Lauf bei jedem versuchten Konto scheiterte.",
|
||||
"none": "In den letzten {hours} Stunden ist kein Job fehlgeschlagen.",
|
||||
"unavailable": "Keine Job-Queue lesbar. Dieses Deployment läuft ohne Hintergrund-Worker.",
|
||||
"countOne": "{count} Fehlschlag",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Background jobs",
|
||||
"description": "Queues whose jobs gave up after exhausting their retries.",
|
||||
"description": "Queues whose jobs gave up after their retries, or whose run ended with every account it tried failing.",
|
||||
"none": "No job has failed in the last {hours} hours.",
|
||||
"unavailable": "No job queue to read. This deployment runs without the background worker.",
|
||||
"countOne": "{count} failure",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Trabajos en segundo plano",
|
||||
"description": "Colas cuyos trabajos se rindieron tras agotar sus reintentos.",
|
||||
"description": "Colas cuyos trabajos se rindieron tras sus reintentos o cuya ejecución falló en cada cuenta intentada.",
|
||||
"none": "Ningún trabajo ha fallado en las últimas {hours} horas.",
|
||||
"unavailable": "No hay ninguna cola que consultar: este despliegue funciona sin el worker en segundo plano.",
|
||||
"countOne": "{count} fallo",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Tâches en arrière-plan",
|
||||
"description": "Files dont les tâches ont abandonné après avoir épuisé leurs tentatives.",
|
||||
"description": "Files dont les tâches ont abandonné après leurs tentatives, ou dont l’exécution a échoué pour chaque compte.",
|
||||
"none": "Aucune tâche n’a échoué au cours des {hours} dernières heures.",
|
||||
"unavailable": "Aucune file à consulter : ce déploiement fonctionne sans le worker en arrière-plan.",
|
||||
"countOne": "{count} échec",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Processi in background",
|
||||
"description": "Code i cui processi hanno rinunciato dopo aver esaurito i tentativi.",
|
||||
"description": "Code i cui processi hanno rinunciato dopo i tentativi o la cui esecuzione è fallita per ogni account.",
|
||||
"none": "Nessun processo è fallito nelle ultime {hours} ore.",
|
||||
"unavailable": "Nessuna coda da leggere: questo deployment funziona senza il worker in background.",
|
||||
"countOne": "{count} errore",
|
||||
|
||||
+1
-1
@@ -8743,7 +8743,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "백그라운드 작업",
|
||||
"description": "재시도를 다 쓰고 포기한 작업이 있는 큐예요.",
|
||||
"description": "재시도를 다 쓰고 포기했거나, 실행에서 시도한 모든 계정이 실패한 큐예요.",
|
||||
"none": "최근 {hours}시간 동안 실패한 작업이 없어요.",
|
||||
"unavailable": "읽을 작업 큐가 없어요. 이 배포본은 백그라운드 워커 없이 돌고 있어요.",
|
||||
"countOne": "실패 {count}건",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Achtergrondtaken",
|
||||
"description": "Wachtrijen waarvan taken het hebben opgegeven nadat alle nieuwe pogingen waren verbruikt.",
|
||||
"description": "Wachtrijen waarvan taken na alle pogingen opgaven, of waarvan een run voor elk geprobeerd account mislukte.",
|
||||
"none": "Er is de afgelopen {hours} uur geen taak mislukt.",
|
||||
"unavailable": "Geen takenwachtrij om te lezen. Deze installatie draait zonder de achtergrondworker.",
|
||||
"countOne": "{count} fout",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Zadania w tle",
|
||||
"description": "Kolejki, których zadania poddały się po wyczerpaniu ponowień.",
|
||||
"description": "Kolejki, których zadania poddały się po ponowieniach albo których przebieg zawiódł dla każdego konta.",
|
||||
"none": "W ciągu ostatnich {hours} godzin żadne zadanie nie zawiodło.",
|
||||
"unavailable": "Brak kolejki zadań do odczytu: to wdrożenie działa bez workera w tle.",
|
||||
"countOne": "{count} niepowodzenie",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Tarefas em segundo plano",
|
||||
"description": "Filas cujas tarefas desistiram depois de esgotar as tentativas.",
|
||||
"description": "Filas cujas tarefas desistiram após as tentativas, ou cuja execução falhou em todas as contas tentadas.",
|
||||
"none": "Nenhuma tarefa falhou nas últimas {hours} horas.",
|
||||
"unavailable": "Nenhuma fila de tarefas para ler. Esta implantação roda sem o worker em segundo plano.",
|
||||
"countOne": "{count} falha",
|
||||
|
||||
+1
-1
@@ -8768,7 +8768,7 @@
|
||||
},
|
||||
"failingJobs": {
|
||||
"title": "Фоновые задания",
|
||||
"description": "Очереди, задания которых сдались после исчерпания повторных попыток.",
|
||||
"description": "Очереди, задания которых сдались после всех попыток или чей запуск не удался ни для одного аккаунта.",
|
||||
"none": "За последние {hours} часов ни одно задание не завершилось неудачей.",
|
||||
"unavailable": "Нет очереди заданий для чтения. Это развёртывание работает без фонового воркера.",
|
||||
"countOne": "{count} неудача",
|
||||
|
||||
@@ -435,7 +435,7 @@
|
||||
"lib/jobs/__tests__/document-summary-reaper.test.ts": 1,
|
||||
"lib/jobs/__tests__/document-summary.test.ts": 19,
|
||||
"lib/jobs/__tests__/geo-backfill.test.ts": 1,
|
||||
"lib/jobs/__tests__/insight-pregenerate.test.ts": 45,
|
||||
"lib/jobs/__tests__/insight-pregenerate.test.ts": 43,
|
||||
"lib/jobs/__tests__/insight-status-generate.test.ts": 2,
|
||||
"lib/jobs/__tests__/measurement-maintenance-schedule.test.ts": 2,
|
||||
"lib/jobs/__tests__/measurement-reminder-byhour.test.ts": 2,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.",
|
||||
"count": 2863,
|
||||
"total": 9122,
|
||||
"total": 9118,
|
||||
"entries": {
|
||||
"__tests__/account-payload-consumer-guard.test.ts": 2,
|
||||
"__tests__/acting-account-boundary-guard.test.ts": 7,
|
||||
@@ -2134,7 +2134,7 @@
|
||||
"lib/insights/chart-tokens.ts": 24,
|
||||
"lib/insights/coach-launch-context.tsx": 10,
|
||||
"lib/insights/comprehensive-aggregator.ts": 27,
|
||||
"lib/insights/comprehensive-generate.ts": 45,
|
||||
"lib/insights/comprehensive-generate.ts": 44,
|
||||
"lib/insights/correlation-channel-series.ts": 16,
|
||||
"lib/insights/correlation-discovery.ts": 30,
|
||||
"lib/insights/correlation-series-builders.ts": 1,
|
||||
@@ -2299,7 +2299,7 @@
|
||||
"lib/jobs/google-health-oauth-state-cleanup.ts": 1,
|
||||
"lib/jobs/google-health-sleep-repair.ts": 1,
|
||||
"lib/jobs/insight-pregenerate-shared.ts": 1,
|
||||
"lib/jobs/insight-pregenerate.ts": 36,
|
||||
"lib/jobs/insight-pregenerate.ts": 33,
|
||||
"lib/jobs/insight-status-generate-shared.ts": 3,
|
||||
"lib/jobs/insight-status-generate.ts": 3,
|
||||
"lib/jobs/intake-auto-skip.ts": 8,
|
||||
|
||||
@@ -168,6 +168,11 @@ const ALLOWED: Readonly<Record<string, { count: number; reason: string }>> = {
|
||||
reason:
|
||||
"No pgboss schema on a web-only deployment; null means no queue to ask, and the evaluation then judges nothing.",
|
||||
},
|
||||
"lib/jobs/job-failures.ts::explainAllFailedRun": {
|
||||
count: 1,
|
||||
reason:
|
||||
"A pg-boss output that does not parse as a run's outcome is not an all-failed run; presentError then shows the raw text as it always did.",
|
||||
},
|
||||
"lib/jobs/job-failures.ts::readActiveJobsStartedBefore": {
|
||||
count: 1,
|
||||
reason:
|
||||
|
||||
@@ -209,4 +209,59 @@ describe("comprehensive warm-path timeout + failure observability", () => {
|
||||
(failAnnotation![0] as { meta?: { reason?: string } }).meta?.reason,
|
||||
).toBe("all-providers-failed");
|
||||
});
|
||||
|
||||
// #1384 — the nightly pass needs to know which configured provider failed,
|
||||
// whose credential a refusal was, and whether anything was sent at all.
|
||||
it("names the provider that failed first and whether its credential was refused", async () => {
|
||||
runRawCompletionWithFallback.mockRejectedValue(
|
||||
Object.assign(new AllProvidersFailedError([]), {
|
||||
attempts: [
|
||||
{
|
||||
providerType: "openai-compatible",
|
||||
attempt: 1,
|
||||
failureReason: "401",
|
||||
httpStatus: 401,
|
||||
},
|
||||
],
|
||||
primaryCredentialExpired: true,
|
||||
sentNothing: false,
|
||||
}),
|
||||
);
|
||||
|
||||
const outcome = await generateComprehensiveInsight("u1", { locale: "de" });
|
||||
|
||||
expect(outcome).toEqual({
|
||||
status: "failed",
|
||||
reason: "all-providers-failed",
|
||||
credentialExpired: true,
|
||||
provider: "openai-compatible",
|
||||
});
|
||||
});
|
||||
|
||||
it("says so when every provider was benched and nothing was sent (#1279)", async () => {
|
||||
runRawCompletionWithFallback.mockRejectedValue(
|
||||
Object.assign(new AllProvidersFailedError([]), {
|
||||
attempts: [
|
||||
{
|
||||
providerType: "local",
|
||||
attempt: 1,
|
||||
failureReason: "benched-for-background-work",
|
||||
httpStatus: null,
|
||||
failureKind: "paused",
|
||||
},
|
||||
],
|
||||
primaryCredentialExpired: false,
|
||||
sentNothing: true,
|
||||
}),
|
||||
);
|
||||
|
||||
const outcome = await generateComprehensiveInsight("u1", { locale: "de" });
|
||||
|
||||
expect(outcome).toEqual({
|
||||
status: "failed",
|
||||
reason: "all-providers-failed",
|
||||
provider: "local",
|
||||
sentNothing: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -134,6 +134,7 @@ describe("generateStatusBatchForUser — the statusText capability comes first",
|
||||
served: 0,
|
||||
batched: 0,
|
||||
fellBack: 0,
|
||||
fellBackAnswered: 0,
|
||||
batchCallMade: false,
|
||||
});
|
||||
});
|
||||
@@ -340,6 +341,15 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => {
|
||||
vi.mocked(runStatusCompletion).mockResolvedValue({
|
||||
kind: "error",
|
||||
} as never);
|
||||
// #1384 — one card's own call came back with a model note; the other six
|
||||
// served their fallback. A nightly warm over a provider that only failed
|
||||
// reported `assessments_warmed: 6`, because every fallback counted.
|
||||
vi.mocked(runPreparedStatusCard).mockResolvedValueOnce({
|
||||
hasProvider: true,
|
||||
text: "note",
|
||||
cached: false,
|
||||
updatedAt: "t",
|
||||
});
|
||||
|
||||
const result = await generateStatusBatchForUser("u1", { locale: "en" });
|
||||
|
||||
@@ -347,6 +357,7 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => {
|
||||
for (const f of finalizers) expect(f).not.toHaveBeenCalled();
|
||||
expect(runPreparedStatusCard).toHaveBeenCalledTimes(7);
|
||||
expect(result.fellBack).toBe(7);
|
||||
expect(result.fellBackAnswered).toBe(1);
|
||||
expect(result.batched).toBe(0);
|
||||
});
|
||||
|
||||
@@ -411,6 +422,7 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => {
|
||||
for (const f of finalizers) expect(f).not.toHaveBeenCalled();
|
||||
expect(runPreparedStatusCard).not.toHaveBeenCalled();
|
||||
expect(result.fellBack).toBe(7);
|
||||
expect(result.fellBackAnswered).toBe(0);
|
||||
expect(result.batchCallMade).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,37 @@
|
||||
/**
|
||||
* What a failed briefing generation knows about its provider chain beyond
|
||||
* the reason (#1384): which configured provider failed first, whether that
|
||||
* provider refused the credential, and whether anything was sent at all.
|
||||
*
|
||||
* The nightly pass needs all three to settle an account honestly. A refusal
|
||||
* of the account's own credential is the account's matter, a refusal of the
|
||||
* operator's shared key is the operator's, and a chain whose every entry the
|
||||
* provider health ledger benched for background work (#1279) sent no request,
|
||||
* so the account was not attempted. Without them every such account read as
|
||||
* a failure, night after night.
|
||||
*/
|
||||
import { AllProvidersFailedError } from "@/lib/ai/provider-runner";
|
||||
|
||||
export interface ChainFailureFacts {
|
||||
/**
|
||||
* The chain's first provider refused the credential (a dead Codex
|
||||
* sign-in, a revoked key). No retry can succeed until someone reconnects
|
||||
* or replaces it.
|
||||
*/
|
||||
credentialExpired?: boolean;
|
||||
/** The chain entry that failed first, e.g. `openai-compatible`. */
|
||||
provider?: string;
|
||||
/** Every entry was benched for background work; no request left. */
|
||||
sentNothing?: boolean;
|
||||
}
|
||||
|
||||
/** The facts of a thrown chain failure; empty for any other error. */
|
||||
export function chainFailureFacts(error: unknown): ChainFailureFacts {
|
||||
if (!(error instanceof AllProvidersFailedError)) return {};
|
||||
const provider = error.attempts?.[0]?.providerType;
|
||||
return {
|
||||
...(error.primaryCredentialExpired ? { credentialExpired: true } : {}),
|
||||
...(provider ? { provider } : {}),
|
||||
...(error.sentNothing ? { sentNothing: true } : {}),
|
||||
};
|
||||
}
|
||||
@@ -93,6 +93,10 @@ import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic";
|
||||
import { insightResultSchema, type InsightResult } from "@/lib/ai/types";
|
||||
import { resolveProviderChain } from "@/lib/ai/provider";
|
||||
import { AllProvidersFailedError } from "@/lib/ai/provider-runner";
|
||||
import {
|
||||
chainFailureFacts,
|
||||
type ChainFailureFacts,
|
||||
} from "@/lib/insights/chain-failure-facts";
|
||||
import {
|
||||
BriefingBudgetExceededError,
|
||||
runBriefingCompletion,
|
||||
@@ -178,17 +182,7 @@ export type GenerateOutcome =
|
||||
| "scope-changed"
|
||||
| "profile-scope-changed";
|
||||
}
|
||||
| {
|
||||
status: "failed";
|
||||
reason: string;
|
||||
/**
|
||||
* v1.42 — the chain's first provider refused the credential (a dead
|
||||
* Codex sign-in, a revoked key). The nightly pass reports the account
|
||||
* as `auth_failed` and does not queue a retry that cannot succeed until
|
||||
* the person reconnects.
|
||||
*/
|
||||
credentialExpired?: boolean;
|
||||
};
|
||||
| ({ status: "failed"; reason: string } & ChainFailureFacts);
|
||||
|
||||
/** Higher, seedless temperature for the daily-briefing phrasing re-roll. */
|
||||
const BRIEFING_REROLL_TEMPERATURE = 0.6;
|
||||
@@ -1290,11 +1284,7 @@ export async function generateComprehensiveInsight(
|
||||
? (e.attempts?.[0]?.failureKind ?? null)
|
||||
: providerFailureKind(e).failure,
|
||||
});
|
||||
const credentialExpired =
|
||||
e instanceof AllProvidersFailedError && e.primaryCredentialExpired;
|
||||
return credentialExpired
|
||||
? { status: "failed", reason, credentialExpired }
|
||||
: { status: "failed", reason };
|
||||
return { status: "failed", reason, ...chainFailureFacts(e) };
|
||||
}
|
||||
|
||||
// Anthropic + local have no native JSON mode, so a ```json-fenced or
|
||||
|
||||
@@ -37,6 +37,7 @@ import {
|
||||
runPreparedStatusCard,
|
||||
type PendingStatusCard,
|
||||
type PreparedStatusCard,
|
||||
type StatusCardResult,
|
||||
} from "@/lib/insights/status-card-generation";
|
||||
import { runStatusCompletion } from "@/lib/insights/status-provider";
|
||||
import {
|
||||
@@ -77,10 +78,27 @@ export interface StatusBatchResult {
|
||||
/** Metrics that fell back to a single-card call (omitted by the batch, or
|
||||
* the batch failed). */
|
||||
fellBack: number;
|
||||
/**
|
||||
* Of `fellBack`, the metrics whose own single-card call came back with a
|
||||
* model-written assessment. The rest served their deterministic fallback:
|
||||
* no provider, a failed or cooling-down provider, or a screened reply.
|
||||
* `batched + fellBackAnswered` is what a warm pass may call warmed (#1384);
|
||||
* `fellBack` alone counted a night of six failed calls as six assessments.
|
||||
*/
|
||||
fellBackAnswered: number;
|
||||
/** True when the one batched provider call was issued. */
|
||||
batchCallMade: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a single-card run ended in a model-written assessment: the persisted
|
||||
* shape is the only one with a provider and no cache flag. The no-key, the
|
||||
* timeout / error stub and the screened fallback all carry `cached: true`.
|
||||
*/
|
||||
function modelAnswered(result: StatusCardResult): boolean {
|
||||
return result.hasProvider === true && result.cached === false;
|
||||
}
|
||||
|
||||
/** Pull `perMetric` out of a (possibly fenced) model completion. */
|
||||
function parsePerMetric(content: string): Record<string, string> | null {
|
||||
const tryParse = (raw: string): Record<string, string> | null => {
|
||||
@@ -119,6 +137,7 @@ export async function generateStatusBatchForUser(
|
||||
served: 0,
|
||||
batched: 0,
|
||||
fellBack: 0,
|
||||
fellBackAnswered: 0,
|
||||
batchCallMade: false,
|
||||
};
|
||||
|
||||
@@ -171,11 +190,19 @@ export async function generateStatusBatchForUser(
|
||||
// One metric still needing the LLM is not worth a batch envelope — run it
|
||||
// through its own single-card path (identical persisted result).
|
||||
if (pending.length === 1) {
|
||||
await runPreparedStatusCard(pending[0]);
|
||||
if (modelAnswered(await runPreparedStatusCard(pending[0]))) {
|
||||
result.fellBackAnswered++;
|
||||
}
|
||||
result.fellBack++;
|
||||
annotate({
|
||||
action: { name: "insights.status.batch" },
|
||||
meta: { served: result.served, batched: 0, fell_back: 1, single: true },
|
||||
meta: {
|
||||
served: result.served,
|
||||
batched: 0,
|
||||
fell_back: 1,
|
||||
fell_back_answered: result.fellBackAnswered,
|
||||
single: true,
|
||||
},
|
||||
});
|
||||
return result;
|
||||
}
|
||||
@@ -272,14 +299,19 @@ export async function generateStatusBatchForUser(
|
||||
// Batch missed this metric (omitted key, or timeout/error/invalid-json):
|
||||
// fall back to the single-card path. On a batch timeout/error this writes
|
||||
// the per-card negative stub via the card's own `timeout`.
|
||||
await runPreparedStatusCard(card);
|
||||
return "fellBack" as const;
|
||||
return modelAnswered(await runPreparedStatusCard(card))
|
||||
? ("fellBackAnswered" as const)
|
||||
: ("fellBack" as const);
|
||||
});
|
||||
|
||||
const outcomes = await Promise.all(fanOuts);
|
||||
for (const o of outcomes) {
|
||||
if (o === "batched") result.batched++;
|
||||
else result.fellBack++;
|
||||
if (o === "batched") {
|
||||
result.batched++;
|
||||
continue;
|
||||
}
|
||||
result.fellBack++;
|
||||
if (o === "fellBackAnswered") result.fellBackAnswered++;
|
||||
}
|
||||
|
||||
annotate({
|
||||
@@ -288,6 +320,7 @@ export async function generateStatusBatchForUser(
|
||||
served: result.served,
|
||||
batched: result.batched,
|
||||
fell_back: result.fellBack,
|
||||
fell_back_answered: result.fellBackAnswered,
|
||||
pending: pending.length,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -103,6 +103,7 @@ vi.mock("@/lib/insights/status-batch", () => ({
|
||||
served: 0,
|
||||
batched: 0,
|
||||
fellBack: 0,
|
||||
fellBackAnswered: 0,
|
||||
batchCallMade: false,
|
||||
})),
|
||||
}));
|
||||
@@ -132,6 +133,7 @@ import {
|
||||
INSIGHT_PREGENERATE_EXPIRE_SECONDS,
|
||||
} from "../insight-pregenerate-shared";
|
||||
import { handleInsightPregenerateJob } from "../reminder/insights-handlers";
|
||||
import { judgeJobRun, type reportJobRun } from "../job-run-report";
|
||||
import { insightRetryOptions } from "../reminder/registrar-shared";
|
||||
import {
|
||||
groupOf,
|
||||
@@ -1556,15 +1558,9 @@ describe("runInsightPregenerate — warm budget honours the response timeout (v1
|
||||
// withheld (screened) briefing and a dead sign-in queue no provider retry,
|
||||
// and a time-budget cut-off reports the unreached accounts as deferred.
|
||||
describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
|
||||
type Reported = {
|
||||
queue: string;
|
||||
runId: string;
|
||||
candidates: Array<{ key: string; outcome: string; cause?: string }>;
|
||||
};
|
||||
|
||||
function harness(users: Array<{ id: string; locale: string }>) {
|
||||
const { prisma } = makePrisma(users);
|
||||
const reportRun = vi.fn(async (input: Reported) => ({
|
||||
const reportRun = vi.fn<typeof reportJobRun>(async (input) => ({
|
||||
total: input.candidates.length,
|
||||
attempted: 0,
|
||||
failed: 0,
|
||||
@@ -1606,7 +1602,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
|
||||
statusGenerators: h.statusGenerators,
|
||||
warmGenericMetrics: h.warmGenericMetrics,
|
||||
enqueueRetry: h.enqueueRetry,
|
||||
reportRun: h.reportRun as never,
|
||||
reportRun: h.reportRun,
|
||||
});
|
||||
|
||||
expect(h.reportRun).toHaveBeenCalledTimes(1);
|
||||
@@ -1639,7 +1635,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
|
||||
statusGenerators: h.statusGenerators,
|
||||
warmGenericMetrics: h.warmGenericMetrics,
|
||||
enqueueRetry: h.enqueueRetry,
|
||||
reportRun: h.reportRun as never,
|
||||
reportRun: h.reportRun,
|
||||
});
|
||||
|
||||
expect(h.enqueueRetry).toHaveBeenCalledTimes(1);
|
||||
@@ -1668,7 +1664,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
|
||||
statusGenerators: h.statusGenerators,
|
||||
warmGenericMetrics: h.warmGenericMetrics,
|
||||
enqueueRetry: h.enqueueRetry,
|
||||
reportRun: h.reportRun as never,
|
||||
reportRun: h.reportRun,
|
||||
shouldStop: () => ++calls > 1,
|
||||
});
|
||||
|
||||
@@ -1677,3 +1673,117 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
|
||||
).toEqual(["ok", "deferred", "deferred"]);
|
||||
});
|
||||
});
|
||||
|
||||
// #1384 — a self-hoster whose only account kept a provider that no longer
|
||||
// works was told every night that the job was failing. The run used the real
|
||||
// verdict here, so what the alert reads (`all_failed`) is what is asserted.
|
||||
describe("runInsightPregenerate — a provider that cannot serve is not a failing run (#1384)", () => {
|
||||
async function runOne(outcome: Record<string, unknown>) {
|
||||
const { prisma } = makePrisma([{ id: "only-account", locale: "en" }]);
|
||||
const reportRun = vi.fn<typeof reportJobRun>(async (input) =>
|
||||
judgeJobRun(input.candidates),
|
||||
);
|
||||
const enqueueRetry = vi.fn().mockResolvedValue(undefined);
|
||||
const result = await runInsightPregenerate(prisma as never, {
|
||||
generate: vi.fn().mockResolvedValue(outcome),
|
||||
statusGenerators: Array.from({ length: 7 }, () =>
|
||||
vi.fn().mockResolvedValue({ hasProvider: true, cached: true }),
|
||||
),
|
||||
warmGenericMetrics: vi.fn().mockResolvedValue(0),
|
||||
enqueueRetry,
|
||||
reportRun,
|
||||
});
|
||||
return {
|
||||
result,
|
||||
enqueueRetry,
|
||||
candidate: reportRun.mock.calls[0][0].candidates[0],
|
||||
};
|
||||
}
|
||||
|
||||
it("skips an account whose every provider was benched, so nothing was sent", async () => {
|
||||
const { result, enqueueRetry, candidate } = await runOne({
|
||||
status: "failed",
|
||||
reason: "all-providers-failed",
|
||||
provider: "local",
|
||||
sentNothing: true,
|
||||
});
|
||||
|
||||
expect(candidate).toEqual(
|
||||
expect.objectContaining({
|
||||
outcome: "skipped",
|
||||
cause: "provider_benched",
|
||||
}),
|
||||
);
|
||||
expect(result.skipped).toBe(1);
|
||||
expect(result.failed).toBe(0);
|
||||
expect(result.allFailed).toBe(false);
|
||||
expect(result.firstFailure).toBeNull();
|
||||
// The retry would find the same bench.
|
||||
expect(enqueueRetry).not.toHaveBeenCalled();
|
||||
expect(annotateSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
action: { name: "insights.pregenerate.provider_benched" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("reports a refused credential of the account's own as auth_failed, naming the provider, without failing the run", async () => {
|
||||
const { result, enqueueRetry, candidate } = await runOne({
|
||||
status: "failed",
|
||||
reason: "all-providers-failed",
|
||||
provider: "openai-compatible",
|
||||
credentialExpired: true,
|
||||
});
|
||||
|
||||
expect(candidate).toEqual(
|
||||
expect.objectContaining({
|
||||
outcome: "auth_failed",
|
||||
cause: "all-providers-failed:openai-compatible",
|
||||
}),
|
||||
);
|
||||
expect(result.allFailed).toBe(false);
|
||||
expect(enqueueRetry).not.toHaveBeenCalled();
|
||||
expect(annotateSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
action: { name: "insights.pregenerate.comprehensive_failed" },
|
||||
meta: expect.objectContaining({
|
||||
provider: "openai-compatible",
|
||||
credential_refused: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("still fails the run when the operator's own key was refused, and says which", async () => {
|
||||
const { result, enqueueRetry, candidate } = await runOne({
|
||||
status: "failed",
|
||||
reason: "all-providers-failed",
|
||||
provider: "admin-openai",
|
||||
credentialExpired: true,
|
||||
});
|
||||
|
||||
expect(candidate.outcome).toBe("failed");
|
||||
expect(result.allFailed).toBe(true);
|
||||
expect(result.firstFailure).toEqual({
|
||||
reason: "all-providers-failed",
|
||||
provider: "admin-openai",
|
||||
});
|
||||
// A refused key does not heal in 45 minutes.
|
||||
expect(enqueueRetry).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("carries the cause and provider of an ordinary failure into the run's facts", async () => {
|
||||
const { result, enqueueRetry } = await runOne({
|
||||
status: "failed",
|
||||
reason: "provider-error",
|
||||
provider: "local",
|
||||
});
|
||||
|
||||
expect(result.allFailed).toBe(true);
|
||||
expect(result.firstFailure).toEqual({
|
||||
reason: "provider-error",
|
||||
provider: "local",
|
||||
});
|
||||
expect(enqueueRetry).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -193,3 +193,36 @@ describe("bounded generic JobOutcome serialization", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// #1384 — a generator reason and a provider type become facts the admin page
|
||||
// and the repeated-failure alert can show.
|
||||
describe("codeFact", () => {
|
||||
it("turns our own hyphenated codes into stable fact codes", async () => {
|
||||
const { codeFact, serializeJobOutcome } = await vi.importActual<
|
||||
typeof import("@/lib/jobs/job-outcome")
|
||||
>("@/lib/jobs/job-outcome");
|
||||
expect(codeFact("all-providers-failed")).toBe("all_providers_failed");
|
||||
expect(codeFact("openai-compatible")).toBe("openai_compatible");
|
||||
expect(codeFact("generator:invalid-json")).toBe("generator_invalid_json");
|
||||
expect(codeFact("timeout")).toBe("timeout");
|
||||
// Nothing usable: the caller leaves the fact out.
|
||||
expect(codeFact("")).toBeNull();
|
||||
expect(codeFact("---")).toBeNull();
|
||||
expect(codeFact(null)).toBeNull();
|
||||
// What it produces is a fact the serializer accepts.
|
||||
expect(
|
||||
serializeJobOutcome(
|
||||
jobDone({
|
||||
failure_cause: codeFact("all-providers-failed")!,
|
||||
failure_provider: codeFact("admin-openai")!,
|
||||
}),
|
||||
),
|
||||
).toEqual({
|
||||
ok: true,
|
||||
did: {
|
||||
failure_cause: "all_providers_failed",
|
||||
failure_provider: "admin_openai",
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -47,13 +47,27 @@ describe("judgeJobRun", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("counts a dead credential as a failure", () => {
|
||||
expect(
|
||||
judgeJobRun([
|
||||
{ key: "a", outcome: "auth_failed" },
|
||||
{ key: "b", outcome: "failed" },
|
||||
]).allFailed,
|
||||
).toBe(true);
|
||||
// #1384 — a key the issuer revoked stays saved and is refused every night.
|
||||
// Counted as a failure, it made a single-account instance's nightly pass
|
||||
// read as all-failed forever and alerted the operator about the job.
|
||||
it("does not fail a run over a refused credential of the account's own", () => {
|
||||
expect(judgeJobRun([{ key: "a", outcome: "auth_failed" }])).toEqual({
|
||||
total: 1,
|
||||
attempted: 1,
|
||||
failed: 0,
|
||||
allFailed: false,
|
||||
partial: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("reads a refused credential beside a real failure as partial", () => {
|
||||
const verdict = judgeJobRun([
|
||||
{ key: "a", outcome: "auth_failed" },
|
||||
{ key: "b", outcome: "failed" },
|
||||
]);
|
||||
expect(verdict.allFailed).toBe(false);
|
||||
expect(verdict.partial).toBe(true);
|
||||
expect(verdict.failed).toBe(1);
|
||||
});
|
||||
|
||||
it("reads one success among failures as partial, not failed", () => {
|
||||
@@ -84,7 +98,7 @@ describe("reportJobRun", () => {
|
||||
queue: "insight-pregenerate",
|
||||
runId: "r1",
|
||||
candidates: [
|
||||
{ key: "a", outcome: "auth_failed", cause: "all-providers-failed" },
|
||||
{ key: "a", outcome: "failed", cause: "all-providers-failed" },
|
||||
{ key: "b", outcome: "skipped", cause: "no-consent" },
|
||||
],
|
||||
});
|
||||
@@ -101,7 +115,31 @@ describe("reportJobRun", () => {
|
||||
consecutiveFailures: 3,
|
||||
failed: 1,
|
||||
total: 1,
|
||||
causes: ["auth_failed:all-providers-failed"],
|
||||
causes: ["failed:all-providers-failed"],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("emits job.run.partial, never job.run.failed, for a run whose only attempt was a refused credential (#1384)", async () => {
|
||||
const verdict = await reportJobRun({
|
||||
queue: "insight-pregenerate",
|
||||
runId: "r1",
|
||||
candidates: [
|
||||
{
|
||||
key: "a",
|
||||
outcome: "auth_failed",
|
||||
cause: "all-providers-failed:openai-compatible",
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(verdict.allFailed).toBe(false);
|
||||
expect(readConsecutiveRunFailures).not.toHaveBeenCalled();
|
||||
expect(emitSignal).toHaveBeenCalledTimes(1);
|
||||
expect(emitSignal.mock.calls[0][0]).toMatchObject({
|
||||
action: "job.run.partial",
|
||||
level: "warn",
|
||||
meta: {
|
||||
causes: ["auth_failed:all-providers-failed:openai-compatible"],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
@@ -87,6 +87,34 @@ describe("judgeQueueRuns", () => {
|
||||
).toBe("failing");
|
||||
});
|
||||
|
||||
// #1384 — the alert quoted the completed run's own outcome JSON, and the
|
||||
// operator it sent to the admin page found nothing there.
|
||||
it("says in words what an all-failed night was, with its cause and provider", () => {
|
||||
const output = JSON.stringify({
|
||||
ok: true,
|
||||
did: {
|
||||
total: 1,
|
||||
failed: 1,
|
||||
all_failed: true,
|
||||
assessments_warmed: 0,
|
||||
failure_cause: "all_providers_failed",
|
||||
failure_provider: "admin_openai",
|
||||
},
|
||||
});
|
||||
const verdict = judgeQueueRuns(
|
||||
[0, 1, 2].map((ago) =>
|
||||
run(ago, { state: "completed", allFailed: true, error: output }),
|
||||
),
|
||||
);
|
||||
expect(verdict.state).toBe("failing");
|
||||
expect(verdict.lastError).toBe(
|
||||
"The run completed, but every account it attempted failed (1 of 1). " +
|
||||
"First cause: all_providers_failed from provider admin_openai. " +
|
||||
"Accounts without a usable provider are skipped and not counted; " +
|
||||
"job.run.failed in the app log lists each account's outcome.",
|
||||
);
|
||||
});
|
||||
|
||||
it("explains a bare timeout the way the admin card does", () => {
|
||||
const verdict = judgeQueueRuns(
|
||||
[0, 1, 2].map((ago) =>
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* How the nightly briefing pass settles an account whose generation came back
|
||||
* `failed` (#1384).
|
||||
*
|
||||
* A failure used to be one of three things: screened, a dead sign-in, or a
|
||||
* provider failure worth a retry in 45 minutes. Two cases were missing, and
|
||||
* both made a self-hoster's only account read as a failing run every night,
|
||||
* which the repeated-failure alert then reported as a broken job:
|
||||
*
|
||||
* - Every provider of the chain was benched for background work by the
|
||||
* provider health ledger (#1279), so no request was sent. The account was
|
||||
* not attempted tonight; its earlier failures were counted by the runs
|
||||
* that made them, and a retry would find the same bench. It is `skipped`.
|
||||
* - The provider refused the credential. A refused credential does not heal
|
||||
* in 45 minutes, so no retry is queued, and whose credential it was
|
||||
* decides who has to act. The operator's shared key or central sign-in is
|
||||
* the operator's to fix, and a run where it was refused for everyone is a
|
||||
* failing run: `failed`. The account's own key, gateway or sign-in is its
|
||||
* owner's to fix: `auth_failed`, which the run reports but does not fail
|
||||
* over (`job-run-report.ts`). An API key revoked at its issuer stays saved
|
||||
* and is refused again every night, unlike a ChatGPT sign-in, which the
|
||||
* refusal parks.
|
||||
*
|
||||
* The model answered and the safety screen withheld it (`outbound-screened`)
|
||||
* stays `screened` with no retry: asking the same model again in 45 minutes
|
||||
* is the same question, and it was screened eight times a night.
|
||||
*/
|
||||
import { isOperatorHeldProvider } from "@/lib/ai/provider-egress";
|
||||
import type { GenerateOutcome } from "@/lib/insights/comprehensive-generate";
|
||||
import type { JobRunCandidate, JobRunOutcome } from "@/lib/jobs/job-run-report";
|
||||
|
||||
type FailedGeneration = Extract<GenerateOutcome, { status: "failed" }>;
|
||||
|
||||
export interface FailedGenerationVerdict {
|
||||
outcome: JobRunOutcome;
|
||||
/** The run line's cause: the reason, and the provider that failed first. */
|
||||
cause: string;
|
||||
/** False when nothing was sent: the account is tallied as skipped. */
|
||||
attempted: boolean;
|
||||
/** Queue the bounded 45-minute retry for this account. */
|
||||
retry: boolean;
|
||||
}
|
||||
|
||||
export function settleFailedGeneration(
|
||||
generated: FailedGeneration,
|
||||
): FailedGenerationVerdict {
|
||||
if (generated.sentNothing) {
|
||||
return {
|
||||
outcome: "skipped",
|
||||
cause: "provider_benched",
|
||||
attempted: false,
|
||||
retry: false,
|
||||
};
|
||||
}
|
||||
const cause = generated.provider
|
||||
? `${generated.reason}:${generated.provider}`
|
||||
: generated.reason;
|
||||
if (generated.reason === "outbound-screened") {
|
||||
return { outcome: "screened", cause, attempted: true, retry: false };
|
||||
}
|
||||
if (generated.credentialExpired) {
|
||||
const operatorHeld =
|
||||
generated.provider !== undefined &&
|
||||
isOperatorHeldProvider(generated.provider);
|
||||
return {
|
||||
outcome: operatorHeld ? "failed" : "auth_failed",
|
||||
cause,
|
||||
attempted: true,
|
||||
retry: false,
|
||||
};
|
||||
}
|
||||
return { outcome: "failed", cause, attempted: true, retry: true };
|
||||
}
|
||||
|
||||
/** The first failed account of a run: its reason and the provider, if known. */
|
||||
export interface FirstFailure {
|
||||
reason: string;
|
||||
provider: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The first failed account's reason and provider, read back off the run's
|
||||
* causes, for the job's facts: what the admin page and the repeated-failure
|
||||
* alert show for a run that completed with every account failed.
|
||||
*/
|
||||
export function firstFailure(
|
||||
candidates: readonly JobRunCandidate[],
|
||||
): FirstFailure | null {
|
||||
const cause = candidates.find((c) => c.outcome === "failed")?.cause;
|
||||
if (!cause) return null;
|
||||
const [reason, provider] = cause.split(":");
|
||||
return { reason, provider: provider ?? null };
|
||||
}
|
||||
@@ -81,6 +81,10 @@ import {
|
||||
} from "@/lib/insights/metric-status-registry";
|
||||
import { withTimeout } from "@/lib/insights/with-timeout";
|
||||
import { comprehensiveWarmBudgetMs } from "@/lib/jobs/model-time";
|
||||
import {
|
||||
firstFailure,
|
||||
settleFailedGeneration,
|
||||
} from "@/lib/jobs/insight-pregenerate-failure";
|
||||
import {
|
||||
INSIGHT_PREGENERATE_QUEUE,
|
||||
enqueueForceWarm,
|
||||
@@ -235,6 +239,8 @@ export interface PregenerateRunResult {
|
||||
* seven specialised cards.
|
||||
*/
|
||||
metricAssessmentsWarmed: number;
|
||||
/** #1384 — the first failed account's reason and provider, as facts. */
|
||||
firstFailure: ReturnType<typeof firstFailure>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -302,9 +308,9 @@ async function warmPerStatusCaches(
|
||||
* once and issues ONE provider call for the metrics still needing the LLM,
|
||||
* fanning the response back into the same per-metric cache rows the
|
||||
* standalone generators wrote. Replaces the prior seven independent warm
|
||||
* calls. Returns the count of metrics that got a fresh (batched or
|
||||
* fallback) assessment so the wide-event tally is comparable to the old
|
||||
* per-card count.
|
||||
* calls. Returns the count of metrics that got a fresh model-written
|
||||
* assessment, batched or through a card's own fallback call, so the
|
||||
* wide-event tally is comparable to the old per-card count.
|
||||
*
|
||||
* Refill-only (`force: false`): a card already generated today and a card
|
||||
* whose content-hash is unchanged both resolve inside the batch as `served`
|
||||
@@ -322,7 +328,7 @@ async function warmStatusBatch(
|
||||
locale,
|
||||
force: false,
|
||||
});
|
||||
warmed += result.batched + result.fellBack;
|
||||
warmed += result.batched + result.fellBackAnswered;
|
||||
} catch (err) {
|
||||
// The batch must never throw the cron's user loop off course — but
|
||||
// a swallowed batch failure left ~7 cards cold with zero signal.
|
||||
@@ -555,6 +561,7 @@ export async function runInsightPregenerate(
|
||||
allFailed: false,
|
||||
assessmentsWarmed: 0,
|
||||
metricAssessmentsWarmed: 0,
|
||||
firstFailure: null,
|
||||
};
|
||||
|
||||
// Master kill-switch — when the operator disabled the briefing
|
||||
@@ -619,6 +626,7 @@ export async function runInsightPregenerate(
|
||||
candidates: runOutcomes,
|
||||
});
|
||||
result.allFailed = verdict.allFailed;
|
||||
result.firstFailure = firstFailure(runOutcomes);
|
||||
|
||||
return result;
|
||||
|
||||
@@ -783,42 +791,34 @@ export async function runInsightPregenerate(
|
||||
result.skipped++;
|
||||
settle("skipped", generated.reason);
|
||||
break;
|
||||
case "failed":
|
||||
result.failed++;
|
||||
// v1.28.30 — THE previously-silent failure path. A generator
|
||||
// that resolves with `{ status: "failed" }` (features error,
|
||||
// oversize payload, invalid JSON after retry, abort) bumped
|
||||
// the tally with no annotation at all — the nightly summary
|
||||
// showed `failed: 1` and nothing else was greppable. Reuse
|
||||
// the `comprehensive_failed` shape with the generator's own
|
||||
// reason as the cause.
|
||||
case "failed": {
|
||||
// Benched, screened, a refused credential and whose, or a
|
||||
// retryable failure: `insight-pregenerate-failure.ts` (#1384).
|
||||
const verdict = settleFailedGeneration(generated);
|
||||
if (verdict.attempted) result.failed++;
|
||||
else result.skipped++;
|
||||
annotate({
|
||||
action: { name: "insights.pregenerate.comprehensive_failed" },
|
||||
action: {
|
||||
name: verdict.attempted
|
||||
? "insights.pregenerate.comprehensive_failed"
|
||||
: "insights.pregenerate.provider_benched",
|
||||
},
|
||||
meta: {
|
||||
locale,
|
||||
stage: "nightly.generator",
|
||||
cause: `generator:${generated.reason}`,
|
||||
provider: generated.provider ?? null,
|
||||
credential_refused: generated.credentialExpired === true,
|
||||
budget_ms: warmBudgetMs,
|
||||
message: null,
|
||||
},
|
||||
});
|
||||
if (generated.reason === "outbound-screened") {
|
||||
// v1.42 — the model answered and the safety screen withheld
|
||||
// it, after the generator's own repair. Asking the same model
|
||||
// again in 45 minutes is not a provider retry; it is the same
|
||||
// question, and it was screened eight times a night.
|
||||
settle("screened", generated.reason);
|
||||
} else if (generated.credentialExpired) {
|
||||
// v1.42 — a dead sign-in (a Codex refresh token the issuer
|
||||
// revoked) does not heal in 45 minutes either; the person has
|
||||
// to reconnect. The refusal parked the connection as expired,
|
||||
// which Settings shows, and the next run no longer counts it.
|
||||
settle("auth_failed", generated.reason);
|
||||
} else {
|
||||
settle("failed", generated.reason);
|
||||
settle(verdict.outcome, verdict.cause);
|
||||
if (verdict.retry) {
|
||||
await enqueueRetry({ userId: candidate.id, locale });
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,7 +92,7 @@ const MAX_ERROR_CHARS = 300;
|
||||
export interface FailingQueue {
|
||||
/** The pg-boss queue name, e.g. `environment-fetch`. */
|
||||
queue: string;
|
||||
/** Jobs that exhausted their retries inside the window. */
|
||||
/** Failed runs inside the window: exhausted retries, or all-failed passes. */
|
||||
failures: number;
|
||||
/** ISO instant of the newest failure. */
|
||||
lastFailedAt: string;
|
||||
@@ -140,10 +140,48 @@ function explainTimeout(message: string, expireSeconds: number | null): string {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Say what a completed run that failed every account means (#1384). Such a
|
||||
* run's `output` is its outcome, `{"ok":true,"did":{…,"all_failed":true}}`,
|
||||
* and quoting that JSON told the operator nothing: not that the job ran, not
|
||||
* which provider failed. Null when `raw` is not such an outcome.
|
||||
*/
|
||||
function explainAllFailedRun(raw: string): string | null {
|
||||
if (!raw.startsWith("{")) return null;
|
||||
let did: Record<string, unknown>;
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as { ok?: unknown; did?: unknown };
|
||||
if (parsed?.ok !== true || typeof parsed.did !== "object") return null;
|
||||
did = (parsed.did ?? {}) as Record<string, unknown>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (did.all_failed !== true) return null;
|
||||
// Every attempted account failed, so the count is both sides of "of".
|
||||
const failed =
|
||||
typeof did.failed === "number" && did.failed > 0
|
||||
? ` (${did.failed} of ${did.failed})`
|
||||
: "";
|
||||
// "none" is how the run says it has no cause or provider to name.
|
||||
const named = (value: unknown): string | null =>
|
||||
typeof value === "string" && value !== "none" ? value : null;
|
||||
const cause = named(did.failure_cause);
|
||||
const provider = named(did.failure_provider);
|
||||
const why = cause
|
||||
? ` First cause: ${cause}${provider ? ` from provider ${provider}` : ""}.`
|
||||
: "";
|
||||
return (
|
||||
`The run completed, but every account it attempted failed${failed}.${why} ` +
|
||||
`Accounts without a usable provider are skipped and not counted; ` +
|
||||
`job.run.failed in the app log lists each account's outcome.`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn whatever pg-boss stored in `output` into one line. A thrown `Error`
|
||||
* serialises to `{ message, stack, … }`; a thrown non-Error can be any JSON at
|
||||
* all, so the raw text is the fallback rather than an invented placeholder.
|
||||
* A completed run that reported every account failed is said in words.
|
||||
*/
|
||||
export function presentError(
|
||||
raw: string | null,
|
||||
@@ -152,6 +190,8 @@ export function presentError(
|
||||
if (raw === null) return "";
|
||||
const trimmed = raw.trim();
|
||||
if (trimmed.length === 0) return "";
|
||||
const allFailed = explainAllFailedRun(trimmed);
|
||||
if (allFailed !== null) return allFailed;
|
||||
if (TIMEOUT_MESSAGE.test(trimmed))
|
||||
return explainTimeout(trimmed, expireSeconds);
|
||||
if (HEARTBEAT_MESSAGE.test(trimmed)) {
|
||||
@@ -168,8 +208,13 @@ export function presentError(
|
||||
}
|
||||
|
||||
/**
|
||||
* Every queue with at least one exhausted-retry failure in the window, newest
|
||||
* first. `null` when the queue schema is not present at all.
|
||||
* Every queue with at least one failed run in the window, newest first:
|
||||
* a job that exhausted its retries, or a fan-out pass that completed but
|
||||
* reported that every account it attempted failed (`did.all_failed`). The
|
||||
* second kind is what the repeated-failure alert counts too
|
||||
* (`readConsecutiveRunFailures`, `system-job-alert.ts`); the page used to
|
||||
* list only the first, so an operator the alert sent here read "no job has
|
||||
* failed" (#1384). `null` when the queue schema is not present at all.
|
||||
*/
|
||||
export async function readFailingQueues(
|
||||
withinHours: number = JOB_FAILURE_WINDOW_HOURS,
|
||||
@@ -194,7 +239,10 @@ export async function readFailingQueues(
|
||||
array_agg(expire_seconds ORDER BY completed_on DESC NULLS LAST)
|
||||
)[1] AS last_expire_seconds
|
||||
FROM pgboss.job
|
||||
WHERE state = 'failed'
|
||||
WHERE (
|
||||
state = 'failed'
|
||||
OR (state = 'completed' AND output->'did'->>'all_failed' = 'true')
|
||||
)
|
||||
AND completed_on > now() - make_interval(hours => ${withinHours}::int)
|
||||
GROUP BY name
|
||||
ORDER BY max(completed_on) DESC
|
||||
|
||||
@@ -130,6 +130,11 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
|
||||
"duration_ms",
|
||||
"errored",
|
||||
"failed",
|
||||
// #1384 — a fan-out pass's first failed candidate: its cause and the
|
||||
// provider that failed for it, as stable codes. What the admin page and
|
||||
// the repeated-failure alert show for a run that completed all-failed.
|
||||
"failure_cause",
|
||||
"failure_provider",
|
||||
"failures",
|
||||
"feedback_buckets",
|
||||
"feedback_total_rows",
|
||||
@@ -423,6 +428,22 @@ export function percentFact(ratio: number | null | undefined): number {
|
||||
return wholeFact(ratio * 100);
|
||||
}
|
||||
|
||||
/**
|
||||
* A short code from our own vocabulary (a generator reason, a provider type)
|
||||
* as a fact: `all-providers-failed` becomes `all_providers_failed`. Null when
|
||||
* nothing of it survives, so the caller leaves the fact out.
|
||||
*/
|
||||
export function codeFact(value: string | null | undefined): string | null {
|
||||
if (value == null) return null;
|
||||
const code = value
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "_")
|
||||
.replace(/^[^a-z]+/, "")
|
||||
.slice(0, 64)
|
||||
.replace(/_+$/, "");
|
||||
return STABLE_CODE_PATTERN.test(code) ? code : null;
|
||||
}
|
||||
|
||||
/** Why one fact is refused, or null when it is a valid fact. */
|
||||
function invalidFact(key: string, value: unknown): string | null {
|
||||
if (!JOB_FACT_ALLOWLIST.has(key))
|
||||
|
||||
@@ -21,6 +21,18 @@
|
||||
* attempted but is not a provider failure, so a run of only screened
|
||||
* candidates is partial, not failed.
|
||||
*
|
||||
* `auth_failed` is attempted and is not a failure of the run either (#1384).
|
||||
* It means the provider refused the account's own credential: a key revoked
|
||||
* at its issuer, a gateway that now wants a key the account no longer
|
||||
* stores. Only that account's owner can change it, and the run itself worked.
|
||||
* It used to count as a failure on the assumption that a refusal is reported
|
||||
* once and then parked; that holds for a ChatGPT sign-in and for nothing else,
|
||||
* so an API key the issuer had revoked failed every night for as long as it
|
||||
* stayed saved. On an instance where that account was the only candidate the
|
||||
* run then read as all-failed, and the operator was told the job was broken
|
||||
* night after night. A refusal of the operator's own credential is reported by
|
||||
* the caller as `failed`, because there the operator is the one who can act.
|
||||
*
|
||||
* The verdict comes back to the caller, which writes `all_failed` into its
|
||||
* `jobDone` facts. That fact is what lets `readConsecutiveRunFailures` count a
|
||||
* completed-but-all-failed night as part of a streak.
|
||||
@@ -46,18 +58,26 @@ export interface JobRunVerdict {
|
||||
total: number;
|
||||
/** Candidates it actually attempted (not skipped, not deferred). */
|
||||
attempted: number;
|
||||
/** `failed` + `auth_failed`. */
|
||||
/** Candidates that failed (`failed`; a refused credential is not one). */
|
||||
failed: number;
|
||||
/** Every attempted candidate failed (and at least one was attempted). */
|
||||
allFailed: boolean;
|
||||
/** Something failed or was withheld, but not everything. */
|
||||
/**
|
||||
* Something failed, was withheld or had its credential refused, but not
|
||||
* everything failed.
|
||||
*/
|
||||
partial: boolean;
|
||||
}
|
||||
|
||||
/** Candidates listed on the run's line; the counts always cover all of them. */
|
||||
const MAX_LISTED_CANDIDATES = 50;
|
||||
|
||||
const FAILED: ReadonlySet<JobRunOutcome> = new Set(["failed", "auth_failed"]);
|
||||
const FAILED: ReadonlySet<JobRunOutcome> = new Set(["failed"]);
|
||||
/** Attempted, not a failure of the run, but worth a `partial` line. */
|
||||
const WITHHELD: ReadonlySet<JobRunOutcome> = new Set([
|
||||
"screened",
|
||||
"auth_failed",
|
||||
]);
|
||||
const NOT_ATTEMPTED: ReadonlySet<JobRunOutcome> = new Set([
|
||||
"skipped",
|
||||
"deferred",
|
||||
@@ -72,14 +92,14 @@ export function judgeJobRun(
|
||||
(c) => !NOT_ATTEMPTED.has(c.outcome),
|
||||
).length;
|
||||
const failed = candidates.filter((c) => FAILED.has(c.outcome)).length;
|
||||
const screened = candidates.filter((c) => c.outcome === "screened").length;
|
||||
const withheld = candidates.filter((c) => WITHHELD.has(c.outcome)).length;
|
||||
const allFailed = attempted > 0 && failed === attempted;
|
||||
return {
|
||||
total,
|
||||
attempted,
|
||||
failed,
|
||||
allFailed,
|
||||
partial: !allFailed && (failed > 0 || screened > 0),
|
||||
partial: !allFailed && (failed > 0 || withheld > 0),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ import {
|
||||
type InsightStatusGeneratePayload,
|
||||
runInsightStatusGenerate,
|
||||
} from "@/lib/jobs/insight-status-generate";
|
||||
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
|
||||
import { codeFact, jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
|
||||
import { jobBudget } from "@/lib/jobs/job-budget";
|
||||
import { withBackgroundEvent } from "@/lib/logging/background";
|
||||
import { generateStatusBatchForUser } from "@/lib/insights/status-batch";
|
||||
@@ -80,7 +80,7 @@ export async function runStatusBatchCron(
|
||||
locale: await resolveJobLocale(user.locale),
|
||||
force: false,
|
||||
});
|
||||
generated += result.batched + result.fellBack;
|
||||
generated += result.batched + result.fellBackAnswered;
|
||||
served += result.served;
|
||||
} catch (error) {
|
||||
failed++;
|
||||
@@ -203,6 +203,10 @@ export async function handleInsightPregenerateJob(
|
||||
all_failed: summary.allFailed,
|
||||
assessments_warmed: summary.assessmentsWarmed,
|
||||
metric_assessments_warmed: summary.metricAssessmentsWarmed,
|
||||
// #1384 — what failed, for the admin page and the alert ("none"
|
||||
// when nothing did, or the generator could not say).
|
||||
failure_cause: codeFact(summary.firstFailure?.reason) ?? "none",
|
||||
failure_provider: codeFact(summary.firstFailure?.provider) ?? "none",
|
||||
});
|
||||
} catch (err) {
|
||||
evt.addWarning(
|
||||
|
||||
@@ -460,6 +460,7 @@
|
||||
"batchCallMade": true,
|
||||
"batched": 6,
|
||||
"fellBack": 1,
|
||||
"fellBackAnswered": 0,
|
||||
"served": 0
|
||||
}
|
||||
},
|
||||
@@ -1719,6 +1720,7 @@
|
||||
"batchCallMade": true,
|
||||
"batched": 0,
|
||||
"fellBack": 7,
|
||||
"fellBackAnswered": 0,
|
||||
"served": 0
|
||||
}
|
||||
},
|
||||
@@ -1831,6 +1833,7 @@
|
||||
"batchCallMade": false,
|
||||
"batched": 0,
|
||||
"fellBack": 1,
|
||||
"fellBackAnswered": 0,
|
||||
"served": 6
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,6 +40,7 @@ vi.mock("@/lib/insights/status-batch", async () => {
|
||||
served: 0,
|
||||
batched: 0,
|
||||
fellBack: 0,
|
||||
fellBackAnswered: 0,
|
||||
batchCallMade: false,
|
||||
};
|
||||
if (!(await aiCapabilityForJob(userId, "statusText")).available) {
|
||||
|
||||
@@ -126,6 +126,41 @@ describe("readFailingQueues", () => {
|
||||
await expect(readFailingQueues()).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
// #1384 — the repeated-failure alert counts a nightly pass that completed
|
||||
// with every account failed, and sent the operator to this page, which
|
||||
// listed only pg-boss failures and said nothing had failed.
|
||||
it("lists a run that completed with every account failed, in words, and not a good run", async () => {
|
||||
const ALL_FAILED_QUEUE = "insight-pregenerate";
|
||||
await boss.createQueue(ALL_FAILED_QUEUE);
|
||||
async function completeOne(did: Record<string, unknown>) {
|
||||
const jobId = await boss.send(ALL_FAILED_QUEUE, {}, { retryLimit: 0 });
|
||||
const [job] = await boss.fetch(ALL_FAILED_QUEUE);
|
||||
if (!job || job.id !== jobId) throw new Error("nothing to fetch");
|
||||
await boss.complete(ALL_FAILED_QUEUE, job.id, { ok: true, did });
|
||||
}
|
||||
// A good night, and the night from the report: one account, one failure.
|
||||
await completeOne({ total: 1, generated: 1, failed: 0, all_failed: false });
|
||||
await completeOne({
|
||||
total: 1,
|
||||
generated: 0,
|
||||
failed: 1,
|
||||
all_failed: true,
|
||||
assessments_warmed: 0,
|
||||
failure_cause: "all_providers_failed",
|
||||
failure_provider: "openai_compatible",
|
||||
});
|
||||
|
||||
const failing = await readFailingQueues();
|
||||
const entry = failing?.find((e) => e.queue === ALL_FAILED_QUEUE);
|
||||
expect(entry?.failures).toBe(1);
|
||||
expect(entry?.lastError).toBe(
|
||||
"The run completed, but every account it attempted failed (1 of 1). " +
|
||||
"First cause: all_providers_failed from provider openai_compatible. " +
|
||||
"Accounts without a usable provider are skipped and not counted; " +
|
||||
"job.run.failed in the app log lists each account's outcome.",
|
||||
);
|
||||
});
|
||||
|
||||
it("ignores a failure older than the window", async () => {
|
||||
await failOneJob(QUEUE, { userId: "user-a" }, "old failure");
|
||||
await getPrismaClient().$executeRawUnsafe(
|
||||
|
||||
Reference in new issue
Block a user