mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 21:10:59 +00:00
Merge remote-tracking branch 'origin/release/v1.46.0' into fix/review-meds-docs
# Conflicts: # src/__tests__/baselines/source-function-lines.json # src/__tests__/baselines/version-stamp-comments.json # src/__tests__/sharing-surface-guard.test.ts
This commit is contained in:
commit
d6c2760415
134 files changed
+3631
-1173
No files matched your search
+55
-89
@@ -15429,45 +15429,6 @@ paths:
|
||||
"403": *a9
|
||||
"422": *a4
|
||||
"429": *a5
|
||||
/api/settings/preview-features:
|
||||
get:
|
||||
tags:
|
||||
- Meta
|
||||
summary: Get the preview-feature opt-ins
|
||||
description: Every flag, off unless the caller switched it on.
|
||||
responses:
|
||||
"200":
|
||||
description: The opt-ins.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/PreviewFeaturesEnvelope"
|
||||
"401": *a1
|
||||
"403": *a9
|
||||
"422": *a4
|
||||
"429": *a5
|
||||
patch:
|
||||
tags:
|
||||
- Meta
|
||||
summary: Switch preview features on or off
|
||||
description: Sets the flags the body names and keeps the rest. An unknown flag or an empty body is refused with 422.
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/PreviewFeaturesPatchBody"
|
||||
responses:
|
||||
"200":
|
||||
description: The opt-ins as stored.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/PreviewFeaturesSavedEnvelope"
|
||||
"401": *a1
|
||||
"403": *a9
|
||||
"422": *a4
|
||||
"429": *a5
|
||||
/api/settings/privacy-summary:
|
||||
get:
|
||||
tags:
|
||||
@@ -27216,6 +27177,61 @@ paths:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ErrorEnvelope"
|
||||
/api/settings/preview-features:
|
||||
get:
|
||||
security: []
|
||||
deprecated: true
|
||||
tags:
|
||||
- Retired
|
||||
summary: GET /api/settings/preview-features — removed in v1.46.0
|
||||
description: >-
|
||||
Removed in v1.46.0. Every request to this path answers 410 Gone, whatever the method and whatever credential it
|
||||
carries. There is no replacement; the capability is gone rather than moved.
|
||||
|
||||
|
||||
Stored per-account opt-ins to preview features, of which there was one, the topic overview pages. Those pages
|
||||
were dropped before they shipped, so the switch governed nothing; the route and its column are gone, and no
|
||||
preview is gated per account.
|
||||
|
||||
|
||||
Retiring a path is deliberate, so the answer says so rather than leaving a client to read a 404 as a broken
|
||||
deployment and retry forever. Treat 410 as terminal: drop the call, degrade the surface that depended on it, and
|
||||
do not offer a retry.
|
||||
responses:
|
||||
"410":
|
||||
description: The path was removed in v1.46.0 and will not answer again. `meta.errorCode` = `route.retired`,
|
||||
`meta.removedIn` carries the version, and `meta.replacedBy` carries the replacement path or null.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ErrorEnvelope"
|
||||
patch:
|
||||
security: []
|
||||
deprecated: true
|
||||
tags:
|
||||
- Retired
|
||||
summary: PATCH /api/settings/preview-features — removed in v1.46.0
|
||||
description: >-
|
||||
Removed in v1.46.0. Every request to this path answers 410 Gone, whatever the method and whatever credential it
|
||||
carries. There is no replacement; the capability is gone rather than moved.
|
||||
|
||||
|
||||
Stored per-account opt-ins to preview features, of which there was one, the topic overview pages. Those pages
|
||||
were dropped before they shipped, so the switch governed nothing; the route and its column are gone, and no
|
||||
preview is gated per account.
|
||||
|
||||
|
||||
Retiring a path is deliberate, so the answer says so rather than leaving a client to read a 404 as a broken
|
||||
deployment and retry forever. Treat 410 as terminal: drop the call, degrade the surface that depended on it, and
|
||||
do not offer a retry.
|
||||
responses:
|
||||
"410":
|
||||
description: The path was removed in v1.46.0 and will not answer again. `meta.errorCode` = `route.retired`,
|
||||
`meta.removedIn` carries the version, and `meta.replacedBy` carries the replacement path or null.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ErrorEnvelope"
|
||||
components:
|
||||
schemas:
|
||||
CreateCycleCustomSymptomRequest:
|
||||
@@ -33268,12 +33284,6 @@ components:
|
||||
- version
|
||||
- list
|
||||
additionalProperties: false
|
||||
PreviewFeaturesPatchBody:
|
||||
type: object
|
||||
properties:
|
||||
topics:
|
||||
type: boolean
|
||||
additionalProperties: false
|
||||
ConsentPostBody:
|
||||
type: object
|
||||
properties:
|
||||
@@ -73603,50 +73613,6 @@ components:
|
||||
- data
|
||||
- error
|
||||
additionalProperties: false
|
||||
PreviewFeaturesEnvelope:
|
||||
type: object
|
||||
properties:
|
||||
data:
|
||||
$ref: "#/components/schemas/PreviewFeatures"
|
||||
error:
|
||||
type: "null"
|
||||
meta:
|
||||
type: object
|
||||
properties:
|
||||
requestId:
|
||||
type: string
|
||||
additionalProperties: false
|
||||
required:
|
||||
- data
|
||||
- error
|
||||
additionalProperties: false
|
||||
PreviewFeatures:
|
||||
type: object
|
||||
properties:
|
||||
topics:
|
||||
type: boolean
|
||||
description: Topic overview pages above the single-metric Insights pages. Off by default.
|
||||
required:
|
||||
- topics
|
||||
additionalProperties: false
|
||||
description: Per-user opt-in to preview features, one boolean per flag, every flag off by default.
|
||||
PreviewFeaturesSavedEnvelope:
|
||||
type: object
|
||||
properties:
|
||||
data:
|
||||
$ref: "#/components/schemas/PreviewFeatures"
|
||||
error:
|
||||
type: "null"
|
||||
meta:
|
||||
type: object
|
||||
properties:
|
||||
requestId:
|
||||
type: string
|
||||
additionalProperties: false
|
||||
required:
|
||||
- data
|
||||
- error
|
||||
additionalProperties: false
|
||||
PrivacySummaryEnvelope:
|
||||
type: object
|
||||
properties:
|
||||
|
||||
@@ -251,6 +251,21 @@ Pass `--only=prod` / `--only=demo` to check a single target, or
|
||||
`--attempts=N --interval=ms` to widen the retry window while a slow
|
||||
pull settles.
|
||||
|
||||
The worker loads each job's handler module on that job's first run, so a
|
||||
module the image cannot load does not stop the worker from starting. Once
|
||||
it has bound its queues, the worker loads every handler a single time in
|
||||
the background and reports each one that fails, normally within a minute
|
||||
of start. Each failure is a `job.run.failed` event at `error` with
|
||||
`reason: "handler_load_failed"` and the queue's name, and stderr also gets
|
||||
`[pg-boss] [registry] preloaded N/M job handlers; failed to load: <queues>`.
|
||||
If either shows up after a deploy, the image is broken for those queues,
|
||||
and their jobs will fail when they run. Roll back (see Rollback above)
|
||||
instead of waiting for the retries to run out:
|
||||
|
||||
```sh
|
||||
docker compose logs --since 5m app 2>&1 | grep -E 'handler_load_failed|failed to load:'
|
||||
```
|
||||
|
||||
## Walk the synthetic journey
|
||||
|
||||
The version check proves the instance serves the right build. It does not
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
-- Drop the per-user preview-feature opt-ins.
|
||||
--
|
||||
-- `users.preview_features_json` (0440) stored one flag, `topics`, for the
|
||||
-- topic overview pages. Those pages were dropped before they shipped, so no
|
||||
-- surface ever read the flag; `/api/settings/preview-features` is retired
|
||||
-- with this release and answers 410.
|
||||
--
|
||||
-- What is lost is a switch that switched nothing. A backup written by a
|
||||
-- release that still had the column carries the field, and the restore
|
||||
-- accepts and ignores it.
|
||||
--
|
||||
-- Dropping a column takes an ACCESS EXCLUSIVE lock on `users` for a catalogue
|
||||
-- update only (no rewrite). The lock wait is bounded so a long-running reader
|
||||
-- cannot queue every sign-in behind this statement; the deploy retries a
|
||||
-- timed-out attempt.
|
||||
--
|
||||
-- Reversibility: re-adding the nullable column restores the shape; every
|
||||
-- account then reads as all flags off, which is what every account had.
|
||||
--
|
||||
-- @retry-on-lock-timeout
|
||||
|
||||
DO $migration$
|
||||
BEGIN
|
||||
PERFORM set_config('lock_timeout', '5s', true);
|
||||
ALTER TABLE "users" DROP COLUMN IF EXISTS "preview_features_json";
|
||||
END
|
||||
$migration$;
|
||||
@@ -387,11 +387,6 @@ model User {
|
||||
// copies a stored browser value up once when the server has none for that list.
|
||||
// Shape: { "version": 1, "labs": { "view": "list", "order": ["<id>"], "sortDir": "manual" }, ... }
|
||||
moduleListLayoutJson Json? @map("module_list_layout_json")
|
||||
/// Per-user opt-in to preview features, one boolean per flag, all off by
|
||||
/// default (`src/lib/settings/preview-features.ts`). Read and written
|
||||
/// through `/api/settings/preview-features` and carried by the backup.
|
||||
/// @pending-consumer(topics preview) the `topics` flag is stored, but no surface reads it and the Settings > System switch is unlisted until the topic overview pages ship.
|
||||
previewFeaturesJson Json? @map("preview_features_json")
|
||||
/// Optional activity goals the person set for themselves: workouts, active
|
||||
/// minutes and steps per week, steps per month. Null or an absent key = no
|
||||
/// goal. Parsed by `src/lib/validations/activity-goals.ts`, written through
|
||||
|
||||
@@ -56,12 +56,15 @@ function code(rel: string): string {
|
||||
return stripComments(readFileSync(join(SRC, rel), "utf8"));
|
||||
}
|
||||
|
||||
/** The wire clients: every `*-client.ts` that dials through `safeFetch`. */
|
||||
/**
|
||||
* The wire clients: every `*-client.ts` that dials through `safeFetch`, or
|
||||
* through the call's request budget (`hops.fetch`, `request-hops.ts`).
|
||||
*/
|
||||
function wireClients(): string[] {
|
||||
return readdirSync(AI_DIR)
|
||||
.filter((f) => f.endsWith("-client.ts"))
|
||||
.map((f) => `lib/ai/${f}`)
|
||||
.filter((rel) => /\bsafeFetch\s*\(/.test(code(rel)))
|
||||
.filter((rel) => /\b(?:safeFetch|hops\.fetch)\s*\(/.test(code(rel)))
|
||||
.sort();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* Structural guard (#1279): a provider client that sends a request again on
|
||||
* its own sends every request through the call's budget, and hands that
|
||||
* budget to every repeat.
|
||||
*
|
||||
* A client that repeats a request (`this.generateCompletion(` or
|
||||
* `this.generateCompletionStream(` inside its own class) decided each repeat
|
||||
* from what it had just learned, and two lessons that undid each other made
|
||||
* one call send requests until something outside stopped it. The budget in
|
||||
* `src/lib/ai/request-hops.ts` refuses the request past two; it can only do
|
||||
* that if no request goes around it and no repeat starts a fresh one. So, in
|
||||
* every such client:
|
||||
*
|
||||
* - no request goes out through `safeFetch(` directly, only `hops.fetch(`;
|
||||
* - every repeat passes `hops` on;
|
||||
* - each public entry takes the budget as its last parameter, defaulting
|
||||
* to a fresh one, so a caller outside starts one per call.
|
||||
*
|
||||
* `request-hops.test.ts` walks the refusal and answerless combinations
|
||||
* against a loopback server and counts the requests.
|
||||
*
|
||||
* Its limit: it reads source text. A client that repeats through a helper
|
||||
* that is not a method named `generateCompletion*`, or sends through another
|
||||
* transport, is not found here.
|
||||
*
|
||||
* Mutation check: drop `hops` from one repeat, or put one `safeFetch(` back
|
||||
* into either client, and this goes red.
|
||||
*/
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
readSourceText,
|
||||
requireFloor,
|
||||
stripComments,
|
||||
walkSourceFiles,
|
||||
} from "./helpers/source-files";
|
||||
|
||||
const AI = join(process.cwd(), "src/lib/ai");
|
||||
|
||||
/** The text of the call that starts at `open` (the index of its `(`). */
|
||||
function callText(source: string, open: number): string {
|
||||
let depth = 0;
|
||||
for (let i = open; i < source.length; i += 1) {
|
||||
if (source[i] === "(") depth += 1;
|
||||
else if (source[i] === ")") {
|
||||
depth -= 1;
|
||||
if (depth === 0) return source.slice(open, i + 1);
|
||||
}
|
||||
}
|
||||
return source.slice(open);
|
||||
}
|
||||
|
||||
const REPEAT = /\bthis\.generateCompletion(?:Stream)?\s*\(/g;
|
||||
|
||||
function repeatingClients(): Array<{ rel: string; source: string }> {
|
||||
const out: Array<{ rel: string; source: string }> = [];
|
||||
for (const rel of walkSourceFiles(AI, { floor: 40 })) {
|
||||
if (rel.includes("__tests__/")) continue;
|
||||
const source = stripComments(readSourceText(join(AI, rel)));
|
||||
if (REPEAT.test(source)) out.push({ rel, source });
|
||||
REPEAT.lastIndex = 0;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
describe("a client's own repeats stay inside the call's request budget", () => {
|
||||
const clients = repeatingClients();
|
||||
|
||||
it("finds the clients that repeat a request", () => {
|
||||
// The Local client and the OpenAI-compatible gateway client today.
|
||||
requireFloor("repeating clients", clients, 2);
|
||||
expect(clients.map((c) => c.rel).sort()).toEqual([
|
||||
"local-client.ts",
|
||||
"openai-client.ts",
|
||||
]);
|
||||
});
|
||||
|
||||
it("sends every request through the budget", () => {
|
||||
for (const { rel, source } of clients) {
|
||||
expect({ rel, direct: /\bsafeFetch\s*\(/.test(source) }).toEqual({
|
||||
rel,
|
||||
direct: false,
|
||||
});
|
||||
expect({ rel, counted: /\bhops\.fetch\s*\(/.test(source) }).toEqual({
|
||||
rel,
|
||||
counted: true,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("hands the budget to every repeat", () => {
|
||||
const repeats: Array<{ rel: string; text: string }> = [];
|
||||
for (const { rel, source } of clients) {
|
||||
for (const m of source.matchAll(REPEAT)) {
|
||||
const open = (m.index ?? 0) + m[0].length - 1;
|
||||
repeats.push({ rel, text: callText(source, open) });
|
||||
}
|
||||
}
|
||||
requireFloor("repeats", repeats, 6);
|
||||
expect(
|
||||
repeats.filter(({ text }) => !/,\s*hops\s*,?\s*\)$/.test(text)),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("starts one budget per call at each public entry", () => {
|
||||
const entries: Array<{ rel: string; text: string }> = [];
|
||||
for (const { rel, source } of clients) {
|
||||
for (const m of source.matchAll(
|
||||
/\basync\s+generateCompletion(?:Stream)?\s*\(/g,
|
||||
)) {
|
||||
const open = (m.index ?? 0) + m[0].length - 1;
|
||||
entries.push({ rel, text: callText(source, open) });
|
||||
}
|
||||
}
|
||||
requireFloor("public entries", entries, 3);
|
||||
expect(
|
||||
entries.filter(
|
||||
({ text }) =>
|
||||
!/hops:\s*RequestHops\s*=\s*new RequestHops\(\)\s*,?\s*\)$/.test(
|
||||
text,
|
||||
),
|
||||
),
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -142,6 +142,14 @@ describe("background-task settle contract", () => {
|
||||
expect(pendingBackgroundTaskCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("drains a handle that is not a promise instead of spinning on it", async () => {
|
||||
// A mocked background starter that returns nothing hands the registry
|
||||
// `undefined`; left in the set, settle would loop on it forever.
|
||||
trackBackgroundTask(undefined as unknown as Promise<unknown>);
|
||||
await settleBackgroundTasks();
|
||||
expect(pendingBackgroundTaskCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("registers what fireAndForget starts", async () => {
|
||||
let release: () => void = () => {};
|
||||
fireAndForget(
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
|
||||
"count": 87,
|
||||
"total": 97455,
|
||||
"count": 86,
|
||||
"total": 96631,
|
||||
"entries": {
|
||||
"app/medications/page-client.tsx": 839,
|
||||
"components/charts/mood-chart.tsx": 1166,
|
||||
@@ -29,7 +29,6 @@
|
||||
"lib/ai/coach/stream-events.ts": 817,
|
||||
"lib/ai/coach/types.ts": 1011,
|
||||
"lib/ai/codex-client.ts": 1045,
|
||||
"lib/ai/local-client.ts": 824,
|
||||
"lib/ai/prompts/insight-generator.ts": 1110,
|
||||
"lib/ai/provider.ts": 1632,
|
||||
"lib/analytics/score/reader.ts": 998,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
|
||||
"count": 350,
|
||||
"total": 113403,
|
||||
"count": 349,
|
||||
"total": 113048,
|
||||
"entries": {
|
||||
"app/achievements/page.tsx :: AchievementsPage": 206,
|
||||
"app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 314,
|
||||
@@ -31,7 +31,6 @@
|
||||
"app/api/medications/[id]/dose-history/route.ts :: apiHandler(arg)": 256,
|
||||
"app/api/medications/[id]/inventory/[itemId]/route.ts :: apiHandler(arg)": 204,
|
||||
"app/api/medications/[id]/schedule-revisions/[revisionId]/route.ts :: apiHandler(arg)": 250,
|
||||
"app/api/medications/extract/route.ts :: handleExtract": 201,
|
||||
"app/api/mental-health/assessments/route.ts :: postAssessment": 249,
|
||||
"app/api/mood-entries/[id]/route.ts :: apiHandler(arg) #2": 296,
|
||||
"app/api/mood-entries/route.ts :: postMoodEntry": 324,
|
||||
@@ -250,12 +249,12 @@
|
||||
"lib/ai/coach/glp1-snapshot.ts :: buildGlp1SnapshotBlock": 218,
|
||||
"lib/ai/coach/snapshot.ts :: buildCoachSnapshotImpl": 881,
|
||||
"lib/ai/coach/tools/loop.ts :: runCoachToolLoop": 438,
|
||||
"lib/ai/coach/turn/model.ts :: runTurnModel": 540,
|
||||
"lib/ai/coach/turn/model.ts :: runTurnModel": 538,
|
||||
"lib/ai/coach/turn/pipeline.ts :: runTurn": 274,
|
||||
"lib/ai/coach/turn/reply-guards.ts :: guardReply": 331,
|
||||
"lib/ai/codex-client.ts :: parseResponse": 302,
|
||||
"lib/ai/local-client.ts :: generateCompletionStream": 294,
|
||||
"lib/ai/openai-client.ts :: generateCompletion": 335,
|
||||
"lib/ai/local-client.ts :: generateCompletionStream": 222,
|
||||
"lib/ai/openai-client.ts :: generateCompletion": 266,
|
||||
"lib/analytics/bp-in-target-fast-path.ts :: computeFromRollups": 203,
|
||||
"lib/analytics/correlations-fast-path.ts :: computeCorrelationHypothesesFastPath": 256,
|
||||
"lib/analytics/intraday-pulse-io.ts :: loadIntradayPulse": 206,
|
||||
@@ -297,7 +296,7 @@
|
||||
"lib/insights/comprehensive-generate.ts :: generateComprehensiveInsight": 876,
|
||||
"lib/insights/features.ts :: extractFeatures": 796,
|
||||
"lib/insights/mood-aggregates.ts :: fetchMoodAggregates": 246,
|
||||
"lib/insights/status-provider.ts :: runStatusCompletion": 226,
|
||||
"lib/insights/status-provider.ts :: runStatusCompletion": 215,
|
||||
"lib/integrations/connection-health-load.ts :: loadConnectionFacts": 227,
|
||||
"lib/integrations/status.ts :: recordSyncFailure": 292,
|
||||
"lib/jobs/apple-health-import-worker.ts :: handleAppleHealthImport": 300,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.",
|
||||
"count": 2897,
|
||||
"total": 9270,
|
||||
"total": 9249,
|
||||
"entries": {
|
||||
"__tests__/account-payload-consumer-guard.test.ts": 2,
|
||||
"__tests__/acting-account-boundary-guard.test.ts": 7,
|
||||
@@ -175,9 +175,9 @@
|
||||
"app/api/auth/refresh/route.ts": 2,
|
||||
"app/api/auth/register/__tests__/route.test.ts": 5,
|
||||
"app/api/auth/register/route.ts": 10,
|
||||
"app/api/biomarkers/__tests__/error-contract.test.ts": 1,
|
||||
"app/api/biomarkers/[id]/__tests__/route.test.ts": 2,
|
||||
"app/api/biomarkers/[id]/route.ts": 6,
|
||||
"app/api/biomarkers/__tests__/error-contract.test.ts": 1,
|
||||
"app/api/biomarkers/route.ts": 4,
|
||||
"app/api/c/[token]/unlock/__tests__/route.test.ts": 1,
|
||||
"app/api/c/[token]/unlock/route.ts": 1,
|
||||
@@ -187,12 +187,12 @@
|
||||
"app/api/coach/about-me/questions/__tests__/route-module-gate.test.ts": 1,
|
||||
"app/api/coach/about-me/questions/route.ts": 3,
|
||||
"app/api/coach/about-me/route.ts": 4,
|
||||
"app/api/coach/plans/__tests__/route.test.ts": 1,
|
||||
"app/api/coach/plans/[id]/route.ts": 2,
|
||||
"app/api/coach/plans/__tests__/route.test.ts": 1,
|
||||
"app/api/coach/plans/route.ts": 1,
|
||||
"app/api/coach/reminder-suggestions/route.ts": 1,
|
||||
"app/api/coach/reminders/__tests__/route.test.ts": 1,
|
||||
"app/api/coach/reminders/[id]/route.ts": 1,
|
||||
"app/api/coach/reminders/__tests__/route.test.ts": 1,
|
||||
"app/api/coach/reminders/route.ts": 1,
|
||||
"app/api/coach/suggested-actions/__tests__/route.test.ts": 1,
|
||||
"app/api/coach/suggested-actions/route.ts": 1,
|
||||
@@ -200,9 +200,9 @@
|
||||
"app/api/consent/ai/latest/__tests__/route.test.ts": 3,
|
||||
"app/api/consent/ai/latest/route.ts": 3,
|
||||
"app/api/consent/ai/route.ts": 3,
|
||||
"app/api/custom-metrics/[id]/entries/[entryId]/route.ts": 2,
|
||||
"app/api/custom-metrics/[id]/entries/__tests__/note-encryption.test.ts": 1,
|
||||
"app/api/custom-metrics/[id]/entries/__tests__/soft-delete-restore.test.ts": 1,
|
||||
"app/api/custom-metrics/[id]/entries/[entryId]/route.ts": 2,
|
||||
"app/api/custom-metrics/[id]/entries/route.ts": 4,
|
||||
"app/api/custom-metrics/[id]/route.ts": 1,
|
||||
"app/api/custom-metrics/route.ts": 1,
|
||||
@@ -212,8 +212,8 @@
|
||||
"app/api/cycle/day-logs/[id]/route.ts": 4,
|
||||
"app/api/cycle/day-logs/route.ts": 1,
|
||||
"app/api/cycle/period/route.ts": 4,
|
||||
"app/api/cycle/symptoms/custom/__tests__/custom-crud.test.ts": 1,
|
||||
"app/api/cycle/symptoms/custom/[key]/route.ts": 1,
|
||||
"app/api/cycle/symptoms/custom/__tests__/custom-crud.test.ts": 1,
|
||||
"app/api/cycle/symptoms/custom/route.ts": 3,
|
||||
"app/api/dashboard/chart-overlay-prefs/__tests__/route.test.ts": 1,
|
||||
"app/api/dashboard/chart-overlay-prefs/route.ts": 2,
|
||||
@@ -223,11 +223,10 @@
|
||||
"app/api/dashboard/summary/route.ts": 5,
|
||||
"app/api/dashboard/widgets/__tests__/route.test.ts": 4,
|
||||
"app/api/dashboard/widgets/route.ts": 14,
|
||||
"app/api/devices/__tests__/route.test.ts": 1,
|
||||
"app/api/devices/[id]/__tests__/route.test.ts": 1,
|
||||
"app/api/devices/[id]/route.ts": 1,
|
||||
"app/api/devices/__tests__/route.test.ts": 1,
|
||||
"app/api/devices/route.ts": 4,
|
||||
"app/api/documents/inbound/__tests__/route.test.ts": 1,
|
||||
"app/api/documents/inbound/[id]/__tests__/route.test.ts": 1,
|
||||
"app/api/documents/inbound/[id]/chat/__tests__/route.test.ts": 2,
|
||||
"app/api/documents/inbound/[id]/chat/route.ts": 1,
|
||||
@@ -242,6 +241,7 @@
|
||||
"app/api/documents/inbound/[id]/suggest/route.ts": 2,
|
||||
"app/api/documents/inbound/[id]/summary/route.ts": 2,
|
||||
"app/api/documents/inbound/[id]/thumbnail/route.ts": 1,
|
||||
"app/api/documents/inbound/__tests__/route.test.ts": 1,
|
||||
"app/api/documents/inbound/reindex/route.ts": 1,
|
||||
"app/api/documents/inbound/route.ts": 2,
|
||||
"app/api/documents/inbound/usage/route.ts": 1,
|
||||
@@ -260,10 +260,10 @@
|
||||
"app/api/export/mood/route.ts": 2,
|
||||
"app/api/family-history/[id]/route.ts": 2,
|
||||
"app/api/family-history/route.ts": 1,
|
||||
"app/api/fhir/Patient/$everything/route.ts": 1,
|
||||
"app/api/fhir/__tests__/observation-series.test.ts": 1,
|
||||
"app/api/fhir/__tests__/route.test.ts": 1,
|
||||
"app/api/fhir/metadata/route.ts": 1,
|
||||
"app/api/fhir/Patient/$everything/route.ts": 1,
|
||||
"app/api/fitbit/callback/route.ts": 1,
|
||||
"app/api/fitbit/connect/route.ts": 1,
|
||||
"app/api/fitbit/credentials/route.ts": 1,
|
||||
@@ -292,6 +292,11 @@
|
||||
"app/api/insights/breathing-screening/route.ts": 2,
|
||||
"app/api/insights/cards/__tests__/route.test.ts": 2,
|
||||
"app/api/insights/cards/route.ts": 9,
|
||||
"app/api/insights/chat/[id]/attachments/[documentId]/route.ts": 1,
|
||||
"app/api/insights/chat/[id]/attachments/__tests__/route.test.ts": 1,
|
||||
"app/api/insights/chat/[id]/attachments/route.ts": 1,
|
||||
"app/api/insights/chat/[id]/messages/[messageId]/trail/__tests__/route.test.ts": 1,
|
||||
"app/api/insights/chat/[id]/route.ts": 1,
|
||||
"app/api/insights/chat/__tests__/dialog-harness.ts": 1,
|
||||
"app/api/insights/chat/__tests__/golden-transcripts.test.ts": 1,
|
||||
"app/api/insights/chat/__tests__/route-activity.test.ts": 1,
|
||||
@@ -303,19 +308,14 @@
|
||||
"app/api/insights/chat/__tests__/route-snapshot-once.test.ts": 2,
|
||||
"app/api/insights/chat/__tests__/route-tool-mode.test.ts": 4,
|
||||
"app/api/insights/chat/__tests__/route-workout-scope.test.ts": 1,
|
||||
"app/api/insights/chat/[id]/attachments/__tests__/route.test.ts": 1,
|
||||
"app/api/insights/chat/[id]/attachments/[documentId]/route.ts": 1,
|
||||
"app/api/insights/chat/[id]/attachments/route.ts": 1,
|
||||
"app/api/insights/chat/[id]/messages/[messageId]/trail/__tests__/route.test.ts": 1,
|
||||
"app/api/insights/chat/[id]/route.ts": 1,
|
||||
"app/api/insights/chat/fenced/__tests__/route.test.ts": 1,
|
||||
"app/api/insights/chat/fenced/route.ts": 1,
|
||||
"app/api/insights/chat/messages/[id]/feedback/route.ts": 3,
|
||||
"app/api/insights/chat/route.ts": 9,
|
||||
"app/api/insights/coach-read/route.ts": 2,
|
||||
"app/api/insights/coach/facts/[id]/route.ts": 2,
|
||||
"app/api/insights/coach/facts/__tests__/route-memory.test.ts": 1,
|
||||
"app/api/insights/coach/facts/__tests__/route.test.ts": 1,
|
||||
"app/api/insights/coach/facts/[id]/route.ts": 2,
|
||||
"app/api/insights/coach/facts/route.ts": 1,
|
||||
"app/api/insights/coach/nudge-status/route.ts": 1,
|
||||
"app/api/insights/coach/seen/route.ts": 1,
|
||||
@@ -326,9 +326,9 @@
|
||||
"app/api/insights/derived/batch/__tests__/route.test.ts": 3,
|
||||
"app/api/insights/derived/batch/route.ts": 6,
|
||||
"app/api/insights/derived/route.ts": 5,
|
||||
"app/api/insights/ecg/__tests__/route.test.ts": 1,
|
||||
"app/api/insights/ecg/[id]/__tests__/route.test.ts": 1,
|
||||
"app/api/insights/ecg/[id]/route.ts": 2,
|
||||
"app/api/insights/ecg/__tests__/route.test.ts": 1,
|
||||
"app/api/insights/ecg/route.ts": 1,
|
||||
"app/api/insights/feedback/route.ts": 2,
|
||||
"app/api/insights/generate/__tests__/route.test.ts": 12,
|
||||
@@ -367,8 +367,8 @@
|
||||
"app/api/internal/deploy-webhook/route.ts": 1,
|
||||
"app/api/internal/web-vitals/__tests__/route.test.ts": 1,
|
||||
"app/api/internal/web-vitals/route.ts": 1,
|
||||
"app/api/labs/__tests__/error-contract.test.ts": 1,
|
||||
"app/api/labs/[id]/route.ts": 7,
|
||||
"app/api/labs/__tests__/error-contract.test.ts": 1,
|
||||
"app/api/labs/ocr/capability/route.ts": 1,
|
||||
"app/api/labs/ocr/commit/__tests__/route.test.ts": 2,
|
||||
"app/api/labs/ocr/commit/route.ts": 4,
|
||||
@@ -388,13 +388,13 @@
|
||||
"app/api/measurement-reminders/[id]/skip/route.ts": 1,
|
||||
"app/api/measurement-reminders/[id]/snooze/route.ts": 1,
|
||||
"app/api/measurement-reminders/route.ts": 3,
|
||||
"app/api/measurements/[id]/__tests__/route.test.ts": 2,
|
||||
"app/api/measurements/[id]/route.ts": 16,
|
||||
"app/api/measurements/__tests__/arrival-wiring.test.ts": 1,
|
||||
"app/api/measurements/__tests__/group-by-day.test.ts": 8,
|
||||
"app/api/measurements/__tests__/put-duplicate-timestamp.test.ts": 1,
|
||||
"app/api/measurements/__tests__/range-aggregation-route.test.ts": 3,
|
||||
"app/api/measurements/__tests__/zod-multi-issue.test.ts": 4,
|
||||
"app/api/measurements/[id]/__tests__/route.test.ts": 2,
|
||||
"app/api/measurements/[id]/route.ts": 16,
|
||||
"app/api/measurements/batch/__tests__/aggregated-hr-wire.test.ts": 1,
|
||||
"app/api/measurements/batch/__tests__/duplicate-prefilter.test.ts": 2,
|
||||
"app/api/measurements/batch/__tests__/hr-bucket-observability.test.ts": 1,
|
||||
@@ -411,7 +411,6 @@
|
||||
"app/api/measurements/series-batch/route.ts": 1,
|
||||
"app/api/measurements/series/__tests__/route.test.ts": 5,
|
||||
"app/api/measurements/series/route.ts": 6,
|
||||
"app/api/medications/__tests__/route.test.ts": 10,
|
||||
"app/api/medications/[id]/__tests__/route.test.ts": 8,
|
||||
"app/api/medications/[id]/api-endpoint/route.ts": 2,
|
||||
"app/api/medications/[id]/cadence/route.ts": 10,
|
||||
@@ -423,10 +422,10 @@
|
||||
"app/api/medications/[id]/dose-history/route.ts": 14,
|
||||
"app/api/medications/[id]/glp1/__tests__/route.test.ts": 3,
|
||||
"app/api/medications/[id]/glp1/route.ts": 8,
|
||||
"app/api/medications/[id]/intake/__tests__/route.test.ts": 11,
|
||||
"app/api/medications/[id]/intake/__tests__/source-provenance.test.ts": 1,
|
||||
"app/api/medications/[id]/intake/[eventId]/__tests__/route.test.ts": 9,
|
||||
"app/api/medications/[id]/intake/[eventId]/route.ts": 4,
|
||||
"app/api/medications/[id]/intake/__tests__/route.test.ts": 11,
|
||||
"app/api/medications/[id]/intake/__tests__/source-provenance.test.ts": 1,
|
||||
"app/api/medications/[id]/intake/bulk-delete/route.ts": 7,
|
||||
"app/api/medications/[id]/intake/import/__tests__/route.test.ts": 2,
|
||||
"app/api/medications/[id]/intake/import/route.ts": 6,
|
||||
@@ -434,11 +433,12 @@
|
||||
"app/api/medications/[id]/inventory/[itemId]/route.ts": 8,
|
||||
"app/api/medications/[id]/inventory/route.ts": 4,
|
||||
"app/api/medications/[id]/phase-config/route.ts": 4,
|
||||
"app/api/medications/[id]/schedule-revisions/__tests__/route.test.ts": 1,
|
||||
"app/api/medications/[id]/schedule-revisions/[revisionId]/route.ts": 9,
|
||||
"app/api/medications/[id]/schedule-revisions/__tests__/route.test.ts": 1,
|
||||
"app/api/medications/[id]/schedule-revisions/route.ts": 3,
|
||||
"app/api/medications/[id]/side-effects/[logId]/route.ts": 2,
|
||||
"app/api/medications/[id]/side-effects/route.ts": 5,
|
||||
"app/api/medications/__tests__/route.test.ts": 10,
|
||||
"app/api/medications/categories/[key]/route.ts": 1,
|
||||
"app/api/medications/categories/route.ts": 1,
|
||||
"app/api/medications/compliance/__tests__/route.test.ts": 2,
|
||||
@@ -455,9 +455,9 @@
|
||||
"app/api/medications/route.ts": 2,
|
||||
"app/api/mental-health/assessments/__tests__/route.test.ts": 2,
|
||||
"app/api/mental-health/assessments/route.ts": 6,
|
||||
"app/api/mood-entries/__tests__/route.test.ts": 7,
|
||||
"app/api/mood-entries/[id]/__tests__/route.test.ts": 4,
|
||||
"app/api/mood-entries/[id]/route.ts": 22,
|
||||
"app/api/mood-entries/__tests__/route.test.ts": 7,
|
||||
"app/api/mood-entries/bulk-delete/__tests__/route.test.ts": 1,
|
||||
"app/api/mood-entries/bulk-delete/route.ts": 4,
|
||||
"app/api/mood-entries/bulk/__tests__/route.test.ts": 5,
|
||||
@@ -547,12 +547,12 @@
|
||||
"app/api/withings/connect/__tests__/route.test.ts": 1,
|
||||
"app/api/withings/connect/route.ts": 3,
|
||||
"app/api/withings/status/route.ts": 1,
|
||||
"app/api/withings/webhook/__tests__/route.test.ts": 1,
|
||||
"app/api/withings/webhook/[token]/__tests__/route.test.ts": 1,
|
||||
"app/api/withings/webhook/__tests__/route.test.ts": 1,
|
||||
"app/api/withings/webhook/route.ts": 2,
|
||||
"app/api/workouts/[id]/route.ts": 2,
|
||||
"app/api/workouts/__tests__/batch-create.test.ts": 3,
|
||||
"app/api/workouts/__tests__/canonical-dedup.test.ts": 3,
|
||||
"app/api/workouts/[id]/route.ts": 2,
|
||||
"app/api/workouts/batch/route.ts": 3,
|
||||
"app/api/workouts/route.ts": 2,
|
||||
"app/auth/claim/page.tsx": 1,
|
||||
@@ -654,20 +654,20 @@
|
||||
"app/loading.tsx": 2,
|
||||
"app/mcp/__tests__/route.test.ts": 5,
|
||||
"app/measurements/page.tsx": 5,
|
||||
"app/medications/[id]/history/page.tsx": 1,
|
||||
"app/medications/[id]/page.tsx": 2,
|
||||
"app/medications/__tests__/category-filter.test.tsx": 2,
|
||||
"app/medications/__tests__/detail-page-history-surface.test.ts": 1,
|
||||
"app/medications/__tests__/list-view-toggle.test.tsx": 5,
|
||||
"app/medications/__tests__/page-prefetch.test.tsx": 1,
|
||||
"app/medications/__tests__/take-all-due-affordance.test.tsx": 1,
|
||||
"app/medications/[id]/history/page.tsx": 1,
|
||||
"app/medications/[id]/page.tsx": 2,
|
||||
"app/medications/new/page.tsx": 1,
|
||||
"app/medications/page-client.tsx": 50,
|
||||
"app/mental-wellbeing/page.tsx": 1,
|
||||
"app/mood/page-client.tsx": 2,
|
||||
"app/not-found.tsx": 2,
|
||||
"app/onboarding/__tests__/page.test.tsx": 1,
|
||||
"app/onboarding/[step]/page.tsx": 1,
|
||||
"app/onboarding/__tests__/page.test.tsx": 1,
|
||||
"app/onboarding/handover/page.tsx": 1,
|
||||
"app/onboarding/page.tsx": 2,
|
||||
"app/privacy/__tests__/page.test.tsx": 1,
|
||||
@@ -1748,10 +1748,10 @@
|
||||
"lib/ai/insight-interpretation.ts": 1,
|
||||
"lib/ai/legacy-payload.ts": 2,
|
||||
"lib/ai/legacy-private-host-report.ts": 1,
|
||||
"lib/ai/local-client.ts": 18,
|
||||
"lib/ai/local-client.ts": 12,
|
||||
"lib/ai/medical-references.ts": 1,
|
||||
"lib/ai/mock-client.ts": 5,
|
||||
"lib/ai/openai-client.ts": 16,
|
||||
"lib/ai/openai-client.ts": 9,
|
||||
"lib/ai/prompts/__tests__/assessment-prompt-contract.test.ts": 5,
|
||||
"lib/ai/prompts/__tests__/briefing-personalisation.test.ts": 1,
|
||||
"lib/ai/prompts/__tests__/coach-prompt-locale.test.ts": 1,
|
||||
@@ -1783,7 +1783,7 @@
|
||||
"lib/ai/prompts/weight.ts": 2,
|
||||
"lib/ai/provider-chain.ts": 1,
|
||||
"lib/ai/provider-health-ledger.ts": 3,
|
||||
"lib/ai/provider-runner.ts": 15,
|
||||
"lib/ai/provider-runner.ts": 11,
|
||||
"lib/ai/provider.ts": 10,
|
||||
"lib/ai/reasoning/__tests__/controls.test.ts": 1,
|
||||
"lib/ai/reasoning/controls.ts": 2,
|
||||
@@ -2158,7 +2158,7 @@
|
||||
"lib/insights/chart-tokens.ts": 24,
|
||||
"lib/insights/coach-launch-context.tsx": 10,
|
||||
"lib/insights/comprehensive-aggregator.ts": 27,
|
||||
"lib/insights/comprehensive-generate.ts": 48,
|
||||
"lib/insights/comprehensive-generate.ts": 45,
|
||||
"lib/insights/correlation-channel-series.ts": 16,
|
||||
"lib/insights/correlation-discovery.ts": 30,
|
||||
"lib/insights/correlation-series-builders.ts": 1,
|
||||
@@ -2752,7 +2752,7 @@
|
||||
"lib/rollups/mood-rollups.ts": 4,
|
||||
"lib/rollups/tiered-context.ts": 6,
|
||||
"lib/score-config/rows.ts": 1,
|
||||
"lib/settings/section-slugs.ts": 20,
|
||||
"lib/settings/section-slugs.ts": 19,
|
||||
"lib/sharing/__tests__/account-access-parse.test.ts": 3,
|
||||
"lib/sharing/__tests__/account-access.test.ts": 2,
|
||||
"lib/sharing/__tests__/account-label.test.ts": 1,
|
||||
|
||||
@@ -91,3 +91,56 @@ describe("detached work cannot rename the request's event", () => {
|
||||
expect(startsDetached(detached)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Read-path work a request starts and never awaits. Each of these runs real
|
||||
* statements against Postgres after the response is already on its way, so a
|
||||
* bare `void` hides it from `settleBackgroundTasks()`: the statement counter
|
||||
* then closes its window while a stale account's rollup recompute is still
|
||||
* writing, and the next test inherits the tail. The dashboard summary budget
|
||||
* read 30 against 28 that way, depending on which side of the window a full
|
||||
* mood recompute landed.
|
||||
*/
|
||||
const READ_PATH_BACKGROUND_STARTERS = [
|
||||
"ensureUserRollupsFresh",
|
||||
"ensureUserMoodRollupsFresh",
|
||||
"requestTodayBriefingWarm",
|
||||
"touchGrantUsage",
|
||||
"recordDelegatedAccess",
|
||||
] as const;
|
||||
|
||||
describe("read-path background work is registered", () => {
|
||||
const files = readSourceFolder("src", {
|
||||
floor: 3000,
|
||||
include: (rel) =>
|
||||
!rel.startsWith("generated/") &&
|
||||
!rel.includes("__tests__/") &&
|
||||
!/\.test\.tsx?$/.test(rel),
|
||||
});
|
||||
const bare = new RegExp(
|
||||
String.raw`\bvoid\s+(${READ_PATH_BACKGROUND_STARTERS.join("|")})\s*\(`,
|
||||
);
|
||||
|
||||
it("starts none of them with a bare void", () => {
|
||||
const offenders = files.flatMap(([rel, text]) =>
|
||||
stripComments(text)
|
||||
.split("\n")
|
||||
.flatMap((line, i) => (bare.test(line) ? [`src/${rel}:${i + 1}`] : [])),
|
||||
);
|
||||
expect(
|
||||
offenders,
|
||||
"wrap the call in trackBackgroundTask(runDetached(() => ...)) or " +
|
||||
"fireAndForget(...) so the suite can await it",
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("still finds each starter at a detached call site", () => {
|
||||
const joined = files.map(([, text]) => stripComments(text)).join("\n");
|
||||
for (const name of READ_PATH_BACKGROUND_STARTERS) {
|
||||
expect(
|
||||
new RegExp(String.raw`=>\s*${name}\s*\(`).test(joined),
|
||||
`${name} is no longer started detached anywhere; drop it from the list`,
|
||||
).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -2172,8 +2172,6 @@ const ACTOR_ROUTES: Record<string, string> = {
|
||||
"The UI language belongs to the person reading the screen, never to the record on it. Under a record scope this would transplant a delegate's choice onto the owner's row and send the owner's cron mail in a language they may not read. It has to keep working rather than merely refuse safely: the switcher's mount-time backfill fires on every page load, including the shared dashboard.",
|
||||
"app/api/settings/module-list-layout/route.ts":
|
||||
"How the person's own labs, illness and checkups lists are laid out: card or list view, manual order, sort. It reads and writes the caller's own row only. Inside another record the ids of a saved order simply match nothing and the list keeps the server order, so serving the person's own choice there reveals nothing of the owner's and grants nothing; refusing it under a switch broke the list controls in exactly the context they are used in.",
|
||||
"app/api/settings/preview-features/route.ts":
|
||||
"The person's own opt-in to preview features. A preview entry is chrome of the person reading the screen, never a property of the record on it; it reads and writes the caller's own row and nothing of the owner's.",
|
||||
"app/api/feature-flags/route.ts":
|
||||
"Reads the `AppSettings` singleton and no user row at all, so there is no record for a switch to substitute. It answers about the deployment, which for this purpose is the caller's side of the request; the shell gates the Coach launcher and the assistant surfaces on it, so a 403 here is a piece of chrome deciding it does not exist.",
|
||||
};
|
||||
@@ -2283,11 +2281,14 @@ const ACTOR_ROUTES: Record<string, string> = {
|
||||
* owner-only, and a delegate sees the goals only as the calendar reports them.
|
||||
* 270 -> 273.
|
||||
*
|
||||
* The preview-feature switch went with its column in the next release; the
|
||||
* topic pages it was kept for were dropped. 273 -> 272.
|
||||
*
|
||||
* Ending a running course (#1280) joins the record list and the
|
||||
* manage literal: it was refused under a switch while the medication's PUT,
|
||||
* which used to carry the end, admitted a manager. 273 -> 275.
|
||||
* which used to carry the end, admitted a manager. 272 -> 274.
|
||||
*/
|
||||
const FROZEN_ENTRY_COUNT = 275;
|
||||
const FROZEN_ENTRY_COUNT = 274;
|
||||
|
||||
/**
|
||||
* The two surfaces that authenticate a Bearer token outside `requireAuth` —
|
||||
|
||||
@@ -16,6 +16,10 @@ vi.mock("@/lib/db", () => ({
|
||||
upsert: vi.fn(),
|
||||
},
|
||||
user: { findMany: vi.fn().mockResolvedValue([]) },
|
||||
providerHealth: { deleteMany: vi.fn().mockResolvedValue({ count: 0 }) },
|
||||
insightStatusCache: {
|
||||
updateMany: vi.fn().mockResolvedValue({ count: 0 }),
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
@@ -217,6 +221,31 @@ describe("PUT /api/admin/ai-settings", () => {
|
||||
statusCode: 429,
|
||||
});
|
||||
});
|
||||
|
||||
// #1279 — background work leaves a provider the health ledger benched out
|
||||
// of every account's chain. A fixed operator provider must not sit out a
|
||||
// pause the broken one earned, so a saved change forgets what the ledger
|
||||
// recorded for it on every account, and lets their notes try again.
|
||||
it("forgets the operator provider's recorded failures on every account", async () => {
|
||||
upsertEcho();
|
||||
await PUT(jsonReq({ model: "gpt-4o-mini" }));
|
||||
expect(prisma.providerHealth.deleteMany).toHaveBeenCalledWith({
|
||||
where: { providerType: "admin-openai" },
|
||||
});
|
||||
expect(prisma.insightStatusCache.updateMany).toHaveBeenCalledWith({
|
||||
where: {
|
||||
OR: [{ retryAt: { not: null } }, { failureStreak: { gt: 0 } }],
|
||||
},
|
||||
data: { retryAt: null, negativeReason: null, failureStreak: 0 },
|
||||
});
|
||||
});
|
||||
|
||||
it("forgets nothing when the change is refused", async () => {
|
||||
await expect(
|
||||
PUT(jsonReq({ baseUrl: "https://attacker.example/v1" })),
|
||||
).rejects.toMatchObject({ statusCode: 422 });
|
||||
expect(prisma.providerHealth.deleteMany).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
|
||||
@@ -8,6 +8,8 @@ import { encrypt, decrypt } from "@/lib/crypto";
|
||||
import { checkRateLimit } from "@/lib/rate-limit";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { legacyPrivateHostReport } from "@/lib/ai/legacy-private-host-report";
|
||||
import { forgetProviderHealth } from "@/lib/ai/provider-health-ledger";
|
||||
import { clearStatusNegativeWindows } from "@/lib/insights/status-negative-window";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
@@ -134,6 +136,16 @@ export const PUT = apiHandler(async (request: NextRequest) => {
|
||||
details: auditDetails,
|
||||
});
|
||||
|
||||
// #1279 — background work leaves out a provider the health ledger benched,
|
||||
// on every account whose chain falls back on this one, and a status note
|
||||
// waits out its negative window. A fixed operator provider must not sit out
|
||||
// a pause the broken one earned, so what was recorded against it is
|
||||
// forgotten on every account, as a person's own save does for theirs.
|
||||
await Promise.all([
|
||||
forgetProviderHealth({ providerType: "admin-openai" }),
|
||||
clearStatusNegativeWindows(),
|
||||
]);
|
||||
|
||||
const hasKey = Boolean(settings.adminAiKeyEncrypted);
|
||||
|
||||
return apiSuccess({
|
||||
|
||||
@@ -9,6 +9,7 @@ import { auditLog } from "@/lib/auth/audit";
|
||||
import { getClientIp, apiSuccess, apiError } from "@/lib/api-response";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { forgetProviderHealth } from "@/lib/ai/provider-health-ledger";
|
||||
import { clearStatusNegativeWindows } from "@/lib/insights/status-negative-window";
|
||||
|
||||
/**
|
||||
* Operator-shared central Codex — device-code poll (admin only).
|
||||
@@ -94,7 +95,12 @@ export const POST = apiHandler(async (request: NextRequest) => {
|
||||
cookieStore.delete("central_codex_device");
|
||||
// A new central sign-in: every account's ledger rows describe the refused
|
||||
// one, and the central connection is one credential shared by all of them.
|
||||
await forgetProviderHealth({ providerType: "admin-codex" });
|
||||
// #1279 — and every record's note windows close, as on the admin AI
|
||||
// settings save, so no note sits out a window the broken one earned.
|
||||
await Promise.all([
|
||||
forgetProviderHealth({ providerType: "admin-codex" }),
|
||||
clearStatusNegativeWindows(),
|
||||
]);
|
||||
|
||||
await auditLog("admin.central-codex.connected", {
|
||||
userId: user.id,
|
||||
|
||||
@@ -10,6 +10,7 @@ import { getClientIp, apiSuccess, apiError } from "@/lib/api-response";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { INSIGHTS_CACHE_CLEARED } from "@/lib/insights/cached-text-cipher";
|
||||
import { forgetProviderHealth } from "@/lib/ai/provider-health-ledger";
|
||||
import { clearStatusNegativeWindows } from "@/lib/insights/status-negative-window";
|
||||
|
||||
/**
|
||||
* v1.4.7.1: Device-code flow polling endpoint.
|
||||
@@ -90,8 +91,12 @@ export const POST = apiHandler(async (request: NextRequest) => {
|
||||
|
||||
cookieStore.delete("codex_device");
|
||||
// A new sign-in: whatever the ledger held against the old one no longer
|
||||
// describes this credential.
|
||||
await forgetProviderHealth({ userId: user.id, providerType: "codex" });
|
||||
// describes this credential, and (#1279) the person's notes stop waiting
|
||||
// out windows it earned, as on a save of their provider settings.
|
||||
await Promise.all([
|
||||
forgetProviderHealth({ userId: user.id, providerType: "codex" }),
|
||||
clearStatusNegativeWindows(user.id),
|
||||
]);
|
||||
|
||||
await auditLog("codex.oauth.connected", {
|
||||
userId: user.id,
|
||||
|
||||
@@ -210,6 +210,7 @@ beforeEach(() => {
|
||||
},
|
||||
workingProvider: { providerType: "anthropic", instance: {} },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
} as never);
|
||||
});
|
||||
|
||||
@@ -356,6 +357,7 @@ describe("POST — numeric grounding", () => {
|
||||
},
|
||||
workingProvider: { providerType: "anthropic", instance: {} },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
} as never);
|
||||
const res = await POST(
|
||||
req("doc-1", { message: "what is my ldl?" }) as never,
|
||||
@@ -393,6 +395,7 @@ describe("POST — numeric grounding", () => {
|
||||
},
|
||||
workingProvider: { providerType: "anthropic", instance: {} },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
} as never);
|
||||
const res = await POST(
|
||||
req("doc-1", { message: "how is my ldl?" }) as never,
|
||||
|
||||
@@ -325,6 +325,7 @@ async function runRawCompletionWithFallback(args: {
|
||||
},
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -349,6 +349,7 @@ export function streamingResult(content: string) {
|
||||
model: "m-local",
|
||||
},
|
||||
workingProvider: { providerType: "local" },
|
||||
tokensSpent: 0,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
@@ -264,6 +264,7 @@ async function noToolTurn(content: string): Promise<string> {
|
||||
runStreamingRawCompletionWithFallback.mockResolvedValue({
|
||||
result: { content, tokensUsed: 42, model: "m" },
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
tokensSpent: 0,
|
||||
});
|
||||
const res = await post(chatReq({ message: "How is my BP?" }));
|
||||
await sse.done;
|
||||
|
||||
@@ -220,6 +220,7 @@ describe("coach chat — stateless no-tools snapshot grounding", () => {
|
||||
runStreamingRawCompletionWithFallback.mockResolvedValue({
|
||||
result: { content: "Your BP looks stable.", tokensUsed: 42, model: "m" },
|
||||
workingProvider: { providerType: "admin-openai" },
|
||||
tokensSpent: 0,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -276,6 +277,7 @@ describe("coach chat — snapshot stays grounded across transcript elision", ()
|
||||
runStreamingRawCompletionWithFallback.mockResolvedValue({
|
||||
result: { content: "Your BP looks stable.", tokensUsed: 42, model: "m" },
|
||||
workingProvider: { providerType: "admin-openai" },
|
||||
tokensSpent: 0,
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -569,6 +569,7 @@ describe("coach chat — tool-mode routing (F1)", () => {
|
||||
runStreamingRawCompletionWithFallback.mockResolvedValue({
|
||||
result: { content: "Your BP is steady.", tokensUsed: 42, model: "m" },
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
tokensSpent: 0,
|
||||
});
|
||||
await postAndDrain({ message: "How is my BP?" });
|
||||
|
||||
|
||||
@@ -292,6 +292,7 @@ beforeEach(() => {
|
||||
model: "m",
|
||||
},
|
||||
workingProvider: { providerType: "admin-openai" },
|
||||
tokensSpent: 0,
|
||||
});
|
||||
isModuleEnabled.mockResolvedValue(true);
|
||||
});
|
||||
|
||||
@@ -71,6 +71,9 @@ vi.mock("@/lib/db", () => ({
|
||||
consentReceipt: {
|
||||
findFirst: vi.fn(async () => ({ id: "receipt-1" })),
|
||||
},
|
||||
// The provider health ledger the chain walk reads. Nothing benched by
|
||||
// default; the bench case below seeds a row.
|
||||
providerHealth: { findMany: vi.fn(async () => []) },
|
||||
},
|
||||
}));
|
||||
|
||||
@@ -160,6 +163,8 @@ vi.mock("@/lib/i18n/server-locale", () => ({
|
||||
resolveServerLocale: vi.fn(async () => "en"),
|
||||
}));
|
||||
|
||||
import { NextRequest } from "next/server";
|
||||
|
||||
import { GET, POST, resolveInsightsRateLimit } from "../route";
|
||||
import { enqueueStatusRefillForUser } from "@/lib/insights/comprehensive-generate";
|
||||
import { resolveProvider } from "@/lib/ai/provider";
|
||||
@@ -284,6 +289,51 @@ describe("POST /api/insights/generate — daily token ceiling", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("POST /api/insights/generate — a provider the health ledger benched", () => {
|
||||
// #1279 — "Re-run analysis" is a person waiting on the answer. Background
|
||||
// work leaves a benched provider out; this call walks it at the tail of
|
||||
// the chain, so it reaches the provider and reports what the provider
|
||||
// said, not a connection failure that never happened.
|
||||
it("still asks the benched provider and reports its own refusal", async () => {
|
||||
const now = Date.now();
|
||||
vi.mocked(prisma.providerHealth.findMany).mockResolvedValueOnce([
|
||||
{
|
||||
id: "ph-1",
|
||||
userId: "u-1",
|
||||
providerType: "admin-openai",
|
||||
lastResult: "auth_failed",
|
||||
lastStatus: 401,
|
||||
consecutiveFailures: 1,
|
||||
lastOkAt: null,
|
||||
lastFailureAt: new Date(now),
|
||||
nextRetryAt: new Date(now + 60 * 60 * 1000),
|
||||
updatedAt: new Date(now),
|
||||
},
|
||||
]);
|
||||
const err = Object.assign(new Error("OpenAI request failed (401)"), {
|
||||
httpStatus: 401,
|
||||
});
|
||||
makeProviderThatThrows(err);
|
||||
const provider = await vi.mocked(resolveProvider)("u-1");
|
||||
const body = JSON.stringify({ force: true });
|
||||
const request = new NextRequest("http://localhost/api/insights/generate", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"content-length": String(body.length),
|
||||
},
|
||||
body,
|
||||
});
|
||||
|
||||
const res = await POST(request);
|
||||
|
||||
expect(provider.generateCompletion).toHaveBeenCalledTimes(1);
|
||||
expect(res.status).toBe(422);
|
||||
const envelope = (await res.json()) as ApiErrorEnvelope;
|
||||
expect(envelope.error).toMatch(/AI provider rejected/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe("POST /api/insights/generate — provider error mapping", () => {
|
||||
it("rejects a body over the 16 KB cap with 413 before parsing", async () => {
|
||||
const res = await POST(
|
||||
|
||||
@@ -192,6 +192,7 @@ describe("POST /api/medications/extract — happy path", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai", instance: {} as never },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const res = await POST(
|
||||
@@ -237,6 +238,7 @@ describe("POST /api/medications/extract — happy path", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai", instance: {} as never },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const res = await POST(
|
||||
@@ -261,6 +263,7 @@ describe("POST /api/medications/extract — daily budget", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai", instance: {} as never },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -409,6 +412,7 @@ describe("POST /api/medications/extract — validation + provider errors", () =>
|
||||
},
|
||||
workingProvider: { providerType: "openai", instance: {} as never },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const res = await POST(postReq({ text: "Mounjaro 5mg weekly" }) as never);
|
||||
|
||||
@@ -71,6 +71,7 @@ import {
|
||||
AllProvidersFailedError,
|
||||
runRawCompletionWithFallback,
|
||||
} from "@/lib/ai/provider-runner";
|
||||
import { spentOnFailure } from "@/lib/ai/reasoning/answerless";
|
||||
import { resolveProvider, resolveProviderChain } from "@/lib/ai/provider";
|
||||
import { singleUserTurn } from "@/lib/ai/types";
|
||||
|
||||
@@ -230,13 +231,9 @@ async function handleExtract(request: NextRequest): Promise<Response> {
|
||||
locale,
|
||||
});
|
||||
|
||||
// Atomically reserve the day's budget BEFORE the provider call. The old
|
||||
// `enforceBudget` read-then-write left a TOCTOU window (concurrent calls
|
||||
// could each observe a sub-cap total and all proceed) AND passed no cap, so
|
||||
// it always measured against `OPERATOR_COST_CAP` — rationing a self-hoster
|
||||
// paying for their own key by the operator's ceiling. `resolveDailyCap`
|
||||
// charges the operator ceiling only when the chain's primary is the
|
||||
// operator's own credential; every other chain gets the user-plan ceiling.
|
||||
// Atomically reserve the day's budget BEFORE the provider call, against the
|
||||
// cost owner's ceiling: the operator's only when the chain's primary is the
|
||||
// operator's own credential, the user-plan ceiling for every other chain.
|
||||
const dateKey = accountToday;
|
||||
const estimatedTokens =
|
||||
MAX_OUTPUT_TOKENS +
|
||||
@@ -268,11 +265,9 @@ async function handleExtract(request: NextRequest): Promise<Response> {
|
||||
params: singleUserTurn({
|
||||
system: systemPrompt,
|
||||
user: userPrompt,
|
||||
// Low temperature because we want a deterministic field
|
||||
// extractor, not a creative rephrase.
|
||||
// A deterministic field extractor, not a creative rephrase.
|
||||
temperature: 0.1,
|
||||
// The reply body is a single small JSON object; 600 is well
|
||||
// over the empirical max we have seen on a 2k-char input.
|
||||
// One small JSON object; well over the largest seen on 2k chars.
|
||||
maxTokens: MAX_OUTPUT_TOKENS,
|
||||
// Parsed with `parseModelJson` (JSON.parse) below — opt the OpenAI /
|
||||
// Codex chains into their strict JSON-object mode (gated on this flag).
|
||||
@@ -280,12 +275,17 @@ async function handleExtract(request: NextRequest): Promise<Response> {
|
||||
}),
|
||||
});
|
||||
} catch (err) {
|
||||
// Nothing was generated — refund the whole reservation rather than bill an
|
||||
// invented figure. Best-effort: a failed refund must not mask the 503.
|
||||
await reconcileSpend(userId, reservation.reserved, 0, dateKey, 0, {
|
||||
servedBy: null,
|
||||
reservedOwner: reservation.owner,
|
||||
}).catch(() => {});
|
||||
// Nothing was generated — refund the reservation rather than bill an
|
||||
// invented figure, keeping only what the failed hops reported spending
|
||||
// (#1279). Best-effort: a failed refund must not mask the 503.
|
||||
await reconcileSpend(
|
||||
userId,
|
||||
reservation.reserved,
|
||||
spentOnFailure(err),
|
||||
dateKey,
|
||||
0,
|
||||
{ servedBy: null, reservedOwner: reservation.owner },
|
||||
).catch(() => {});
|
||||
if (err instanceof AllProvidersFailedError) {
|
||||
annotate({
|
||||
action: { name: "medications.extract.provider-failed" },
|
||||
@@ -296,12 +296,12 @@ async function handleExtract(request: NextRequest): Promise<Response> {
|
||||
throw err;
|
||||
}
|
||||
|
||||
// Reconcile against what the provider actually reported. This runs for the
|
||||
// unusable-reply branches too: those tokens were burned upstream even though
|
||||
// the body could not be used, so they stay on the ledger rather than funding
|
||||
// a free retry loop. Falls back to the reservation when the provider reports
|
||||
// no count, so an unreported generation is never billed as zero.
|
||||
const actualTokens = completion.result.tokensUsed ?? reservation.reserved;
|
||||
// What the provider reported, plus (#1279) what the hops that failed before
|
||||
// it spent; for the unusable-reply branches too, since those tokens were
|
||||
// burned upstream. No count falls back to the reservation, never to zero.
|
||||
const actualTokens =
|
||||
(completion.result.tokensUsed ?? reservation.reserved) +
|
||||
completion.tokensSpent;
|
||||
await reconcileSpend(userId, reservation.reserved, actualTokens, dateKey, 0, {
|
||||
servedBy: completion.workingProvider.providerType,
|
||||
reservedOwner: reservation.owner,
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
/**
|
||||
* GET / PATCH the caller's preview-feature opt-ins.
|
||||
*
|
||||
* GET answers every flag, off unless the caller switched it on. PATCH sets
|
||||
* the flags the body names and keeps the rest; unknown flags are refused
|
||||
* with the multi-issue 422 so a typo cannot silently do nothing. The blob
|
||||
* lives on its own `User` column (`preview_features_json`).
|
||||
*
|
||||
* The opt-ins are the caller's own, so this is an actor surface
|
||||
* (`requireActorAuth`): a browser acting for another record keeps reaching
|
||||
* the person's own switch. The PATCH reads and writes under a row lock, so two
|
||||
* flags switched at once both stay switched.
|
||||
*/
|
||||
import { apiHandler, requireActorAuth } from "@/lib/api-handler";
|
||||
import { apiSuccess } from "@/lib/api-response";
|
||||
import { parseJsonBody } from "@/lib/api/body";
|
||||
import { prisma, toJson } from "@/lib/db";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import {
|
||||
PREVIEW_FEATURE_KEYS,
|
||||
resolvePreviewFeatures,
|
||||
type PreviewFeatures,
|
||||
} from "@/lib/settings/preview-features";
|
||||
import { previewFeaturesPatchSchema } from "@/lib/validations/settings-prefs";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
async function readStored(
|
||||
userId: string,
|
||||
db: Pick<typeof prisma, "user"> = prisma,
|
||||
): Promise<unknown> {
|
||||
const row = await db.user.findUnique({
|
||||
where: { id: userId },
|
||||
select: { previewFeaturesJson: true },
|
||||
});
|
||||
return row?.previewFeaturesJson;
|
||||
}
|
||||
|
||||
function enabledFlags(features: PreviewFeatures): string[] {
|
||||
return PREVIEW_FEATURE_KEYS.filter((key) => features[key]);
|
||||
}
|
||||
|
||||
export const GET = apiHandler(async () => {
|
||||
const { user } = await requireActorAuth();
|
||||
const features = resolvePreviewFeatures(await readStored(user.id));
|
||||
annotate({
|
||||
action: { name: "settings.preview-features.read" },
|
||||
meta: { enabled: enabledFlags(features) },
|
||||
});
|
||||
return apiSuccess(features);
|
||||
});
|
||||
|
||||
export const PATCH = apiHandler(async (request: Request) => {
|
||||
const { user } = await requireActorAuth();
|
||||
|
||||
const parsed = await parseJsonBody(request, previewFeaturesPatchSchema, {
|
||||
maxBytes: 1024,
|
||||
action: "settings.preview-features.validation-failed",
|
||||
});
|
||||
if (!parsed.ok) return parsed.error;
|
||||
|
||||
const next = await prisma.$transaction(async (tx) => {
|
||||
await tx.$queryRaw`SELECT 1 FROM "users" WHERE "id" = ${user.id} FOR UPDATE`;
|
||||
const current = resolvePreviewFeatures(await readStored(user.id, tx));
|
||||
const merged: PreviewFeatures = {
|
||||
topics: parsed.data.topics ?? current.topics,
|
||||
};
|
||||
await tx.user.update({
|
||||
where: { id: user.id },
|
||||
data: { previewFeaturesJson: toJson(merged) },
|
||||
});
|
||||
return merged;
|
||||
});
|
||||
|
||||
annotate({
|
||||
action: { name: "settings.preview-features.update" },
|
||||
meta: { enabled: enabledFlags(next) },
|
||||
});
|
||||
return apiSuccess(next);
|
||||
});
|
||||
@@ -22,6 +22,8 @@ vi.mock("@/lib/modules/gate", () => ({ isModuleEnabled: vi.fn() }));
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
measurement: { findMany: vi.fn() },
|
||||
// The mirror walk's token lookup: no token switched, so no walk.
|
||||
apiToken: { findMany: vi.fn(async () => []) },
|
||||
moodEntry: { findMany: vi.fn() },
|
||||
medicationIntakeEvent: { findMany: vi.fn() },
|
||||
cycleDayLog: { findMany: vi.fn() },
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
/**
|
||||
* #1126 / #1279 — a gateway or local entry whose own reasoning setting the
|
||||
* endpoint refuses.
|
||||
*
|
||||
* The entry's setting (`reasoningEffort`, stamped by the resolver) rides a
|
||||
* call that carries no level of its own. An endpoint that refuses the field
|
||||
* (LiteLLM in front of Ollama) answered every such call with a 400 that was
|
||||
* never learned, because only the call's own level counted as "sent": every
|
||||
* call failed. It is learned now like any refused field: the call is asked
|
||||
* once more without it, inside its budget of two requests, later calls send
|
||||
* no field, and the lesson is reported once.
|
||||
*
|
||||
* Driven through the real clients against a loopback server.
|
||||
*/
|
||||
import http from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import {
|
||||
afterAll,
|
||||
afterEach,
|
||||
beforeAll,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from "vitest";
|
||||
|
||||
const emitSignal = vi.hoisted(() => vi.fn());
|
||||
vi.mock("@/lib/logging/signal", () => ({
|
||||
emitSignal: (...a: unknown[]) => emitSignal(...a),
|
||||
}));
|
||||
|
||||
import { LocalOpenAICompatibleClient } from "../local-client";
|
||||
import { OpenAIClient } from "../openai-client";
|
||||
import { resetJsonModeDialectCache } from "../json-dialect";
|
||||
import { resetReasoningDialectCache } from "../reasoning/dialect-cache";
|
||||
import { singleUserTurn } from "../types";
|
||||
import { _resetPinnedDispatcherForTests } from "@/lib/safe-fetch-dispatcher";
|
||||
|
||||
const ANSWER = '{"summary":"Steady."}';
|
||||
|
||||
let server: http.Server;
|
||||
let origin = "";
|
||||
let seen: unknown[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
server = http.createServer((req, res) => {
|
||||
let raw = "";
|
||||
req.on("data", (c) => (raw += c));
|
||||
req.on("end", () => {
|
||||
const body = JSON.parse(raw) as Record<string, unknown>;
|
||||
seen.push(body.reasoning_effort);
|
||||
if (body.reasoning_effort !== undefined) {
|
||||
res.writeHead(400, { "Content-Type": "application/json" });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: {
|
||||
message:
|
||||
"litellm.UnsupportedParamsError: ollama does not support parameters: ['reasoning_effort']",
|
||||
},
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
message: { role: "assistant", content: ANSWER },
|
||||
finish_reason: "stop",
|
||||
},
|
||||
],
|
||||
usage: { total_tokens: 400 },
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => server.listen(0, "127.0.0.1", r));
|
||||
origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
_resetPinnedDispatcherForTests();
|
||||
await new Promise<void>((r) => server.close(() => r()));
|
||||
});
|
||||
|
||||
const savedOrigins = process.env.AI_PRIVATE_ORIGINS;
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.AI_PRIVATE_ORIGINS = origin;
|
||||
seen = [];
|
||||
emitSignal.mockClear();
|
||||
resetReasoningDialectCache();
|
||||
resetJsonModeDialectCache();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (savedOrigins === undefined) delete process.env.AI_PRIVATE_ORIGINS;
|
||||
else process.env.AI_PRIVATE_ORIGINS = savedOrigins;
|
||||
});
|
||||
|
||||
function gateway() {
|
||||
const client = new OpenAIClient({
|
||||
apiKey: "",
|
||||
model: "qwen3",
|
||||
baseUrl: `${origin}/v1`,
|
||||
providerType: "openai-compatible",
|
||||
});
|
||||
client.reasoningEffort = "high";
|
||||
return client;
|
||||
}
|
||||
|
||||
function local() {
|
||||
const client = new LocalOpenAICompatibleClient({
|
||||
baseUrl: `${origin}/v1`,
|
||||
model: "qwen3",
|
||||
});
|
||||
client.reasoningEffort = "high";
|
||||
return client;
|
||||
}
|
||||
|
||||
/** A status card call: no level of its own. */
|
||||
const params = () =>
|
||||
singleUserTurn({
|
||||
system: "s",
|
||||
user: "u",
|
||||
maxTokens: 300,
|
||||
timeoutMs: 30_000,
|
||||
responseFormat: "json",
|
||||
});
|
||||
|
||||
describe.each([
|
||||
["the gateway", gateway],
|
||||
["the Local client", local],
|
||||
] as const)("%s, an entry setting the endpoint refuses", (_, make) => {
|
||||
it("answers on the second request, and later calls send no field", async () => {
|
||||
const first = await make().generateCompletion(params());
|
||||
expect(first.content).toBe(ANSWER);
|
||||
expect(seen).toEqual(["high", undefined]);
|
||||
|
||||
seen = [];
|
||||
for (let i = 0; i < 3; i += 1) {
|
||||
expect((await make().generateCompletion(params())).content).toBe(ANSWER);
|
||||
}
|
||||
expect(seen).toEqual([undefined, undefined, undefined]);
|
||||
});
|
||||
|
||||
it("says so once, not once per call", async () => {
|
||||
for (let i = 0; i < 3; i += 1) await make().generateCompletion(params());
|
||||
const lines = emitSignal.mock.calls
|
||||
.map((c) => c[0] as { action: string })
|
||||
.filter((s) => s.action === "ai.provider.reasoning_field_refused");
|
||||
expect(lines).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -232,7 +232,7 @@ describe("Local reasoning replies", () => {
|
||||
expect(result.reasoning?.summary).toEqual(["The user asks about sleep."]);
|
||||
});
|
||||
|
||||
it("never answers with unlabelled text that filled the reasoning ceiling", async () => {
|
||||
it("never answers with unlabelled text that filled the reasoning ceiling, once the model reasons", async () => {
|
||||
const cut = async () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
@@ -245,15 +245,47 @@ describe("Local reasoning replies", () => {
|
||||
}),
|
||||
{ status: 200 },
|
||||
);
|
||||
const mock = vi.fn().mockImplementation(cut);
|
||||
// The model has answered with a reasoning field before.
|
||||
const mock = vi
|
||||
.fn()
|
||||
.mockImplementationOnce(
|
||||
buffered({ content: "ok", reasoning: "Thinking first" }),
|
||||
)
|
||||
.mockImplementation(cut);
|
||||
vi.stubGlobal("fetch", mock);
|
||||
await expect(
|
||||
client().generateCompletion(
|
||||
params({ reasoning: { effort: "medium", summaries: true } }),
|
||||
),
|
||||
).rejects.toMatchObject({ httpStatus: 0, kind: "reasoning_only" });
|
||||
const c = client();
|
||||
const wanted = params({ reasoning: { effort: "medium", summaries: true } });
|
||||
expect((await c.generateCompletion(wanted)).content).toBe("ok");
|
||||
await expect(c.generateCompletion(wanted)).rejects.toMatchObject({
|
||||
httpStatus: 0,
|
||||
kind: "reasoning_only",
|
||||
});
|
||||
// Asked once more with `none` (#1267), which this model ignores.
|
||||
expect(mock).toHaveBeenCalledTimes(2);
|
||||
expect(mock).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("keeps a cut answer from a model never seen reasoning, whatever the level", async () => {
|
||||
const mock = vi.fn().mockImplementation(
|
||||
async () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
choices: [
|
||||
{
|
||||
message: { content: '{"summary": "Steady and' },
|
||||
finish_reason: "length",
|
||||
},
|
||||
],
|
||||
}),
|
||||
{ status: 200 },
|
||||
),
|
||||
);
|
||||
vi.stubGlobal("fetch", mock);
|
||||
const result = await client().generateCompletion(
|
||||
params({ reasoning: { effort: "medium", summaries: true } }),
|
||||
);
|
||||
expect(result.content).toBe('{"summary": "Steady and');
|
||||
expect(result.finishReason).toBe("length");
|
||||
expect(mock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps a cut answer when the ceiling held no room for reasoning", async () => {
|
||||
|
||||
@@ -111,6 +111,19 @@ describe("#1279: background work and a benched provider", () => {
|
||||
]);
|
||||
}
|
||||
expect(gateway.calls).toBe(HARD_FAILURE_SKIP_THRESHOLD);
|
||||
// The bench sent nothing; a hop that reached the provider did.
|
||||
const benched = await walk(
|
||||
"job",
|
||||
"u-bench",
|
||||
"openai-compatible",
|
||||
gateway,
|
||||
ledger,
|
||||
).then(
|
||||
() => null,
|
||||
(e: unknown) => e,
|
||||
);
|
||||
expect(benched).toBeInstanceOf(AllProvidersFailedError);
|
||||
expect((benched as AllProvidersFailedError).sentNothing).toBe(true);
|
||||
// One line for the window, not one per job.
|
||||
expect(signals("ai.chain.background_paused")).toHaveLength(1);
|
||||
expect(signals("ai.chain.background_paused")[0].meta).toMatchObject({
|
||||
@@ -139,6 +152,30 @@ describe("#1279: background work and a benched provider", () => {
|
||||
expect((err as AllProvidersFailedError).attempts[0].failureKind).toBe(
|
||||
"reasoning_only",
|
||||
);
|
||||
expect((err as AllProvidersFailedError).sentNothing).toBe(false);
|
||||
});
|
||||
|
||||
it("walks the benched provider for a job-rationed call a person waits on", async () => {
|
||||
const ledger = createInMemoryProviderHealthLedger();
|
||||
const gateway = new AnswerlessProvider();
|
||||
for (let i = 0; i < PROVIDER_PAUSE_THRESHOLD; i += 1) {
|
||||
await walk("job", "u-rerun", "openai-compatible", gateway, ledger).catch(
|
||||
() => undefined,
|
||||
);
|
||||
}
|
||||
const before = gateway.calls;
|
||||
const err = await runRawCompletionWithFallback({
|
||||
userId: "u-rerun",
|
||||
surface: "job",
|
||||
interactive: true,
|
||||
providers: [{ providerType: "openai-compatible", instance: gateway }],
|
||||
params: singleUserTurn({ system: "s", user: "u" }),
|
||||
ledger,
|
||||
}).catch((e) => e);
|
||||
expect(gateway.calls).toBe(before + 1);
|
||||
expect((err as AllProvidersFailedError).attempts[0].failureKind).toBe(
|
||||
"reasoning_only",
|
||||
);
|
||||
});
|
||||
|
||||
it("logs once more when a new window opens", async () => {
|
||||
@@ -187,6 +224,36 @@ describe("#1279: background work and a benched provider", () => {
|
||||
expect(local.calls).toBe(PROVIDER_PAUSE_THRESHOLD);
|
||||
});
|
||||
|
||||
it("reports what the hops before the answering one spent", async () => {
|
||||
const ledger = createInMemoryProviderHealthLedger();
|
||||
const out = await runRawCompletionWithFallback({
|
||||
userId: "u-spent-then-answered",
|
||||
surface: "job",
|
||||
providers: [
|
||||
{
|
||||
providerType: "openai-compatible",
|
||||
instance: new AnswerlessProvider(),
|
||||
},
|
||||
{
|
||||
providerType: "anthropic",
|
||||
instance: {
|
||||
type: "anthropic",
|
||||
generateCompletion: async () => ({
|
||||
content: "ok",
|
||||
tokensUsed: 100,
|
||||
model: "m",
|
||||
providerType: "anthropic",
|
||||
}),
|
||||
} as AIProvider,
|
||||
},
|
||||
],
|
||||
params: singleUserTurn({ system: "s", user: "u" }),
|
||||
ledger,
|
||||
});
|
||||
expect(out.result.tokensUsed).toBe(100);
|
||||
expect(out.tokensSpent).toBe(2700);
|
||||
});
|
||||
|
||||
it("sums what the failed hops reported spending", async () => {
|
||||
const ledger = createInMemoryProviderHealthLedger();
|
||||
const err = await runRawCompletionWithFallback({
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
/**
|
||||
* #1279 — one completion call sends at most two requests, whatever the
|
||||
* endpoint answers.
|
||||
*
|
||||
* A gateway in front of Ollama (LiteLLM: "ollama does not support
|
||||
* parameters: ['reasoning_effort']") refuses the reasoning field outright,
|
||||
* and the model behind it reasons unasked and answers nothing. The answerless
|
||||
* rule asked again with `none`, the refusal rule wrote "send no field" over
|
||||
* that lesson, the next request was answerless again, and so on: one call
|
||||
* sent requests until something outside stopped it.
|
||||
*
|
||||
* Driven through the real clients, the real `safeFetch` and the pinned
|
||||
* dispatcher against a loopback server that answers each request by a
|
||||
* script, so every combination of refusal and answerless reply is walked.
|
||||
*/
|
||||
import http from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import {
|
||||
afterAll,
|
||||
afterEach,
|
||||
beforeAll,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from "vitest";
|
||||
|
||||
vi.mock("@/lib/logging/signal", () => ({ emitSignal: () => {} }));
|
||||
|
||||
import { LocalOpenAICompatibleClient } from "../local-client";
|
||||
import { OpenAIClient } from "../openai-client";
|
||||
import { resetJsonModeDialectCache } from "../json-dialect";
|
||||
import { resetReasoningDialectCache } from "../reasoning/dialect-cache";
|
||||
import { MAX_REQUESTS_PER_CALL } from "../request-hops";
|
||||
import { singleUserTurn, type CompletionParams } from "../types";
|
||||
import { _resetPinnedDispatcherForTests } from "@/lib/safe-fetch-dispatcher";
|
||||
|
||||
interface Seen {
|
||||
effort: unknown;
|
||||
json: boolean;
|
||||
stream: boolean;
|
||||
}
|
||||
|
||||
/** Decides one reply from what the request carried. */
|
||||
type Script = (req: Seen) => { status: number; body: unknown };
|
||||
|
||||
const REFUSE_REASONING = {
|
||||
status: 400,
|
||||
body: {
|
||||
error: {
|
||||
message:
|
||||
"litellm.UnsupportedParamsError: ollama does not support parameters: ['reasoning_effort']",
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const REFUSE_JSON = {
|
||||
status: 400,
|
||||
body: { error: { message: "Unsupported parameter: response_format" } },
|
||||
};
|
||||
|
||||
function answerless() {
|
||||
return {
|
||||
status: 200,
|
||||
body: {
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: null,
|
||||
reasoning_content: "thinking ".repeat(50),
|
||||
},
|
||||
finish_reason: "length",
|
||||
},
|
||||
],
|
||||
usage: { total_tokens: 1000 },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Refuses the reasoning field, reasons without answering otherwise. */
|
||||
const gatewayRefusingTheField: Script = (req) =>
|
||||
req.effort !== undefined ? REFUSE_REASONING : answerless();
|
||||
|
||||
/** Refuses the JSON flag and the reasoning field, answerless otherwise. */
|
||||
const refusesEverything: Script = (req) =>
|
||||
req.json
|
||||
? REFUSE_JSON
|
||||
: req.effort !== undefined
|
||||
? REFUSE_REASONING
|
||||
: answerless();
|
||||
|
||||
/** Refuses every streamed request and the reasoning field. */
|
||||
const refusesStreams: Script = (req) =>
|
||||
req.stream
|
||||
? { status: 400, body: { error: { message: "no stream" } } }
|
||||
: gatewayRefusingTheField(req);
|
||||
|
||||
let server: http.Server;
|
||||
let origin = "";
|
||||
let script: Script = gatewayRefusingTheField;
|
||||
let seen: Seen[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
server = http.createServer((req, res) => {
|
||||
let raw = "";
|
||||
req.on("data", (c) => (raw += c));
|
||||
req.on("end", () => {
|
||||
if (!raw) {
|
||||
res.writeHead(404);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
const body = JSON.parse(raw) as Record<string, unknown>;
|
||||
const request: Seen = {
|
||||
effort: body.reasoning_effort,
|
||||
json: body.response_format !== undefined,
|
||||
stream: body.stream === true,
|
||||
};
|
||||
seen.push(request);
|
||||
// A backstop for the unbounded case, so a failing run ends.
|
||||
const reply =
|
||||
seen.length > 40
|
||||
? { status: 500, body: { error: { message: "stop" } } }
|
||||
: script(request);
|
||||
res.writeHead(reply.status, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify(reply.body));
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => server.listen(0, "127.0.0.1", r));
|
||||
origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
_resetPinnedDispatcherForTests();
|
||||
await new Promise<void>((r) => server.close(() => r()));
|
||||
});
|
||||
|
||||
const savedOrigins = process.env.AI_PRIVATE_ORIGINS;
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.AI_PRIVATE_ORIGINS = origin;
|
||||
seen = [];
|
||||
resetReasoningDialectCache();
|
||||
resetJsonModeDialectCache();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (savedOrigins === undefined) delete process.env.AI_PRIVATE_ORIGINS;
|
||||
else process.env.AI_PRIVATE_ORIGINS = savedOrigins;
|
||||
});
|
||||
|
||||
function gateway() {
|
||||
return new OpenAIClient({
|
||||
apiKey: "",
|
||||
model: "qwen3",
|
||||
baseUrl: `${origin}/v1`,
|
||||
providerType: "openai-compatible",
|
||||
});
|
||||
}
|
||||
|
||||
function local() {
|
||||
return new LocalOpenAICompatibleClient({
|
||||
baseUrl: `${origin}/v1`,
|
||||
model: "qwen3",
|
||||
});
|
||||
}
|
||||
|
||||
function params(extra: Partial<CompletionParams> = {}): CompletionParams {
|
||||
return {
|
||||
...singleUserTurn({
|
||||
system: "s",
|
||||
user: "u",
|
||||
maxTokens: 300,
|
||||
timeoutMs: 30_000,
|
||||
responseFormat: "json",
|
||||
}),
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
type Call = (p: CompletionParams) => Promise<unknown>;
|
||||
|
||||
const CALLERS: ReadonlyArray<[string, () => Call]> = [
|
||||
["the gateway", () => (p) => gateway().generateCompletion(p)],
|
||||
["the Local client", () => (p) => local().generateCompletion(p)],
|
||||
[
|
||||
"the Local client's stream",
|
||||
() => (p) => local().generateCompletionStream(p, () => {}),
|
||||
],
|
||||
];
|
||||
|
||||
/** Run `calls` calls in a row; the requests each one sent. */
|
||||
async function requestsPerCall(
|
||||
call: Call,
|
||||
calls: number,
|
||||
extra: Partial<CompletionParams> = {},
|
||||
) {
|
||||
const out: Array<{
|
||||
requests: Seen[];
|
||||
error: Record<string, unknown> | null;
|
||||
}> = [];
|
||||
for (let i = 0; i < calls; i += 1) {
|
||||
const before = seen.length;
|
||||
const error = await call(params(extra)).then(
|
||||
() => null,
|
||||
(e: unknown) => e as Record<string, unknown>,
|
||||
);
|
||||
out.push({ requests: seen.slice(before), error });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
describe("one call, at most two requests", () => {
|
||||
it("the budget is two", () => {
|
||||
expect(MAX_REQUESTS_PER_CALL).toBe(2);
|
||||
});
|
||||
|
||||
describe.each(CALLERS)("%s", (_, make) => {
|
||||
it.each([
|
||||
["a gateway that refuses the reasoning field", gatewayRefusingTheField],
|
||||
["an endpoint that refuses the JSON flag too", refusesEverything],
|
||||
["a server that refuses streams", refusesStreams],
|
||||
] as const)("%s", async (_name, s) => {
|
||||
script = s;
|
||||
for (const extra of [
|
||||
{},
|
||||
{ reasoning: { effort: "low" as const, summaries: false } },
|
||||
{ reasoning: { effort: "off" as const, summaries: false } },
|
||||
]) {
|
||||
resetReasoningDialectCache();
|
||||
resetJsonModeDialectCache();
|
||||
const calls = await requestsPerCall(make(), 6, extra);
|
||||
for (const { requests } of calls) {
|
||||
expect(requests.length).toBeLessThanOrEqual(MAX_REQUESTS_PER_CALL);
|
||||
expect(requests.length).toBeGreaterThan(0);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("stops asking with `none` once the endpoint refused it, and fails as the reasoning it is", async () => {
|
||||
script = gatewayRefusingTheField;
|
||||
const calls = await requestsPerCall(make(), 4);
|
||||
// The first call: no field, answerless; asked again with `none`,
|
||||
// refused. Every later call sends no field and one request.
|
||||
expect(calls[0].requests.map((r) => r.effort)).toEqual([
|
||||
undefined,
|
||||
"none",
|
||||
]);
|
||||
for (const later of calls.slice(1)) {
|
||||
expect(later.requests.map((r) => r.effort)).toEqual([undefined]);
|
||||
}
|
||||
for (const { error } of calls) {
|
||||
expect(error?.kind).toBe("reasoning_only");
|
||||
expect(error?.httpStatus).toBe(0);
|
||||
}
|
||||
// The first call's answerless request is what it cost.
|
||||
expect(calls[0].error?.tokensUsed).toBe(1000);
|
||||
});
|
||||
|
||||
it("never asks a model with `none` behind an endpoint that refused the field", async () => {
|
||||
script = gatewayRefusingTheField;
|
||||
// A call with a level learns that the field is refused first.
|
||||
const first = await requestsPerCall(make(), 1, {
|
||||
reasoning: { effort: "low", summaries: false },
|
||||
});
|
||||
expect(first[0].requests.map((r) => r.effort)).toEqual([
|
||||
"low",
|
||||
undefined,
|
||||
]);
|
||||
// Its answerless repeat taught that the model reasons unasked; no call
|
||||
// after it sends a reasoning field again.
|
||||
const later = await requestsPerCall(make(), 3);
|
||||
for (const { requests } of later) {
|
||||
expect(requests.map((r) => r.effort)).toEqual([undefined]);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,178 @@
|
||||
/**
|
||||
* #1276 — a reply that ran into the token ceiling with unlabelled text is the
|
||||
* model's answer, cut off, unless this endpoint and model have shown that
|
||||
* they reason.
|
||||
*
|
||||
* The rule read every such reply as reasoning the server did not separate,
|
||||
* whenever the ceiling held room for reasoning: a call carrying any level
|
||||
* (Off included) to a model that does not reason at all (llama3.1) lost its
|
||||
* own truncated answer. The briefing never reached its JSON-shape retry, the
|
||||
* Coach named the wrong error, the failure counted toward the bench, and the
|
||||
* client learned to ask with `none` for an hour. Now the text is reasoning
|
||||
* only once the model has shown it reasons (a reasoning field, a think tag,
|
||||
* or a lesson that says so); otherwise it is the answer with
|
||||
* `finishReason: "length"`.
|
||||
*
|
||||
* Driven through the real clients against a loopback server.
|
||||
*/
|
||||
import http from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import {
|
||||
afterAll,
|
||||
afterEach,
|
||||
beforeAll,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from "vitest";
|
||||
|
||||
vi.mock("@/lib/logging/signal", () => ({ emitSignal: () => {} }));
|
||||
|
||||
import { LocalOpenAICompatibleClient } from "../local-client";
|
||||
import { OpenAIClient } from "../openai-client";
|
||||
import { resetJsonModeDialectCache } from "../json-dialect";
|
||||
import {
|
||||
learnedReasoningDialect,
|
||||
resetReasoningDialectCache,
|
||||
} from "../reasoning/dialect-cache";
|
||||
import { singleUserTurn, type CompletionParams } from "../types";
|
||||
import { _resetPinnedDispatcherForTests } from "@/lib/safe-fetch-dispatcher";
|
||||
|
||||
const CUT = '{"summary": "Your weight has been stable over the last weeks and';
|
||||
|
||||
/** What the next replies carry, one per request; the last one repeats. */
|
||||
let replies: Array<Record<string, unknown>> = [];
|
||||
let seen: unknown[] = [];
|
||||
let server: http.Server;
|
||||
let origin = "";
|
||||
|
||||
function cut() {
|
||||
return { message: { role: "assistant", content: CUT }, finish: "length" };
|
||||
}
|
||||
|
||||
function reasonedAnswer() {
|
||||
return {
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: '{"summary":"Steady."}',
|
||||
reasoning_content: "The readings are flat.",
|
||||
},
|
||||
finish: "stop",
|
||||
};
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
server = http.createServer((req, res) => {
|
||||
let raw = "";
|
||||
req.on("data", (c) => (raw += c));
|
||||
req.on("end", () => {
|
||||
const body = JSON.parse(raw) as Record<string, unknown>;
|
||||
seen.push(body.reasoning_effort);
|
||||
const next = (replies.length > 1 ? replies.shift() : replies[0]) as {
|
||||
message: unknown;
|
||||
finish: string;
|
||||
};
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
choices: [
|
||||
{ index: 0, message: next.message, finish_reason: next.finish },
|
||||
],
|
||||
usage: { total_tokens: 3000 },
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => server.listen(0, "127.0.0.1", r));
|
||||
origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
_resetPinnedDispatcherForTests();
|
||||
await new Promise<void>((r) => server.close(() => r()));
|
||||
});
|
||||
|
||||
const savedOrigins = process.env.AI_PRIVATE_ORIGINS;
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.AI_PRIVATE_ORIGINS = origin;
|
||||
seen = [];
|
||||
replies = [];
|
||||
resetReasoningDialectCache();
|
||||
resetJsonModeDialectCache();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (savedOrigins === undefined) delete process.env.AI_PRIVATE_ORIGINS;
|
||||
else process.env.AI_PRIVATE_ORIGINS = savedOrigins;
|
||||
});
|
||||
|
||||
function params(effort: "off" | "low"): CompletionParams {
|
||||
return {
|
||||
...singleUserTurn({
|
||||
system: "s",
|
||||
user: "u",
|
||||
maxTokens: 2500,
|
||||
timeoutMs: 30_000,
|
||||
responseFormat: "json",
|
||||
}),
|
||||
reasoning: { effort, summaries: false },
|
||||
};
|
||||
}
|
||||
|
||||
describe.each([
|
||||
[
|
||||
"the gateway",
|
||||
() =>
|
||||
new OpenAIClient({
|
||||
apiKey: "",
|
||||
model: "llama3.1",
|
||||
baseUrl: `${origin}/v1`,
|
||||
providerType: "openai-compatible",
|
||||
}),
|
||||
"openai",
|
||||
],
|
||||
[
|
||||
"the Local client",
|
||||
() =>
|
||||
new LocalOpenAICompatibleClient({
|
||||
baseUrl: `${origin}/v1`,
|
||||
model: "llama3.1",
|
||||
}),
|
||||
"local",
|
||||
],
|
||||
] as const)("%s", (_, make, provider) => {
|
||||
const key = () => ({
|
||||
provider,
|
||||
endpoint: `${origin}/v1`,
|
||||
model: "llama3.1",
|
||||
});
|
||||
|
||||
it.each(["off", "low"] as const)(
|
||||
"returns a model's cut answer as the answer (level %s), and learns nothing",
|
||||
async (effort) => {
|
||||
replies = [cut()];
|
||||
for (let i = 0; i < 2; i += 1) {
|
||||
const result = await make().generateCompletion(params(effort));
|
||||
expect(result.content).toBe(CUT);
|
||||
expect(result.finishReason).toBe("length");
|
||||
}
|
||||
// One request per call, each sent as the call asked.
|
||||
const asked = effort === "off" ? "none" : "low";
|
||||
expect(seen).toEqual([asked, asked]);
|
||||
expect(learnedReasoningDialect(key())).toBeUndefined();
|
||||
},
|
||||
);
|
||||
|
||||
it("reads the same cut text as reasoning once the model has shown it reasons", async () => {
|
||||
replies = [reasonedAnswer(), cut()];
|
||||
const first = await make().generateCompletion(params("low"));
|
||||
expect(first.content).toBe('{"summary":"Steady."}');
|
||||
const err = await make()
|
||||
.generateCompletion(params("low"))
|
||||
.catch((e: unknown) => e as Record<string, unknown>);
|
||||
expect(err).toMatchObject({ httpStatus: 0, kind: "reasoning_only" });
|
||||
});
|
||||
});
|
||||
@@ -7,7 +7,9 @@
|
||||
* times in a row: a chain whose only entry answered every call with reasoning
|
||||
* and no answer was called by every background job, all day, because the tail
|
||||
* is still walked. So a job leaves every benched provider out until its
|
||||
* window lifts, and the window grows while the failures continue
|
||||
* window lifts (a job-rationed call a person waits on, the on-demand
|
||||
* briefing, passes `interactive` and walks it at the tail like any
|
||||
* interactive call), and the window grows while the failures continue
|
||||
* (`hardFailureCooldownMs` in `provider-health-ledger.ts`). The ledger is
|
||||
* durable, so the bench holds across workers and restarts.
|
||||
*
|
||||
|
||||
@@ -209,6 +209,7 @@ async function turn(level: ReasoningLevel) {
|
||||
result: await client.generateCompletion(params),
|
||||
workingProvider: { providerType: "openai-compatible" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
}),
|
||||
);
|
||||
return runCoachToolLoop({
|
||||
@@ -268,11 +269,20 @@ describe("#1276: a gateway model that always reasons", () => {
|
||||
reasoning: { effort: "medium", summaries: true },
|
||||
})
|
||||
.then(
|
||||
(r) => ({ ok: true as const, content: r.content }),
|
||||
(r) => ({ ok: true as const, finishReason: r.finishReason }),
|
||||
(e: { kind?: string }) => ({ ok: false as const, kind: e.kind }),
|
||||
);
|
||||
// Never the thought as an answer.
|
||||
expect(first).toEqual({ ok: false, kind: "reasoning_only" });
|
||||
// Never the thought as an answer, once it is labelled as one. The
|
||||
// template-opened reply carries no label at all (no field, no tag) and
|
||||
// comes from a model not yet seen reasoning: text cut off at the
|
||||
// ceiling from such a model is its answer, cut off (a model that does
|
||||
// not reason, asked with any level, must keep its answer), so it comes
|
||||
// back as the answer with `finishReason: "length"`.
|
||||
expect(first).toEqual(
|
||||
mode === "template-opened"
|
||||
? { ok: true, finishReason: "length" }
|
||||
: { ok: false, kind: "reasoning_only" },
|
||||
);
|
||||
|
||||
// The next turn reasons a normal amount.
|
||||
reasoningWords = 1200;
|
||||
|
||||
@@ -61,6 +61,7 @@ function round(
|
||||
},
|
||||
workingProvider: { providerType: "codex" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ function round(content: string, toolCalls?: Call[]) {
|
||||
},
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -98,6 +98,7 @@ async function capTurn(
|
||||
result: await client.generateCompletion({ ...params, ...extra }),
|
||||
workingProvider: { providerType },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
}),
|
||||
);
|
||||
return runCoachToolLoop({
|
||||
|
||||
@@ -65,6 +65,7 @@ describe("coach tool loop — which results reach the grounding guard", () => {
|
||||
},
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
result: {
|
||||
@@ -76,6 +77,7 @@ describe("coach tool loop — which results reach the grounding guard", () => {
|
||||
},
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const loop = await runCoachToolLoop(baseArgs);
|
||||
@@ -110,6 +112,7 @@ describe("coach tool loop — which results reach the grounding guard", () => {
|
||||
},
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
result: {
|
||||
@@ -121,6 +124,7 @@ describe("coach tool loop — which results reach the grounding guard", () => {
|
||||
},
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const loop = await runCoachToolLoop(baseArgs);
|
||||
|
||||
@@ -42,6 +42,7 @@ function completion(opts: {
|
||||
},
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ function round(content: string, toolCalls?: unknown[]) {
|
||||
},
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -401,7 +401,7 @@ export async function runCoachToolLoop(args: {
|
||||
});
|
||||
if (spend) {
|
||||
await spend.settleRound({
|
||||
tokens: result.tokensUsed ?? 0,
|
||||
tokens: (result.tokensUsed ?? 0) + fallback.tokensSpent,
|
||||
cachedTokens: result.cachedInputTokens ?? 0,
|
||||
servedBy: fallback.workingProvider.providerType as ProviderChainType,
|
||||
final: isFinal,
|
||||
|
||||
@@ -106,6 +106,14 @@ describe("reserveTurnBudget and the turn ledger", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps what a failed call reported spending when the turn closes (#1279)", async () => {
|
||||
const l = await ledger();
|
||||
await l.close(2_700);
|
||||
expect(b.reconcileSpend.mock.calls.map((c) => [c[1], c[2]])).toEqual([
|
||||
[9_000, 2_700],
|
||||
]);
|
||||
});
|
||||
|
||||
it("settles a forced final round against the final reserve", async () => {
|
||||
const l = await ledger();
|
||||
await l.settleRound({
|
||||
|
||||
@@ -41,8 +41,10 @@ export interface TurnLedger extends RoundSpend {
|
||||
/**
|
||||
* Give back everything still reserved and not settled. Called once the
|
||||
* turn is over, whatever way it ended; a second call does nothing.
|
||||
* #1279 — `spent`, what a failed provider call reported spending, stays
|
||||
* charged: a model that reasons without answering is billed for it.
|
||||
*/
|
||||
close(): Promise<void>;
|
||||
close(spent?: number): Promise<void>;
|
||||
}
|
||||
|
||||
export async function reserveTurnBudget(args: {
|
||||
@@ -193,13 +195,13 @@ export function createTurnLedger(args: {
|
||||
}
|
||||
await settle(portion, tokens, cachedTokens, servedBy);
|
||||
},
|
||||
async close() {
|
||||
async close(spent = 0) {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
const unused = open.reduce((sum, n) => sum + n, 0) + finalHeld;
|
||||
open.length = 0;
|
||||
finalHeld = 0;
|
||||
if (unused > 0) await settle(unused, 0, 0, null);
|
||||
if (unused > 0 || spent > 0) await settle(unused, spent, 0, null);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
AllProvidersFailedError,
|
||||
runStreamingRawCompletionWithFallback,
|
||||
} from "@/lib/ai/provider-runner";
|
||||
import { spentOnFailure } from "@/lib/ai/reasoning/answerless";
|
||||
import type { ProviderChainType } from "@/lib/ai/provider-chain";
|
||||
import { singleUserTurn, type CompletionResult } from "@/lib/ai/types";
|
||||
import { PROMPT_VERSION } from "@/lib/ai/prompts/insight-generator";
|
||||
@@ -689,7 +690,8 @@ export async function runTurnModel(args: {
|
||||
});
|
||||
const result = fallback.result;
|
||||
await args.ledger.settleRound({
|
||||
tokens: result.tokensUsed ?? 0,
|
||||
// #1279 — with what the hops that failed before it spent.
|
||||
tokens: (result.tokensUsed ?? 0) + fallback.tokensSpent,
|
||||
cachedTokens: result.cachedInputTokens ?? 0,
|
||||
servedBy: fallback.workingProvider.providerType,
|
||||
final: false,
|
||||
@@ -729,20 +731,17 @@ export async function runTurnModel(args: {
|
||||
};
|
||||
} catch (err) {
|
||||
// The provider chain failed: every round that returned is already
|
||||
// settled; give back what is still reserved before the error frame.
|
||||
await args.ledger.close();
|
||||
// #781 — the client walked away mid-generation. The request's abort
|
||||
// signal is threaded into every provider call, so the teardown surfaces
|
||||
// here as an abort-shaped failure with `request.signal` already flipped.
|
||||
// Close the turn honestly instead of letting the user message dangle
|
||||
// unanswered: persist an EMPTY assistant marker tagged
|
||||
// `providerType: "cancelled"` (the same channel "refusal" and "nudge"
|
||||
// already use for non-provider rows) so the thread shows the turn as
|
||||
// interrupted and offers a retry on reload. The content is empty by
|
||||
// construction — nothing guarded was produced, and anything the client
|
||||
// ever SAW was persisted before the first token frame left, so no
|
||||
// unguarded partial text is ever written. The reservation was refunded
|
||||
// above; a retry pays exactly what a fresh turn pays.
|
||||
// settled; give back what is still reserved before the error frame, less
|
||||
// what the failed hops reported spending (#1279).
|
||||
await args.ledger.close(spentOnFailure(err));
|
||||
// #781 — the client walked away mid-generation (the abort signal is
|
||||
// threaded into every provider call). Close the turn honestly instead of
|
||||
// leaving the user message dangling: persist an EMPTY assistant marker
|
||||
// tagged `providerType: "cancelled"` (the channel "refusal" and "nudge"
|
||||
// use for non-provider rows), so the thread shows the turn as interrupted
|
||||
// and offers a retry. Empty by construction: nothing guarded was
|
||||
// produced, and anything the client SAW was persisted before the first
|
||||
// token frame left. A retry pays exactly what a fresh turn pays.
|
||||
if (signal.aborted) {
|
||||
await appendMessage({
|
||||
conversationId,
|
||||
|
||||
+230
-256
@@ -1,13 +1,7 @@
|
||||
import { safeFetch } from "@/lib/safe-fetch";
|
||||
import { callTimeoutMs } from "./effective-timeout";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { aiEgressPolicyFor } from "./local-host-allowlist";
|
||||
import type {
|
||||
AIProvider,
|
||||
AiMessage,
|
||||
CompletionParams,
|
||||
CompletionResult,
|
||||
} from "./types";
|
||||
import type { AIProvider, CompletionParams, CompletionResult } from "./types";
|
||||
import {
|
||||
buildOpenAIMessages,
|
||||
mapFinishReason,
|
||||
@@ -24,24 +18,29 @@ import {
|
||||
resetJsonModeDialectCache,
|
||||
type JsonModeDialect,
|
||||
} from "./json-dialect";
|
||||
import { emptyReplyError, withStrictJsonPrefix } from "./local-wire";
|
||||
import { RequestHops } from "./request-hops";
|
||||
import { learnFromRefusal } from "./wire-refusal";
|
||||
import type { ReasoningEffort } from "./reasoning-effort";
|
||||
import {
|
||||
ANSWERS_ONLY_UNREASONED,
|
||||
annotateReasoningDowngrade,
|
||||
isReasoningParameterRejection,
|
||||
learnedReasoningDialect,
|
||||
rememberReasoningDialect,
|
||||
noteReasoningSeen,
|
||||
refusesReasoningField,
|
||||
} from "./reasoning/dialect-cache";
|
||||
import { uncontrolledThinkingRoom } from "./reasoning/levels";
|
||||
import {
|
||||
askAgainUnreasoned,
|
||||
knownReasoner,
|
||||
reasonsUnasked,
|
||||
withEarlierSpend,
|
||||
repeatUnreasoned,
|
||||
} from "./reasoning/answerless";
|
||||
import {
|
||||
ReasoningCollector,
|
||||
StreamedThinkSplitter,
|
||||
separateMessage,
|
||||
takeReasoning,
|
||||
type LocalReasoner,
|
||||
} from "./local-reasoning";
|
||||
|
||||
interface LocalClientConfig {
|
||||
@@ -57,16 +56,10 @@ interface LocalClientConfig {
|
||||
operatorTrusted?: boolean;
|
||||
}
|
||||
|
||||
const STRICT_JSON_PREFIX =
|
||||
"Return strict JSON only, no markdown, no commentary.\n\n";
|
||||
|
||||
/**
|
||||
* v1.28.28 (#470) — per-endpoint JSON-mode dialect, learned at runtime.
|
||||
* v1.33.1 — the machinery moved to `json-dialect.ts` when the
|
||||
* OpenAI-compatible gateway provider needed the same learning; the two
|
||||
* clients share one cache keyed by base URL. The names below stay exported
|
||||
* from here because this client's tests and call sites were the first
|
||||
* consumers.
|
||||
* #470 — the per-endpoint JSON-mode dialect, learned at runtime in
|
||||
* `json-dialect.ts` (one cache keyed by base URL, shared with the gateway
|
||||
* client). The names stay exported here for this client's first consumers.
|
||||
*/
|
||||
type LocalJsonDialect = JsonModeDialect;
|
||||
|
||||
@@ -76,58 +69,11 @@ export const resetLocalJsonDialectCache = resetJsonModeDialectCache;
|
||||
export { isResponseFormatRejection };
|
||||
|
||||
/**
|
||||
* v1.22 (#89) — absolute backstop for the streaming path. The real ceiling is
|
||||
* the per-idle-gap timer (`CompletionParams.timeoutMs`); this only stops a
|
||||
* server that streams forever from pinning a worker. Ten minutes is far beyond
|
||||
* any legitimate single Coach turn.
|
||||
* #89 — the streaming path's absolute backstop. The real ceiling is the
|
||||
* per-idle-gap timer; this only stops a server that streams forever.
|
||||
*/
|
||||
const STREAM_ABSOLUTE_CEILING_MS = 10 * 60_000;
|
||||
|
||||
/**
|
||||
* Prepend the strict-JSON instruction to the FIRST user turn's text. Local
|
||||
* model templates respect an in-message instruction reliably, so it is sent
|
||||
* in BOTH dialects — it is the only JSON steering left when an endpoint
|
||||
* rejects `response_format` (dialect `"none"`). Mirrors the pre-refactor
|
||||
* behaviour, which prepended to the single user prompt.
|
||||
*/
|
||||
function withStrictJsonPrefix(messages: AiMessage[]): AiMessage[] {
|
||||
const idx = messages.findIndex((m) => m.role === "user");
|
||||
if (idx === -1) return messages;
|
||||
const out = [...messages];
|
||||
const target = out[idx];
|
||||
if (typeof target.content === "string") {
|
||||
out[idx] = { ...target, content: `${STRICT_JSON_PREFIX}${target.content}` };
|
||||
} else {
|
||||
// Vision parts: prepend a leading text part with the instruction.
|
||||
out[idx] = {
|
||||
...target,
|
||||
content: [{ type: "text", text: STRICT_JSON_PREFIX }, ...target.content],
|
||||
};
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* The empty-reply error. `httpStatus: 0` is the sentinel the chain classifier
|
||||
* treats as a hard failure; `kind` says whether the reply held reasoning and
|
||||
* no answer, or nothing at all, so the log names it instead of "network".
|
||||
*/
|
||||
function emptyReplyError(hadReasoning: boolean, spent: number | null): Error {
|
||||
const err = new Error(
|
||||
hadReasoning
|
||||
? "Local AI returned reasoning but no answer"
|
||||
: "Local AI returned empty content",
|
||||
);
|
||||
Object.assign(err, {
|
||||
httpStatus: 0,
|
||||
kind: hadReasoning ? "reasoning_only" : "empty_response",
|
||||
upstream: "local",
|
||||
// #1279 — what the reply cost, for the day's ledger to charge.
|
||||
...(spent !== null ? { tokensUsed: spent } : {}),
|
||||
});
|
||||
return err;
|
||||
}
|
||||
|
||||
/**
|
||||
* Talks to OpenAI-compatible servers: local runtimes (Ollama, LocalAI,
|
||||
* LM Studio, vLLM, …) AND hosted gateways (LiteLLM, OpenRouter, any
|
||||
@@ -138,12 +84,10 @@ function emptyReplyError(hadReasoning: boolean, spent: number | null): Error {
|
||||
* `LocalJsonDialect` above). The strict-JSON instruction prepended to the
|
||||
* user message stays in both dialects.
|
||||
*
|
||||
* v1.20.0 — tool-calling DEGRADES silently here: most local servers reject an
|
||||
* unknown `tools` field, so the client never forwards it (capability flag
|
||||
* below). F1's tool loop must therefore tolerate a provider that never returns
|
||||
* `toolCalls` and answer from base context. Prompt-caching, where the local
|
||||
* server supports it (vLLM / Ollama prefix cache), is transparent — no flag,
|
||||
* no harm.
|
||||
* Tool calling degrades silently: most local servers reject an unknown
|
||||
* `tools` field, so the client never forwards it (`supportsTools`), and the
|
||||
* Coach's tool loop answers from base context. Prompt caching, where the
|
||||
* server has it (vLLM / Ollama prefix cache), is transparent.
|
||||
*/
|
||||
export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
readonly type = "local" as const;
|
||||
@@ -228,7 +172,11 @@ export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
// v1.41 — a call that carries its own resolved level wins over it.
|
||||
// #1267 — once a model answered nothing while reasoning, the entry's
|
||||
// setting gives way to the `none` that `callReasoningEffort` sends.
|
||||
...(!params.reasoning && this.reasoningEffort && callEffort === null
|
||||
// #1279 — not once the endpoint refused the field.
|
||||
...(!params.reasoning &&
|
||||
this.reasoningEffort &&
|
||||
callEffort === null &&
|
||||
this.sentReasoningEffort(params) !== null
|
||||
? { reasoning_effort: this.reasoningEffort }
|
||||
: {}),
|
||||
...(callEffort ? { reasoning_effort: callEffort } : {}),
|
||||
@@ -279,19 +227,36 @@ export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
// expires, a call without a level of its own included.
|
||||
if (learned === ANSWERS_ONLY_UNREASONED) return "none";
|
||||
if (!params.reasoning) return null;
|
||||
if (learned === "none") return null;
|
||||
if (refusesReasoningField(learned)) return null;
|
||||
return params.reasoning.effort === "off" ? "none" : params.reasoning.effort;
|
||||
}
|
||||
|
||||
/**
|
||||
* #1276 — what this endpoint and model have shown of their reasoning, for
|
||||
* one request (`LocalReasoner`). Seen on a request that asked to reason
|
||||
* counts only for requests that ask too.
|
||||
*/
|
||||
private reasoner(params: CompletionParams, room: boolean): LocalReasoner {
|
||||
const key = this.reasoningDialectKey();
|
||||
const sent = this.sentReasoningEffort(params);
|
||||
const asked = sent !== null && sent !== "none";
|
||||
return {
|
||||
room,
|
||||
known: knownReasoner(key, asked),
|
||||
seen: () => noteReasoningSeen(key, asked),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The `reasoning_effort` this request carries, whichever level decided it:
|
||||
* the call's own (`callReasoningEffort`), else the entry's #1126 setting.
|
||||
* the call's own (`callReasoningEffort`), else the entry's #1126 setting,
|
||||
* unless the endpoint refused the field (#1279).
|
||||
*/
|
||||
private sentReasoningEffort(params: CompletionParams): string | null {
|
||||
return (
|
||||
this.callReasoningEffort(params) ??
|
||||
(params.reasoning ? null : this.reasoningEffort)
|
||||
);
|
||||
const call = this.callReasoningEffort(params);
|
||||
if (call !== null || params.reasoning) return call;
|
||||
const learned = learnedReasoningDialect(this.reasoningDialectKey());
|
||||
return refusesReasoningField(learned) ? null : this.reasoningEffort;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -308,31 +273,93 @@ export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
*/
|
||||
private learnAnswerlessReasoning(
|
||||
params: CompletionParams,
|
||||
hops: RequestHops,
|
||||
hadReasoning: boolean,
|
||||
finishReason: string | undefined,
|
||||
): boolean {
|
||||
return askAgainUnreasoned({
|
||||
key: this.reasoningDialectKey(),
|
||||
provider: "local",
|
||||
signals: "ai.local",
|
||||
sent: this.sentReasoningEffort(params),
|
||||
hadReasoning,
|
||||
finishReason,
|
||||
return (
|
||||
// The lesson holds even when the budget leaves no repeat for it.
|
||||
askAgainUnreasoned({
|
||||
key: this.reasoningDialectKey(),
|
||||
provider: "local",
|
||||
signals: "ai.local",
|
||||
sent: this.sentReasoningEffort(params),
|
||||
hadReasoning,
|
||||
finishReason,
|
||||
}) && hops.canSend
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* #1279 — the call once more with `none` (streamed when `onDelta` is
|
||||
* given), carrying this reply's spend; see `repeatUnreasoned`.
|
||||
*/
|
||||
private askUnreasoned(
|
||||
params: CompletionParams,
|
||||
hops: RequestHops,
|
||||
reply: { hadReasoning: boolean; spent: number | null },
|
||||
onDelta?: (delta: string) => void,
|
||||
): Promise<CompletionResult> {
|
||||
return repeatUnreasoned(
|
||||
onDelta
|
||||
? this.generateCompletionStream(params, onDelta, hops)
|
||||
: this.generateCompletion(params, hops),
|
||||
reply.spent,
|
||||
() => emptyReplyError(reply.hadReasoning, reply.spent),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A non-2xx reply. A refused reasoning field or JSON flag is learned and
|
||||
* the call asked once more without it (`wire-refusal.ts`), and a refused
|
||||
* stream (`streamed`; some servers 400 on `stream` / `stream_options`)
|
||||
* once more buffered, while the call's request budget allows. Otherwise
|
||||
* the structured error; local endpoints and their proxies commonly leak
|
||||
* bearer tokens in error envelopes, so the excerpt is stripped first.
|
||||
*/
|
||||
private async failed(
|
||||
res: Response,
|
||||
params: CompletionParams,
|
||||
hops: RequestHops,
|
||||
sent: { effort: string | null; json: boolean; streamed: boolean },
|
||||
): Promise<CompletionResult> {
|
||||
const bodyExcerpt = sanitiseBodyExcerpt(await res.text().catch(() => ""));
|
||||
const lesson = learnFromRefusal({
|
||||
status: res.status,
|
||||
body: bodyExcerpt,
|
||||
model: this.config.model,
|
||||
reasoning: sent.effort
|
||||
? {
|
||||
sent: sent.effort,
|
||||
key: this.reasoningDialectKey(),
|
||||
provider: "local",
|
||||
names: /reasoning|think/i,
|
||||
}
|
||||
: null,
|
||||
json: sent.json
|
||||
? { endpoint: this.config.baseUrl, action: "ai.local.jsonDialect" }
|
||||
: null,
|
||||
});
|
||||
if ((lesson || sent.streamed) && hops.canSend) {
|
||||
return this.generateCompletion(params, hops);
|
||||
}
|
||||
throw Object.assign(new Error(`Local AI request failed (${res.status})`), {
|
||||
httpStatus: res.status,
|
||||
upstream: "local",
|
||||
model: this.config.model,
|
||||
baseUrl: this.config.baseUrl,
|
||||
bodyExcerpt,
|
||||
...(lesson === "reasoning" ? { reasoningRefused: true } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
/** #1279 — the streamed call once more, carrying this reply's spend. */
|
||||
private streamAgain(
|
||||
params: CompletionParams,
|
||||
onDelta: (delta: string) => void,
|
||||
spent: number | null,
|
||||
): Promise<CompletionResult> {
|
||||
const again = this.generateCompletionStream(params, onDelta);
|
||||
return withEarlierSpend(again, spent);
|
||||
}
|
||||
|
||||
/**
|
||||
* `hops` is the call's request budget (`request-hops.ts`): a caller never
|
||||
* passes it, every repeat inside the call hands it on.
|
||||
*/
|
||||
async generateCompletion(
|
||||
params: CompletionParams,
|
||||
hops: RequestHops = new RequestHops(),
|
||||
): Promise<CompletionResult> {
|
||||
const { url, headers, body, jsonDialect, room } = this.buildRequest(
|
||||
params,
|
||||
@@ -357,7 +384,7 @@ export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
const egress = aiEgressPolicyFor(url, {
|
||||
operatorTrusted: this.config.operatorTrusted,
|
||||
});
|
||||
const res = await safeFetch(
|
||||
const res = await hops.fetch(
|
||||
url,
|
||||
{
|
||||
method: "POST",
|
||||
@@ -375,55 +402,12 @@ export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
);
|
||||
|
||||
if (!res.ok) {
|
||||
// Mirror the openai-client structured-error pattern. Local endpoints
|
||||
// also commonly leak bearer tokens in error envelopes (proxies), so
|
||||
// strip anything that looks like a key before logging.
|
||||
const rawBody = await res.text().catch(() => "");
|
||||
const bodyExcerpt = sanitiseBodyExcerpt(rawBody);
|
||||
// v1.41 — a server that refuses the reasoning field is asked once more
|
||||
// without it and remembers. Before the JSON dialect, so a complaint
|
||||
// about reasoning never switches JSON mode off.
|
||||
const sentEffort = this.callReasoningEffort(params);
|
||||
if (
|
||||
sentEffort &&
|
||||
isReasoningParameterRejection(
|
||||
res.status,
|
||||
bodyExcerpt,
|
||||
/reasoning|think/i,
|
||||
this.config.model,
|
||||
)
|
||||
) {
|
||||
rememberReasoningDialect(this.reasoningDialectKey(), "none");
|
||||
annotateReasoningDowngrade("local", sentEffort, "none");
|
||||
return this.generateCompletion(params);
|
||||
}
|
||||
// v1.28.28 (#470) — dialect self-heal. A 4xx whose body names
|
||||
// `response_format` (or an unknown parameter) means this endpoint
|
||||
// rejects the standard JSON flag: learn the no-flag dialect for the
|
||||
// baseUrl and retry ONCE without it (the recursion terminates because
|
||||
// the cached dialect is now "none"). Unrelated 4xx/5xx errors are not
|
||||
// retried.
|
||||
if (
|
||||
params.responseFormat === "json" &&
|
||||
jsonDialect === "response_format" &&
|
||||
isResponseFormatRejection(res.status, bodyExcerpt)
|
||||
) {
|
||||
rememberJsonModeDialect(this.config.baseUrl, "none");
|
||||
annotate({
|
||||
action: { name: "ai.local.jsonDialect" },
|
||||
meta: { dialect: "none", httpStatus: res.status },
|
||||
});
|
||||
return this.generateCompletion(params);
|
||||
}
|
||||
const err = new Error(`Local AI request failed (${res.status})`);
|
||||
Object.assign(err, {
|
||||
httpStatus: res.status,
|
||||
upstream: "local",
|
||||
model: this.config.model,
|
||||
baseUrl: this.config.baseUrl,
|
||||
bodyExcerpt,
|
||||
return this.failed(res, params, hops, {
|
||||
effort: this.sentReasoningEffort(params),
|
||||
json:
|
||||
params.responseFormat === "json" && jsonDialect === "response_format",
|
||||
streamed: false,
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
|
||||
// v1.28.28 (#470) — the standard `response_format` request succeeded:
|
||||
@@ -470,12 +454,12 @@ export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
params,
|
||||
json.choices?.[0]?.message,
|
||||
finish,
|
||||
room,
|
||||
this.reasoner(params, room),
|
||||
);
|
||||
if (!content) {
|
||||
const spent = json.usage?.total_tokens ?? null;
|
||||
if (this.learnAnswerlessReasoning(params, hadReasoning, finish)) {
|
||||
return withEarlierSpend(this.generateCompletion(params), spent);
|
||||
if (this.learnAnswerlessReasoning(params, hops, hadReasoning, finish)) {
|
||||
return this.askUnreasoned(params, hops, { hadReasoning, spent });
|
||||
}
|
||||
// The reasoning that did arrive still reaches `onReasoning`, as before.
|
||||
collector.result(
|
||||
@@ -507,6 +491,68 @@ export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A server that ignored the stream flag and answered buffered (or sent no
|
||||
* readable body): parsed like the non-streaming path rather than paying a
|
||||
* second round-trip, and the answer handed to `onDelta` whole.
|
||||
*/
|
||||
private async bufferedInsteadOfStream(
|
||||
res: Response,
|
||||
params: CompletionParams,
|
||||
onDelta: (delta: string) => void,
|
||||
hops: RequestHops,
|
||||
room: boolean,
|
||||
): Promise<CompletionResult> {
|
||||
const json = (await res.json().catch(() => null)) as {
|
||||
choices?: Array<{
|
||||
message?: { content?: string };
|
||||
finish_reason?: string;
|
||||
}>;
|
||||
usage?: {
|
||||
total_tokens?: number;
|
||||
prompt_tokens_details?: { cached_tokens?: number };
|
||||
};
|
||||
} | null;
|
||||
// The server ignored `stream` and answered buffered — the embedded
|
||||
// failure can ride this body exactly as it rides the non-streaming one.
|
||||
raiseEmbeddedError(json, {
|
||||
upstream: "local",
|
||||
label: "local AI endpoint",
|
||||
model: this.config.model,
|
||||
});
|
||||
const { content, hadReasoning, collector } = separateMessage(
|
||||
params,
|
||||
json?.choices?.[0]?.message,
|
||||
json?.choices?.[0]?.finish_reason,
|
||||
this.reasoner(params, room),
|
||||
);
|
||||
if (!content) {
|
||||
const spent = json?.usage?.total_tokens ?? null;
|
||||
const finish = json?.choices?.[0]?.finish_reason;
|
||||
if (this.learnAnswerlessReasoning(params, hops, hadReasoning, finish)) {
|
||||
return this.askUnreasoned(
|
||||
params,
|
||||
hops,
|
||||
{ hadReasoning, spent },
|
||||
onDelta,
|
||||
);
|
||||
}
|
||||
collector.result(null);
|
||||
throw emptyReplyError(hadReasoning, spent);
|
||||
}
|
||||
const reasoning = collector.result(null);
|
||||
onDelta(content);
|
||||
return {
|
||||
content,
|
||||
tokensUsed: json?.usage?.total_tokens ?? null,
|
||||
cachedInputTokens: json ? parseCachedTokens(json) : null,
|
||||
model: this.config.model,
|
||||
providerType: "local",
|
||||
finishReason: mapFinishReason(json?.choices?.[0]?.finish_reason),
|
||||
...(reasoning ? { reasoning } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* v1.22 (#89) — true token streaming over the OpenAI-compatible
|
||||
* `stream: true` wire. POSTs with `stream: true`, reads the SSE response body
|
||||
@@ -530,8 +576,12 @@ export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
async generateCompletionStream(
|
||||
params: CompletionParams,
|
||||
onDelta: (delta: string) => void,
|
||||
hops: RequestHops = new RequestHops(),
|
||||
): Promise<CompletionResult> {
|
||||
const { url, headers, body, room } = this.buildRequest(params, true);
|
||||
const { url, headers, body, jsonDialect, room } = this.buildRequest(
|
||||
params,
|
||||
true,
|
||||
);
|
||||
const egress = aiEgressPolicyFor(url, {
|
||||
operatorTrusted: this.config.operatorTrusted,
|
||||
});
|
||||
@@ -547,7 +597,7 @@ export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await safeFetch(
|
||||
res = await hops.fetch(
|
||||
url,
|
||||
{ method: "POST", headers, body },
|
||||
{
|
||||
@@ -558,68 +608,26 @@ export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
signal,
|
||||
},
|
||||
);
|
||||
} catch {
|
||||
} catch (err) {
|
||||
// Network / private-host / abort before the stream opened → fall back to
|
||||
// the buffered path, which surfaces the structured error the chain runner
|
||||
// understands (or succeeds on a server that simply doesn't stream).
|
||||
return this.generateCompletion(params);
|
||||
if (!hops.canSend) throw err;
|
||||
return this.generateCompletion(params, hops);
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
// The server rejected the streaming request (some servers 400 on
|
||||
// `stream`/`stream_options`). Retry once on the buffered path so a
|
||||
// non-streaming server still works and surfaces its real status.
|
||||
return this.generateCompletion(params);
|
||||
return this.failed(res, params, hops, {
|
||||
effort: this.sentReasoningEffort(params),
|
||||
json:
|
||||
params.responseFormat === "json" && jsonDialect === "response_format",
|
||||
streamed: true,
|
||||
});
|
||||
}
|
||||
|
||||
const contentType = res.headers.get("content-type") ?? "";
|
||||
if (!contentType.includes("text/event-stream") || !res.body) {
|
||||
// The server ignored the stream flag and returned a buffered completion
|
||||
// (or there is no readable body). Parse it like the non-streaming path
|
||||
// rather than paying a second round-trip.
|
||||
const json = (await res.json().catch(() => null)) as {
|
||||
choices?: Array<{
|
||||
message?: { content?: string };
|
||||
finish_reason?: string;
|
||||
}>;
|
||||
usage?: {
|
||||
total_tokens?: number;
|
||||
prompt_tokens_details?: { cached_tokens?: number };
|
||||
};
|
||||
} | null;
|
||||
// The server ignored `stream` and answered buffered — the embedded
|
||||
// failure can ride this body exactly as it rides the non-streaming one.
|
||||
raiseEmbeddedError(json, {
|
||||
upstream: "local",
|
||||
label: "local AI endpoint",
|
||||
model: this.config.model,
|
||||
});
|
||||
const { content, hadReasoning, collector } = separateMessage(
|
||||
params,
|
||||
json?.choices?.[0]?.message,
|
||||
json?.choices?.[0]?.finish_reason,
|
||||
room,
|
||||
);
|
||||
if (!content) {
|
||||
const spent = json?.usage?.total_tokens ?? null;
|
||||
const finish = json?.choices?.[0]?.finish_reason;
|
||||
if (this.learnAnswerlessReasoning(params, hadReasoning, finish)) {
|
||||
return this.streamAgain(params, onDelta, spent);
|
||||
}
|
||||
collector.result(null);
|
||||
throw emptyReplyError(hadReasoning, spent);
|
||||
}
|
||||
const reasoning = collector.result(null);
|
||||
onDelta(content);
|
||||
return {
|
||||
content,
|
||||
tokensUsed: json?.usage?.total_tokens ?? null,
|
||||
cachedInputTokens: json ? parseCachedTokens(json) : null,
|
||||
model: this.config.model,
|
||||
providerType: "local",
|
||||
finishReason: mapFinishReason(json?.choices?.[0]?.finish_reason),
|
||||
...(reasoning ? { reasoning } : {}),
|
||||
};
|
||||
return this.bufferedInsteadOfStream(res, params, onDelta, hops, room);
|
||||
}
|
||||
|
||||
const reader = res.body.getReader();
|
||||
@@ -636,44 +644,12 @@ export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
// reasoning never reaches `onDelta`; it would otherwise show up as the
|
||||
// answer. #1276 — for every caller (only one that asked is told).
|
||||
const collector = new ReasoningCollector(params);
|
||||
let thinkState: "detect" | "inside" | "passthrough" = "detect";
|
||||
let pending = "";
|
||||
const passContent = (text: string) => {
|
||||
const splitter = new StreamedThinkSplitter(collector, (text) => {
|
||||
if (!text) return;
|
||||
collector.finish();
|
||||
content += text;
|
||||
onDelta(text);
|
||||
};
|
||||
const onContent = (delta: string) => {
|
||||
if (thinkState === "passthrough") {
|
||||
passContent(delta);
|
||||
return;
|
||||
}
|
||||
pending += delta;
|
||||
if (thinkState === "detect") {
|
||||
const lead = pending.replace(/^\s+/, "");
|
||||
if (lead.length < "<think>".length && "<think>".startsWith(lead)) {
|
||||
return;
|
||||
}
|
||||
if (!lead.startsWith("<think>")) {
|
||||
thinkState = "passthrough";
|
||||
const out = pending;
|
||||
pending = "";
|
||||
passContent(out);
|
||||
return;
|
||||
}
|
||||
thinkState = "inside";
|
||||
pending = lead.slice("<think>".length);
|
||||
}
|
||||
const close = pending.indexOf("</think>");
|
||||
if (close === -1) return;
|
||||
collector.add(pending.slice(0, close));
|
||||
collector.finish();
|
||||
const rest = pending.slice(close + "</think>".length).replace(/^\s+/, "");
|
||||
pending = "";
|
||||
thinkState = "passthrough";
|
||||
passContent(rest);
|
||||
};
|
||||
});
|
||||
|
||||
// An SSE frame can carry the same embedded failure the buffered body
|
||||
// carries. Remember the first one; it only matters if the stream ends
|
||||
@@ -741,7 +717,7 @@ export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
collector.add(reasoningDelta);
|
||||
}
|
||||
const delta = choiceDelta?.content;
|
||||
if (delta) onContent(delta);
|
||||
if (delta) splitter.push(delta);
|
||||
if (typeof chunk.usage?.total_tokens === "number") {
|
||||
tokensUsed = chunk.usage.total_tokens;
|
||||
}
|
||||
@@ -773,24 +749,16 @@ export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
reader.cancel().catch(() => {});
|
||||
}
|
||||
|
||||
// A reply short enough to still sit in the `<think>` detector is prose;
|
||||
// an unclosed block is reasoning that ran out of budget.
|
||||
// (The closures above move `thinkState`; read it afresh.)
|
||||
const endState = thinkState as "detect" | "inside" | "passthrough";
|
||||
if (endState === "detect") {
|
||||
thinkState = "passthrough";
|
||||
passContent(pending);
|
||||
} else if (endState === "inside") {
|
||||
collector.add(pending);
|
||||
}
|
||||
pending = "";
|
||||
const { unclosed } = splitter.end();
|
||||
// #1276 — a lone `</think>`, or unlabelled text that filled the ceiling.
|
||||
// Sent deltas are a liveness signal only; the guards read `content`.
|
||||
const reasoner = this.reasoner(params, room);
|
||||
const tail = takeReasoning(collector, content, {
|
||||
fieldedReasoning: streamedReasoning,
|
||||
finishReason,
|
||||
reasoningRoom: room,
|
||||
reasoner,
|
||||
});
|
||||
if (streamedReasoning || splitter.opened || tail.took) reasoner.seen();
|
||||
content = tail.content;
|
||||
if (!content) {
|
||||
// A stream that ended with a failure frame and no token reports the
|
||||
@@ -801,10 +769,16 @@ export class LocalOpenAICompatibleClient implements AIProvider {
|
||||
model: this.config.model,
|
||||
});
|
||||
// An unclosed inline `<think>` block is reasoning too.
|
||||
const hadReasoning =
|
||||
streamedReasoning || endState === "inside" || tail.took;
|
||||
if (this.learnAnswerlessReasoning(params, hadReasoning, finishReason)) {
|
||||
return this.streamAgain(params, onDelta, tokensUsed);
|
||||
const hadReasoning = streamedReasoning || unclosed || tail.took;
|
||||
if (
|
||||
this.learnAnswerlessReasoning(params, hops, hadReasoning, finishReason)
|
||||
) {
|
||||
return this.askUnreasoned(
|
||||
params,
|
||||
hops,
|
||||
{ hadReasoning, spent: tokensUsed },
|
||||
onDelta,
|
||||
);
|
||||
}
|
||||
collector.result(reasoningTokens);
|
||||
throw emptyReplyError(hadReasoning, tokensUsed);
|
||||
|
||||
@@ -74,12 +74,23 @@ export function messageReasoning(message: unknown): string | null {
|
||||
return text ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* What one request's endpoint and model have shown of their reasoning
|
||||
* (#1276): `room` the ceiling held room for it, `known` the model was seen
|
||||
* reasoning before (`knownReasoner`), `seen` records a reply that reasoned.
|
||||
*/
|
||||
export interface LocalReasoner {
|
||||
room: boolean;
|
||||
known: boolean;
|
||||
seen(): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* #1276 — take the reasoning out of `content` (`answerOf`: an inline
|
||||
* `<think>` block or only its closing tag, or unlabelled text that filled a
|
||||
* ceiling sized for thinking) and hand it to `collector`. Returns the answer
|
||||
* and whether anything was taken. For every caller: reasoning is never an
|
||||
* answer, asked for or not.
|
||||
* `<think>` block or only its closing tag, or, from a model known to reason,
|
||||
* unlabelled text that filled a ceiling sized for thinking) and hand it to
|
||||
* `collector`. Returns the answer and whether anything was taken. For every
|
||||
* caller: reasoning is never an answer, asked for or not.
|
||||
*/
|
||||
export function takeReasoning(
|
||||
collector: ReasoningCollector,
|
||||
@@ -87,10 +98,16 @@ export function takeReasoning(
|
||||
reply: {
|
||||
fieldedReasoning: boolean;
|
||||
finishReason: string | undefined;
|
||||
reasoningRoom: boolean;
|
||||
reasoner: LocalReasoner;
|
||||
},
|
||||
): { content: string; took: boolean } {
|
||||
const separated = answerOf(content, { ...reply, hasToolCalls: false });
|
||||
const separated = answerOf(content, {
|
||||
fieldedReasoning: reply.fieldedReasoning,
|
||||
finishReason: reply.finishReason,
|
||||
hasToolCalls: false,
|
||||
reasoningRoom: reply.reasoner.room,
|
||||
knownReasoner: reply.reasoner.known,
|
||||
});
|
||||
if (separated.reasoning) collector.add(separated.reasoning);
|
||||
if (separated.unseparated) {
|
||||
emitSignal({
|
||||
@@ -110,7 +127,7 @@ export function separateMessage(
|
||||
params: CompletionParams,
|
||||
message: { content?: string | null } | undefined,
|
||||
finishReason: string | undefined,
|
||||
reasoningRoom: boolean,
|
||||
reasoner: LocalReasoner,
|
||||
): { content: string; hadReasoning: boolean; collector: ReasoningCollector } {
|
||||
const collector = new ReasoningCollector(params);
|
||||
const fielded = messageReasoning(message);
|
||||
@@ -118,7 +135,77 @@ export function separateMessage(
|
||||
const { content, took } = takeReasoning(collector, message?.content ?? "", {
|
||||
fieldedReasoning: fielded !== null,
|
||||
finishReason,
|
||||
reasoningRoom,
|
||||
reasoner,
|
||||
});
|
||||
return { content, hadReasoning: fielded !== null || took, collector };
|
||||
const hadReasoning = fielded !== null || took;
|
||||
if (hadReasoning) reasoner.seen();
|
||||
return { content, hadReasoning, collector };
|
||||
}
|
||||
|
||||
/**
|
||||
* A leading inline `<think>` block in streamed `content`, split off delta by
|
||||
* delta: the reasoning goes to the collector, never to `pass`, which gets the
|
||||
* answer as it arrives.
|
||||
*/
|
||||
export class StreamedThinkSplitter {
|
||||
private state: "detect" | "inside" | "passthrough" = "detect";
|
||||
private pending = "";
|
||||
/** A `<think>` block opened. */
|
||||
opened = false;
|
||||
|
||||
constructor(
|
||||
private readonly collector: ReasoningCollector,
|
||||
private readonly pass: (text: string) => void,
|
||||
) {}
|
||||
|
||||
push(delta: string): void {
|
||||
if (this.state === "passthrough") {
|
||||
this.pass(delta);
|
||||
return;
|
||||
}
|
||||
this.pending += delta;
|
||||
if (this.state === "detect") {
|
||||
const lead = this.pending.replace(/^\s+/, "");
|
||||
if (lead.length < "<think>".length && "<think>".startsWith(lead)) {
|
||||
return;
|
||||
}
|
||||
if (!lead.startsWith("<think>")) {
|
||||
this.state = "passthrough";
|
||||
const out = this.pending;
|
||||
this.pending = "";
|
||||
this.pass(out);
|
||||
return;
|
||||
}
|
||||
this.state = "inside";
|
||||
this.opened = true;
|
||||
this.pending = lead.slice("<think>".length);
|
||||
}
|
||||
const close = this.pending.indexOf("</think>");
|
||||
if (close === -1) return;
|
||||
this.collector.add(this.pending.slice(0, close));
|
||||
this.collector.finish();
|
||||
const rest = this.pending
|
||||
.slice(close + "</think>".length)
|
||||
.replace(/^\s+/, "");
|
||||
this.pending = "";
|
||||
this.state = "passthrough";
|
||||
this.pass(rest);
|
||||
}
|
||||
|
||||
/**
|
||||
* The stream ended. A reply short enough to still sit in the detector is
|
||||
* prose; an unclosed block is reasoning that ran out of budget, and the
|
||||
* result says so.
|
||||
*/
|
||||
end(): { unclosed: boolean } {
|
||||
const unclosed = this.state === "inside";
|
||||
if (this.state === "detect") {
|
||||
this.state = "passthrough";
|
||||
this.pass(this.pending);
|
||||
} else if (unclosed) {
|
||||
this.collector.add(this.pending);
|
||||
}
|
||||
this.pending = "";
|
||||
return { unclosed };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* Request and error shapes of the Local client that need nothing of its state.
|
||||
* Split out of `local-client.ts`; that client is their only importer.
|
||||
*
|
||||
* Server-only.
|
||||
*/
|
||||
import type { AiMessage } from "./types";
|
||||
|
||||
const STRICT_JSON_PREFIX =
|
||||
"Return strict JSON only, no markdown, no commentary.\n\n";
|
||||
|
||||
/**
|
||||
* Prepend the strict-JSON instruction to the FIRST user turn's text. Local
|
||||
* model templates respect an in-message instruction reliably, so it is sent
|
||||
* in BOTH dialects — it is the only JSON steering left when an endpoint
|
||||
* rejects `response_format` (dialect `"none"`). Mirrors the pre-refactor
|
||||
* behaviour, which prepended to the single user prompt.
|
||||
*/
|
||||
export function withStrictJsonPrefix(messages: AiMessage[]): AiMessage[] {
|
||||
const idx = messages.findIndex((m) => m.role === "user");
|
||||
if (idx === -1) return messages;
|
||||
const out = [...messages];
|
||||
const target = out[idx];
|
||||
if (typeof target.content === "string") {
|
||||
out[idx] = { ...target, content: `${STRICT_JSON_PREFIX}${target.content}` };
|
||||
} else {
|
||||
// Vision parts: prepend a leading text part with the instruction.
|
||||
out[idx] = {
|
||||
...target,
|
||||
content: [{ type: "text", text: STRICT_JSON_PREFIX }, ...target.content],
|
||||
};
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* The empty-reply error. `httpStatus: 0` is the sentinel the chain classifier
|
||||
* treats as a hard failure; `kind` says whether the reply held reasoning and
|
||||
* no answer, or nothing at all, so the log names it instead of "network".
|
||||
*/
|
||||
export function emptyReplyError(
|
||||
hadReasoning: boolean,
|
||||
spent: number | null,
|
||||
): Error {
|
||||
const err = new Error(
|
||||
hadReasoning
|
||||
? "Local AI returned reasoning but no answer"
|
||||
: "Local AI returned empty content",
|
||||
);
|
||||
Object.assign(err, {
|
||||
httpStatus: 0,
|
||||
kind: hadReasoning ? "reasoning_only" : "empty_response",
|
||||
upstream: "local",
|
||||
// #1279 — what the reply cost, for the day's ledger to charge.
|
||||
...(spent !== null ? { tokensUsed: spent } : {}),
|
||||
});
|
||||
return err;
|
||||
}
|
||||
+147
-126
@@ -1,4 +1,3 @@
|
||||
import { safeFetch } from "@/lib/safe-fetch";
|
||||
import { callTimeoutMs } from "./effective-timeout";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { emitSignal } from "@/lib/logging/signal";
|
||||
@@ -15,16 +14,18 @@ import {
|
||||
} from "./openai-capabilities";
|
||||
import {
|
||||
ANSWERS_ONLY_UNREASONED,
|
||||
annotateReasoningDowngrade,
|
||||
isReasoningParameterRejection,
|
||||
learnedReasoningDialect,
|
||||
rememberReasoningDialect,
|
||||
noteReasoningSeen,
|
||||
refusesReasoningField,
|
||||
} from "./reasoning/dialect-cache";
|
||||
import {
|
||||
askAgainUnreasoned,
|
||||
knownReasoner,
|
||||
reasonsUnasked,
|
||||
withEarlierSpend,
|
||||
repeatUnreasoned,
|
||||
} from "./reasoning/answerless";
|
||||
import { RequestHops } from "./request-hops";
|
||||
import { learnFromRefusal } from "./wire-refusal";
|
||||
import {
|
||||
REASONING_THINKING_BUDGET,
|
||||
uncontrolledThinkingRoom,
|
||||
@@ -39,7 +40,6 @@ import {
|
||||
} from "./reasoning/support";
|
||||
import {
|
||||
hasLearnedJsonModeDialect,
|
||||
isResponseFormatRejection,
|
||||
jsonModeDialectFor,
|
||||
rememberJsonModeDialect,
|
||||
} from "./json-dialect";
|
||||
@@ -167,8 +167,13 @@ export class OpenAIClient implements AIProvider {
|
||||
return this.type === "openai-compatible";
|
||||
}
|
||||
|
||||
/**
|
||||
* `hops` is the call's request budget (`request-hops.ts`): a caller never
|
||||
* passes it, every repeat inside the call hands it on.
|
||||
*/
|
||||
async generateCompletion(
|
||||
params: CompletionParams,
|
||||
hops: RequestHops = new RequestHops(),
|
||||
): Promise<CompletionResult> {
|
||||
const url = `${this.config.baseUrl.replace(/\/$/, "")}/chat/completions`;
|
||||
|
||||
@@ -209,10 +214,18 @@ export class OpenAIClient implements AIProvider {
|
||||
this.config.model,
|
||||
);
|
||||
const reasoningWire = this.planReasoning(params, hasTools, capabilities);
|
||||
// #1126 — the gateway entry's own setting, for a call without a resolved
|
||||
// level (sent below unless the wire already carries `none`).
|
||||
// #1126 — the gateway entry's own setting, for a call without a level,
|
||||
// unless the wire carries one or the endpoint refused the field (#1279).
|
||||
const entryEffort =
|
||||
!params.reasoning && this.isGateway ? this.reasoningEffort : null;
|
||||
!params.reasoning &&
|
||||
this.isGateway &&
|
||||
this.reasoningEffort &&
|
||||
reasoningWire.kind === "none" &&
|
||||
!this.refusesField()
|
||||
? this.reasoningEffort
|
||||
: null;
|
||||
const sent =
|
||||
reasoningWire.kind !== "none" ? reasoningWire.effort : entryEffort;
|
||||
const thinkingTokens = this.thinkingRoom(params, reasoningWire);
|
||||
const tokenBudget = (params.maxTokens ?? 1000) + thinkingTokens;
|
||||
const capabilityParams = capabilities.supportsSamplingControls
|
||||
@@ -234,7 +247,7 @@ export class OpenAIClient implements AIProvider {
|
||||
const trimmedKey = this.config.apiKey?.trim();
|
||||
if (trimmedKey) headers.Authorization = `Bearer ${trimmedKey}`;
|
||||
|
||||
const res = await safeFetch(
|
||||
const res = await hops.fetch(
|
||||
url,
|
||||
{
|
||||
method: "POST",
|
||||
@@ -256,16 +269,11 @@ export class OpenAIClient implements AIProvider {
|
||||
? { prompt_cache_key: params.cacheKey }
|
||||
: {}),
|
||||
// #1126 — the gateway entry's reasoning setting; Default sends
|
||||
// nothing. `api.openai.com` and Codex never get it. v1.41 — a call
|
||||
// that carries its own resolved level (`params.reasoning`) wins.
|
||||
// #1267 — once a model answered nothing while reasoning, the wire
|
||||
// nothing. `api.openai.com` and Codex never get it. A call that
|
||||
// carries its own resolved level (`params.reasoning`) wins, and
|
||||
// (#1267) once a model answered nothing while reasoning, the wire
|
||||
// below carries `none` instead (`planReasoning`).
|
||||
...(!params.reasoning &&
|
||||
this.isGateway &&
|
||||
this.reasoningEffort &&
|
||||
reasoningWire.kind === "none"
|
||||
? { reasoning_effort: this.reasoningEffort }
|
||||
: {}),
|
||||
...(entryEffort ? { reasoning_effort: this.reasoningEffort } : {}),
|
||||
...(reasoningWire.kind === "reasoning_effort"
|
||||
? { reasoning_effort: reasoningWire.effort }
|
||||
: reasoningWire.kind === "openrouter"
|
||||
@@ -273,30 +281,15 @@ export class OpenAIClient implements AIProvider {
|
||||
: {}),
|
||||
}),
|
||||
},
|
||||
// 60 s ceiling so a tar-pit upstream cannot pin a worker
|
||||
// indefinitely. Real completions land well inside this budget.
|
||||
// v1.11.2 — the base URL is user/admin-overridable (BYO gateway), so pin
|
||||
// the connect-time DNS check: a base URL resolving to a private/metadata
|
||||
// address is rejected, closing the SSRF/rebinding surface.
|
||||
// v1.20.1 — compose the caller's cancel signal (Coach SSE disconnect) so
|
||||
// a mid-generation abort tears the upstream call down early.
|
||||
// The record owner's response-timeout setting, else the surface value, else 60 s.
|
||||
// v1.33.1 (#470) — the gateway tag reuses the Local provider's host
|
||||
// policy verbatim: public hosts always; a private host only when the
|
||||
// operator granted its origin (`AI_PRIVATE_ORIGINS` or the legacy host
|
||||
// list), and then only through the pinned operator-approved dispatcher.
|
||||
//
|
||||
// v1.37.30 — the `admin-key` tag consults the same policy. Its base URL
|
||||
// is the one other operator-typed URL in this client (an operator can
|
||||
// point the global provider at a private OpenAI-compatible proxy, e.g.
|
||||
// an OAuth sidecar), and both ends of the grant are the operator's own:
|
||||
// the URL comes from the admin settings, the allowlist from the
|
||||
// operator's environment. Every non-operator construction of this tag
|
||||
// pins the public `api.openai.com` constant, for which the policy
|
||||
// resolves to `true` without ever consulting the allowlist — so the
|
||||
// personal-key posture is unchanged. The `codex` tag keeps the hard
|
||||
// pin: its base URL is a repository constant, there is nothing an
|
||||
// operator legitimately redirects.
|
||||
// The record owner's response-timeout setting (else the surface value,
|
||||
// else 60 s), and the caller's cancel signal (a Coach disconnect tears
|
||||
// the upstream call down). The gateway's base URL and the operator's
|
||||
// `admin-key` one are typed by a person, so they take the Local
|
||||
// provider's host policy: public hosts always, a private host only for
|
||||
// an origin the operator granted (`AI_PRIVATE_ORIGINS`), dialled through
|
||||
// the pinned operator-approved dispatcher; a personal key pins the
|
||||
// public `api.openai.com`, for which the policy never consults the
|
||||
// allowlist. `codex` keeps the hard pin: its URL is a repository constant.
|
||||
{
|
||||
timeoutMs: callTimeoutMs(params, this.responseTimeoutSeconds),
|
||||
...(this.isGateway || this.type === "admin-key"
|
||||
@@ -309,68 +302,7 @@ export class OpenAIClient implements AIProvider {
|
||||
);
|
||||
|
||||
if (!res.ok) {
|
||||
// Pull as much of the body as we can so upstream incidents (OpenAI 5xx,
|
||||
// model-not-found, quota exceeded) are diagnosable from logs instead of
|
||||
// surfacing as an opaque "OpenAI request failed (500)". Strip anything
|
||||
// that looks like an API key from the excerpt before logging.
|
||||
const rawBody = await res.text().catch(() => "");
|
||||
const bodyExcerpt = sanitiseBodyExcerpt(rawBody);
|
||||
// v1.41 — an endpoint or model that refuses the reasoning field is
|
||||
// asked once more without it and remembers, so the answer never fails
|
||||
// on a reasoning parameter. Checked before the JSON dialect: a body
|
||||
// naming the reasoning field must not switch JSON mode off.
|
||||
if (
|
||||
reasoningWire.kind !== "none" &&
|
||||
isReasoningParameterRejection(
|
||||
res.status,
|
||||
bodyExcerpt,
|
||||
/reasoning|thinking/i,
|
||||
this.config.model,
|
||||
)
|
||||
) {
|
||||
rememberReasoningDialect(this.reasoningDialectKey(), "none");
|
||||
annotateReasoningDowngrade(this.type, reasoningWire.effort, "none");
|
||||
return this.generateCompletion(params);
|
||||
}
|
||||
// v1.33.1 (#470) — dialect self-heal, gateway only. A 4xx whose body
|
||||
// names `response_format` (or an unknown parameter) means this endpoint
|
||||
// rejects the standard JSON flag: learn the no-flag dialect for the base
|
||||
// URL and retry ONCE without it. The recursion terminates because the
|
||||
// cached dialect is now "none". Unrelated 4xx/5xx errors are not retried
|
||||
// and surface as the structured error they always were.
|
||||
if (
|
||||
this.isGateway &&
|
||||
sendJsonFormat &&
|
||||
isResponseFormatRejection(res.status, bodyExcerpt)
|
||||
) {
|
||||
rememberJsonModeDialect(this.config.baseUrl, "none");
|
||||
annotate({
|
||||
action: { name: "ai.compat.jsonDialect" },
|
||||
meta: { dialect: "none", httpStatus: res.status },
|
||||
});
|
||||
return this.generateCompletion(params);
|
||||
}
|
||||
// v1.42 — a proxy that stopped offering the configured model answers
|
||||
// with a bare 500 that names nothing. Ask its model listing (free, no
|
||||
// completion) and say so plainly when the name is gone, so the chain,
|
||||
// the health card and the log name the actual fix.
|
||||
const modelNotServed = await this.modelNoLongerServed(res.status);
|
||||
const err = new Error(
|
||||
modelNotServed
|
||||
? `${this.isGateway ? "OpenAI-compatible gateway" : "OpenAI"} request failed (${res.status}): model "${this.config.model}" is not offered by this endpoint`
|
||||
: `${this.isGateway ? "OpenAI-compatible gateway" : "OpenAI"} request failed (${res.status})`,
|
||||
);
|
||||
// Keep the body in a structured field rather than the message so
|
||||
// Error.message stays short and the excerpt lands in dedicated log
|
||||
// fields (bodyExcerpt) that can be filtered/truncated centrally.
|
||||
Object.assign(err, {
|
||||
httpStatus: res.status,
|
||||
upstream: this.isGateway ? "openai-compatible" : "openai",
|
||||
model: this.config.model,
|
||||
bodyExcerpt,
|
||||
...(modelNotServed ? { modelNotServed: true } : {}),
|
||||
});
|
||||
throw err;
|
||||
return this.failed(res, params, hops, { sent, sendJsonFormat });
|
||||
}
|
||||
|
||||
// v1.33.1 (#470) — the standard `response_format` request succeeded: pin
|
||||
@@ -415,7 +347,10 @@ export class OpenAIClient implements AIProvider {
|
||||
|
||||
const choice = json.choices?.[0];
|
||||
const toolCalls = parseOpenAIToolCalls(choice);
|
||||
const reply = this.separateReasoning(choice, toolCalls, thinkingTokens);
|
||||
const reply = this.separateReasoning(choice, toolCalls, {
|
||||
thinkingTokens,
|
||||
sent,
|
||||
});
|
||||
let content: string = reply.content;
|
||||
|
||||
// v1.28.28 (#470) — gateway JSON-mode shim. LiteLLM (and similar
|
||||
@@ -439,9 +374,8 @@ export class OpenAIClient implements AIProvider {
|
||||
// A reply with tool calls and no prose is valid (F1 tool loop); only an
|
||||
// empty reply with neither content NOR tool calls is an error.
|
||||
if (!content && !toolCalls) {
|
||||
return this.answerless(params, {
|
||||
sent:
|
||||
reasoningWire.kind !== "none" ? reasoningWire.effort : entryEffort,
|
||||
return this.answerless(params, hops, {
|
||||
sent,
|
||||
hadReasoning: reply.hadReasoning,
|
||||
finishReason: choice?.finish_reason,
|
||||
spent: json.usage?.total_tokens ?? null,
|
||||
@@ -464,8 +398,9 @@ export class OpenAIClient implements AIProvider {
|
||||
).map((field) => ({ [field]: message[field] }));
|
||||
const usage = json.usage as
|
||||
{ completion_tokens_details?: { reasoning_tokens?: number } } | undefined;
|
||||
const learnedNone =
|
||||
learnedReasoningDialect(this.reasoningDialectKey()) === "none";
|
||||
const learnedNone = refusesReasoningField(
|
||||
learnedReasoningDialect(this.reasoningDialectKey()),
|
||||
);
|
||||
|
||||
return {
|
||||
content,
|
||||
@@ -519,6 +454,7 @@ export class OpenAIClient implements AIProvider {
|
||||
*/
|
||||
private answerless(
|
||||
params: CompletionParams,
|
||||
hops: RequestHops,
|
||||
reply: {
|
||||
sent: string | null;
|
||||
hadReasoning: boolean;
|
||||
@@ -526,8 +462,21 @@ export class OpenAIClient implements AIProvider {
|
||||
spent: number | null;
|
||||
},
|
||||
): Promise<CompletionResult> {
|
||||
const failure = () =>
|
||||
Object.assign(
|
||||
new Error(
|
||||
`${this.isGateway ? "OpenAI-compatible gateway" : "OpenAI"} returned empty content`,
|
||||
),
|
||||
{
|
||||
httpStatus: 0,
|
||||
kind: reply.hadReasoning ? "reasoning_only" : "empty_response",
|
||||
upstream: this.isGateway ? "openai-compatible" : "openai",
|
||||
...(reply.spent !== null ? { tokensUsed: reply.spent } : {}),
|
||||
},
|
||||
);
|
||||
if (
|
||||
this.isGateway &&
|
||||
// The lesson holds even when the budget leaves no repeat for it.
|
||||
askAgainUnreasoned({
|
||||
key: this.reasoningDialectKey(),
|
||||
provider: this.type,
|
||||
@@ -535,20 +484,80 @@ export class OpenAIClient implements AIProvider {
|
||||
sent: reply.sent,
|
||||
hadReasoning: reply.hadReasoning,
|
||||
finishReason: reply.finishReason,
|
||||
})
|
||||
}) &&
|
||||
hops.canSend
|
||||
) {
|
||||
return withEarlierSpend(this.generateCompletion(params), reply.spent);
|
||||
return repeatUnreasoned(
|
||||
this.generateCompletion(params, hops),
|
||||
reply.spent,
|
||||
failure,
|
||||
);
|
||||
}
|
||||
const err = new Error(
|
||||
`${this.isGateway ? "OpenAI-compatible gateway" : "OpenAI"} returned empty content`,
|
||||
);
|
||||
Object.assign(err, {
|
||||
httpStatus: 0,
|
||||
kind: reply.hadReasoning ? "reasoning_only" : "empty_response",
|
||||
upstream: this.isGateway ? "openai-compatible" : "openai",
|
||||
...(reply.spent !== null ? { tokensUsed: reply.spent } : {}),
|
||||
throw failure();
|
||||
}
|
||||
|
||||
/**
|
||||
* A non-2xx reply. As much of the body as can be read goes into a
|
||||
* structured field (with anything key-shaped stripped), so an upstream
|
||||
* incident is diagnosable from the log instead of an opaque status.
|
||||
*
|
||||
* An endpoint or model that refuses the reasoning field, or (#470, gateway
|
||||
* only) the JSON flag, is asked once more without it and remembers
|
||||
* (`wire-refusal.ts`), inside the call's request budget. A refusal the
|
||||
* budget leaves no repeat for is marked `reasoningRefused`.
|
||||
*
|
||||
* A proxy that stopped offering the configured model answers with a bare
|
||||
* 500 that names nothing; its model listing (free, no completion) says so
|
||||
* plainly, so the chain, the health card and the log name the fix.
|
||||
*/
|
||||
private async failed(
|
||||
res: Response,
|
||||
params: CompletionParams,
|
||||
hops: RequestHops,
|
||||
sent: { sent: string | null; sendJsonFormat: boolean },
|
||||
): Promise<CompletionResult> {
|
||||
const bodyExcerpt = sanitiseBodyExcerpt(await res.text().catch(() => ""));
|
||||
// The entry's own setting (#1126) included: a refused field is learned
|
||||
// whichever level put it on the wire.
|
||||
const effort = sent.sent;
|
||||
const lesson = learnFromRefusal({
|
||||
status: res.status,
|
||||
body: bodyExcerpt,
|
||||
model: this.config.model,
|
||||
reasoning:
|
||||
effort === null
|
||||
? null
|
||||
: {
|
||||
sent: effort,
|
||||
key: this.reasoningDialectKey(),
|
||||
provider: this.type,
|
||||
names: /reasoning|thinking/i,
|
||||
},
|
||||
json:
|
||||
this.isGateway && sent.sendJsonFormat
|
||||
? { endpoint: this.config.baseUrl, action: "ai.compat.jsonDialect" }
|
||||
: null,
|
||||
});
|
||||
throw err;
|
||||
if (lesson && hops.canSend) return this.generateCompletion(params, hops);
|
||||
const modelNotServed = await this.modelNoLongerServed(res.status);
|
||||
const label = this.isGateway ? "OpenAI-compatible gateway" : "OpenAI";
|
||||
// The excerpt rides a structured field, not the message, so it lands in
|
||||
// the log fields that are filtered and truncated centrally.
|
||||
throw Object.assign(
|
||||
new Error(
|
||||
modelNotServed
|
||||
? `${label} request failed (${res.status}): model "${this.config.model}" is not offered by this endpoint`
|
||||
: `${label} request failed (${res.status})`,
|
||||
),
|
||||
{
|
||||
httpStatus: res.status,
|
||||
upstream: this.isGateway ? "openai-compatible" : "openai",
|
||||
model: this.config.model,
|
||||
bodyExcerpt,
|
||||
...(modelNotServed ? { modelNotServed: true } : {}),
|
||||
...(lesson === "reasoning" ? { reasoningRefused: true } : {}),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// ── v1.41 reasoning ───────────────────────────────────────────────────────
|
||||
@@ -598,19 +607,24 @@ export class OpenAIClient implements AIProvider {
|
||||
private separateReasoning(
|
||||
choice: OpenAIChoice | undefined,
|
||||
toolCalls: AiToolCall[] | undefined,
|
||||
thinkingTokens: number,
|
||||
request: { thinkingTokens: number; sent: string | null },
|
||||
): { content: string; reasoning: string | null; hadReasoning: boolean } {
|
||||
const message = (choice?.message ?? {}) as Record<string, unknown>;
|
||||
const fielded = [message.reasoning, message.reasoning_content].find(
|
||||
(v): v is string => typeof v === "string" && v.trim().length > 0,
|
||||
);
|
||||
const raw = choice?.message?.content ?? "";
|
||||
const key = this.reasoningDialectKey();
|
||||
const asked = request.sent !== null && request.sent !== "none";
|
||||
const separated = answerOf(raw, {
|
||||
fieldedReasoning: fielded !== undefined,
|
||||
finishReason: choice?.finish_reason,
|
||||
hasToolCalls: toolCalls !== undefined,
|
||||
reasoningRoom: this.isGateway && thinkingTokens > 0,
|
||||
reasoningRoom: this.isGateway && request.thinkingTokens > 0,
|
||||
knownReasoner: knownReasoner(key, asked),
|
||||
});
|
||||
const hadReasoning = fielded !== undefined || separated.content !== raw;
|
||||
if (hadReasoning) noteReasoningSeen(key, asked);
|
||||
if (separated.unseparated) {
|
||||
emitSignal({
|
||||
action: "ai.gateway.reasoning_unseparated",
|
||||
@@ -621,10 +635,17 @@ export class OpenAIClient implements AIProvider {
|
||||
return {
|
||||
content: separated.content,
|
||||
reasoning: fielded ?? separated.reasoning,
|
||||
hadReasoning: fielded !== undefined || separated.content !== raw,
|
||||
hadReasoning,
|
||||
};
|
||||
}
|
||||
|
||||
/** #1279 — this endpoint and model refused the reasoning field. */
|
||||
private refusesField(): boolean {
|
||||
return refusesReasoningField(
|
||||
learnedReasoningDialect(this.reasoningDialectKey()),
|
||||
);
|
||||
}
|
||||
|
||||
private reasoningDialectKey() {
|
||||
return {
|
||||
provider: "openai",
|
||||
@@ -663,7 +684,7 @@ export class OpenAIClient implements AIProvider {
|
||||
: { kind: "reasoning_effort", effort: "none" };
|
||||
}
|
||||
if (!reasoning) return { kind: "none" };
|
||||
if (learned === "none") return { kind: "none" };
|
||||
if (refusesReasoningField(learned)) return { kind: "none" };
|
||||
const level = reasoning.effort;
|
||||
if (this.isGateway) {
|
||||
const effort = level === "off" ? "none" : level;
|
||||
|
||||
@@ -64,12 +64,14 @@ export const HARD_FAILURE_SKIP_THRESHOLD = 3;
|
||||
* From `PROVIDER_PAUSE_THRESHOLD` consecutive hard failures on, each failure
|
||||
* re-arms the skip window at `PROVIDER_PAUSE_BASE_MS` doubled per further
|
||||
* failure, capped at `PROVIDER_PAUSE_MAX_MS`. Paused is the same skip as
|
||||
* backoff (the provider moves to the tail of the chain, behind everything
|
||||
* healthy, and is still walked as a last resort, so a chain never loses its
|
||||
* only provider). When the window lifts, the next call tries it again: one
|
||||
* success clears the count and resumes it in place, one more failure doubles
|
||||
* the pause. Nothing needs to be reset by hand. The admin provider-health card
|
||||
* shows a paused provider and until when.
|
||||
* backoff: an interactive call moves the provider to the tail of the chain,
|
||||
* behind everything healthy, and still walks it as a last resort, so a person
|
||||
* never loses their only provider; background work leaves it out until the
|
||||
* window lifts (#1279, `background-bench.ts`). When the window lifts, the next
|
||||
* call tries it again: one success clears the count and resumes it in place,
|
||||
* one more failure doubles the pause. Saving a fixed provider forgets its
|
||||
* record (`forgetProviderHealth`). The admin provider-health card shows a
|
||||
* paused provider and until when.
|
||||
*
|
||||
* Conservative on purpose: five failures in a row is far past a brown-out,
|
||||
* the first pause is half an hour, and the ceiling is a day, so a provider
|
||||
@@ -283,12 +285,12 @@ export const postgresProviderHealthLedger: ProviderHealthLedger = {
|
||||
* Drop the recorded outcomes of a credential that has just been replaced.
|
||||
*
|
||||
* The ledger describes a credential, not a provider slot: once the person
|
||||
* (or, for `admin-codex`, the operator) signs in again, the `auth_failed`
|
||||
* row of the refused credential would otherwise keep the fresh one at the
|
||||
* tail of the chain and reported as expired for the rest of its cooldown.
|
||||
* Without `userId` the rows of every account are dropped — the operator's
|
||||
* central connection is one credential shared by all of them. Best effort,
|
||||
* like every ledger write.
|
||||
* (or, for the `admin-*` providers, the operator) signs in again or saves a
|
||||
* fixed setup, the row of the refused one would otherwise keep the fresh one
|
||||
* at the tail of every interactive chain, out of background work entirely
|
||||
* (#1279), and reported as expired for the rest of its window. Without
|
||||
* `userId` the rows of every account are dropped: an operator provider is one
|
||||
* credential shared by all of them. Best effort, like every ledger write.
|
||||
*/
|
||||
export async function forgetProviderHealth(target: {
|
||||
providerType: ProviderChainType;
|
||||
@@ -302,7 +304,8 @@ export async function forgetProviderHealth(target: {
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
// A stale row only deprioritises; it never blocks.
|
||||
// A stale row lapses with its window; until then it keeps background
|
||||
// work off the replaced credential (#1279).
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -38,50 +38,36 @@ import {
|
||||
const NEVER_SKIP: ReadonlySet<ProviderChainType> = new Set(["local"]);
|
||||
|
||||
/**
|
||||
* v1.4.16 — multi-provider redundancy runner.
|
||||
* The multi-provider redundancy runner: a provider completion with a
|
||||
* hard-failure walk across an ordered list of providers, so one provider's
|
||||
* outage (a 401 from an expired Codex sign-in, a 503 brown-out, a network
|
||||
* reset) cascades to the next configured provider instead of failing the
|
||||
* request.
|
||||
*
|
||||
* The chain runner wraps a provider completion with a hard-failure
|
||||
* walkthrough across an ordered list of providers. The motivation is
|
||||
* the maintainer's "ein Provider-Ausfall darf die Insights nicht zerschiessen"
|
||||
* mandate: a 401 from Codex (OAuth expired), a 503 from OpenAI
|
||||
* (upstream brown-out), or a network reset MUST cascade to the next
|
||||
* configured provider rather than 422-ing the whole request.
|
||||
* Hard failures (`isHardProviderFailure`) cascade. A malformed-JSON reply is
|
||||
* NOT a hard failure — it is a prompt-following issue, so it bubbles to the
|
||||
* caller's own validator rather than masking itself as provider
|
||||
* unavailability and walking the chain. Every provider receives the same
|
||||
* enriched system prompt; `outcome.workingProvider.providerType` attributes
|
||||
* feedback per provider.
|
||||
*
|
||||
* Hard failures (`isHardProviderFailure`) cascade. A malformed-JSON
|
||||
* reply is NOT a hard failure — it is a prompt-following issue, so it
|
||||
* bubbles to the caller's own validator rather than masking itself as
|
||||
* provider unavailability and walking the chain.
|
||||
*
|
||||
* Cross-feature coupling:
|
||||
* - B5a's `MEDICAL_REFERENCES` is shared via the system prompt; each
|
||||
* provider in the chain receives the same enriched context.
|
||||
* - B5e (planned v1.4.17) reads `outcome.workingProvider.providerType`
|
||||
* to attribute feedback per provider; that's why we re-export it
|
||||
* out of the runner instead of the wrapper.
|
||||
*
|
||||
* The cache: a provider that worked on the last call gets first crack
|
||||
* on the next, so a user whose Codex OAuth lapsed doesn't re-burn the
|
||||
* 401 round-trip on every request for the next hour. Cache is
|
||||
* in-process (per worker), keyed by `userId`, TTL 1h. No coordination
|
||||
* across workers — when scaling out we'd move this to Redis, but for
|
||||
* v1.4.16 the per-worker cache is enough to absorb the typical
|
||||
* "polling reload" pattern.
|
||||
*
|
||||
* v1.18.7 — DECISION: the volatile per-worker hint is left
|
||||
* in-process and NOT folded into the durable Postgres health ledger. The
|
||||
* ledger already shares the load-bearing signal across workers (the
|
||||
* skip-hint reorder in `resolveChainOrder` pushes a dead-credential /
|
||||
* backoff provider to the chain tail for every worker), so the only thing
|
||||
* The cache: a provider that worked on the last call gets first crack on the
|
||||
* next, so a user whose Codex sign-in lapsed doesn't re-burn the 401
|
||||
* round-trip on every request for the next hour. In-process (per worker),
|
||||
* keyed by `userId`, TTL 1h. It stays out of the durable Postgres health
|
||||
* ledger on purpose: the ledger already shares the load-bearing signal across
|
||||
* workers (the skip-hint reorder in `resolveChainOrder`), so the only thing
|
||||
* uncoordinated is a soft "tried this one last, put it first" reorder whose
|
||||
* worst case is a single redundant first-hop per worker per hour — exactly
|
||||
* the bound the v1.18.7 AI audit deemed acceptable at single-maintainer
|
||||
* scale. Promoting the hint to a per-request ledger write would add write
|
||||
* amplification on the hot path for a negligible saving; revisit only if a
|
||||
* multi-worker deploy shows measurable redundant-hop cost.
|
||||
* worst case is one redundant first hop per worker per hour. Promoting it to
|
||||
* a per-request ledger write would add write amplification on the hot path;
|
||||
* revisit only if a multi-worker deploy shows measurable redundant-hop cost.
|
||||
*/
|
||||
|
||||
const LAST_WORKING_TTL_MS = 60 * 60 * 1000; // 1h
|
||||
|
||||
/** The `failureReason` of a hop the operator cost cap refused unsent. */
|
||||
const OPERATOR_CAP_REFUSAL = "operator-cost-cap-exhausted";
|
||||
|
||||
interface CacheEntry {
|
||||
providerType: ProviderChainType;
|
||||
expiresAt: number;
|
||||
@@ -223,6 +209,13 @@ export class AllProvidersFailedError extends Error {
|
||||
* a model that reasons without answering is paid for every time.
|
||||
*/
|
||||
readonly tokensSpent: number;
|
||||
/**
|
||||
* #1279 — true when no hop sent a request: every provider was benched for
|
||||
* background work or refused by the operator cost cap. Such a failure says
|
||||
* nothing about what was asked, so a caller that backs off the record it
|
||||
* failed on does not count it.
|
||||
*/
|
||||
readonly sentNothing: boolean;
|
||||
|
||||
constructor(attempts: FallbackHop[]) {
|
||||
super(
|
||||
@@ -236,6 +229,13 @@ export class AllProvidersFailedError extends Error {
|
||||
(sum, a) => sum + (a.tokensSpent ?? 0),
|
||||
0,
|
||||
);
|
||||
this.sentNothing =
|
||||
attempts.length > 0 &&
|
||||
attempts.every(
|
||||
(a) =>
|
||||
a.failureKind === "paused" ||
|
||||
a.failureReason === OPERATOR_CAP_REFUSAL,
|
||||
);
|
||||
const first = attempts[0];
|
||||
this.primaryCredentialExpired =
|
||||
first !== undefined &&
|
||||
@@ -327,7 +327,7 @@ async function resolveChainOrder(
|
||||
userId: string,
|
||||
providers: ProviderChainResolved[],
|
||||
ledger: ProviderHealthLedger,
|
||||
surface: BudgetSurface,
|
||||
background: boolean,
|
||||
): Promise<{
|
||||
ordered: ProviderChainResolved[];
|
||||
benched: Array<BenchedProvider<ProviderChainResolved>>;
|
||||
@@ -344,10 +344,9 @@ async function resolveChainOrder(
|
||||
});
|
||||
}
|
||||
// #1279 — background work leaves a benched provider out entirely.
|
||||
const { walkable, benched } =
|
||||
surface === "job"
|
||||
? partitionForBackgroundWork(providers, skips, (t) => NEVER_SKIP.has(t))
|
||||
: { walkable: providers, benched: [] };
|
||||
const { walkable, benched } = background
|
||||
? partitionForBackgroundWork(providers, skips, (t) => NEVER_SKIP.has(t))
|
||||
: { walkable: providers, benched: [] };
|
||||
const afterSkips = applyHealthLedgerSkips(walkable, skips);
|
||||
return { ordered: applyLastWorkingCache(userId, afterSkips), benched };
|
||||
}
|
||||
@@ -511,6 +510,29 @@ function signalFailedHops(
|
||||
}
|
||||
}
|
||||
|
||||
/** What both raw runners take. */
|
||||
interface ChainWalkArgs {
|
||||
userId: string;
|
||||
providers: ProviderChainResolved[];
|
||||
params: CompletionParams;
|
||||
/**
|
||||
* Which ceiling an operator-funded FALLBACK hop is rationed against.
|
||||
* `"job"` gets the background share, exactly as the reservation did;
|
||||
* `"coach"` gets the whole ceiling.
|
||||
*/
|
||||
surface: BudgetSurface;
|
||||
/**
|
||||
* #1279 — a person is waiting on this call although it is rationed as
|
||||
* background work (the on-demand briefing). A `"job"` walk otherwise leaves
|
||||
* every provider the health ledger benched out (`background-bench.ts`);
|
||||
* this one walks them at the tail of the chain like any interactive call,
|
||||
* so the person gets the provider's own answer or refusal.
|
||||
*/
|
||||
interactive?: boolean;
|
||||
/** The health ledger; the durable Postgres one unless a test passes its own. */
|
||||
ledger?: ProviderHealthLedger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared chain walker for the two RAW runners (streaming + buffered). Owns the
|
||||
* one set of chain / fallback / health-ledger semantics — empty-chain guard,
|
||||
@@ -522,13 +544,7 @@ function signalFailedHops(
|
||||
* Schema-class failures still bubble (they are not hard) exactly as before.
|
||||
*/
|
||||
async function runRawChain(
|
||||
args: {
|
||||
userId: string;
|
||||
providers: ProviderChainResolved[];
|
||||
params: CompletionParams;
|
||||
surface: BudgetSurface;
|
||||
ledger?: ProviderHealthLedger;
|
||||
},
|
||||
args: ChainWalkArgs,
|
||||
// v1.41 — `params` reaches every candidate unchanged, `reasoning` and
|
||||
// `onReasoning` included. A fallback hop may see `providerState` another
|
||||
// provider made on an earlier round; each client re-inserts only its own
|
||||
@@ -549,7 +565,7 @@ async function runRawChain(
|
||||
userId,
|
||||
providers,
|
||||
ledger,
|
||||
surface,
|
||||
surface === "job" && !args.interactive,
|
||||
);
|
||||
const hops: FallbackHop[] = [];
|
||||
const failedHops: HopSignal[] = [];
|
||||
@@ -615,7 +631,7 @@ async function runRawChain(
|
||||
const hop: FallbackHop = {
|
||||
providerType: candidate.providerType,
|
||||
attempt: i + 1,
|
||||
failureReason: "operator-cost-cap-exhausted",
|
||||
failureReason: OPERATOR_CAP_REFUSAL,
|
||||
httpStatus: null,
|
||||
};
|
||||
hops.push(hop);
|
||||
@@ -658,6 +674,7 @@ async function runRawChain(
|
||||
result,
|
||||
workingProvider: candidate,
|
||||
fallbackHops: hops,
|
||||
tokensSpent: hops.reduce((sum, h) => sum + (h.tokensSpent ?? 0), 0),
|
||||
};
|
||||
} catch (error) {
|
||||
if (!isHardProviderFailure(error)) {
|
||||
@@ -720,21 +737,12 @@ async function runRawChain(
|
||||
* full reply — to confirm real tokens flowed. The returned shape matches the
|
||||
* buffered runner so the route is path-agnostic.
|
||||
*/
|
||||
export async function runStreamingRawCompletionWithFallback(args: {
|
||||
userId: string;
|
||||
providers: ProviderChainResolved[];
|
||||
params: CompletionParams;
|
||||
/**
|
||||
* v1.38.19 — which ceiling an operator-funded FALLBACK
|
||||
* hop is rationed against. `"job"` gets the background share, exactly as the
|
||||
* reservation did; `"coach"` gets the whole ceiling.
|
||||
*/
|
||||
surface: BudgetSurface;
|
||||
/** Called once per streamed token/chunk for providers that stream. */
|
||||
onDelta: (delta: string) => void;
|
||||
/** See `RunWithFallbackParams.ledger`. */
|
||||
ledger?: ProviderHealthLedger;
|
||||
}): Promise<RunRawWithFallbackResult> {
|
||||
export async function runStreamingRawCompletionWithFallback(
|
||||
args: ChainWalkArgs & {
|
||||
/** Called once per streamed token/chunk for providers that stream. */
|
||||
onDelta: (delta: string) => void;
|
||||
},
|
||||
): Promise<RunRawWithFallbackResult> {
|
||||
return runRawChain(
|
||||
args,
|
||||
// Prefer the provider's streaming wire; fall back to the buffered call for
|
||||
@@ -759,6 +767,12 @@ export interface RunRawWithFallbackResult {
|
||||
result: CompletionResult;
|
||||
workingProvider: ProviderChainResolved;
|
||||
fallbackHops: FallbackHop[];
|
||||
/**
|
||||
* #1279 — tokens the hops that failed before `workingProvider` answered
|
||||
* reported spending, summed. Not in `result.tokensUsed`, which is the
|
||||
* answering call's alone; a caller that reconciles budget charges both.
|
||||
*/
|
||||
tokensSpent: number;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -772,15 +786,9 @@ export interface RunRawWithFallbackResult {
|
||||
* last-working cache + structured logging are shared with the streaming
|
||||
* sibling via `runRawChain`.
|
||||
*/
|
||||
export async function runRawCompletionWithFallback(args: {
|
||||
userId: string;
|
||||
providers: ProviderChainResolved[];
|
||||
params: CompletionParams;
|
||||
/** See {@link runStreamingRawCompletionWithFallback}'s `surface`. */
|
||||
surface: BudgetSurface;
|
||||
/** See `RunWithFallbackParams.ledger`. */
|
||||
ledger?: ProviderHealthLedger;
|
||||
}): Promise<RunRawWithFallbackResult> {
|
||||
export async function runRawCompletionWithFallback(
|
||||
args: ChainWalkArgs,
|
||||
): Promise<RunRawWithFallbackResult> {
|
||||
return runRawChain(args, (candidate) =>
|
||||
candidate.instance.generateCompletion(args.params),
|
||||
);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
answerOf,
|
||||
anthropicModelVersion,
|
||||
anthropicRefusesSampling,
|
||||
anthropicThinkingDialect,
|
||||
@@ -195,6 +196,56 @@ describe("parsing helpers", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves a closing tag mentioned in a sentence alone, unless the model is known to reason", () => {
|
||||
const prose =
|
||||
"Some models end their thinking with </think> before they answer.";
|
||||
expect(splitThinkTags(prose)).toEqual({ content: prose, reasoning: null });
|
||||
// The same text from a model seen reasoning: the template opened the
|
||||
// block, so the tag closes it wherever it sits.
|
||||
expect(splitThinkTags("Weighing the week</think>Steady.", true)).toEqual({
|
||||
content: "Steady.",
|
||||
reasoning: "Weighing the week",
|
||||
});
|
||||
// A tag that opens a line splits whoever sent it.
|
||||
expect(splitThinkTags("Weighing the week\n </think>\nSteady.")).toEqual({
|
||||
content: "Steady.",
|
||||
reasoning: "Weighing the week",
|
||||
});
|
||||
});
|
||||
|
||||
describe("answerOf, a reply cut off at the ceiling", () => {
|
||||
const cut = '{"summary": "Your weight has been stable and';
|
||||
const reply = (knownReasoner: boolean) => ({
|
||||
fieldedReasoning: false,
|
||||
finishReason: "length",
|
||||
hasToolCalls: false,
|
||||
reasoningRoom: true,
|
||||
knownReasoner,
|
||||
});
|
||||
|
||||
it("is the answer from a model not known to reason", () => {
|
||||
expect(answerOf(cut, reply(false))).toEqual({
|
||||
content: cut,
|
||||
reasoning: null,
|
||||
unseparated: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("is reasoning the server did not separate from a model known to reason", () => {
|
||||
expect(answerOf(cut, reply(true))).toEqual({
|
||||
content: "",
|
||||
reasoning: cut,
|
||||
unseparated: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("is the answer when the ceiling held no room for reasoning", () => {
|
||||
expect(
|
||||
answerOf(cut, { ...reply(true), reasoningRoom: false }).content,
|
||||
).toBe(cut);
|
||||
});
|
||||
});
|
||||
|
||||
it("maps wire efforts back onto levels", () => {
|
||||
expect(levelOfWireEffort("none")).toBe("off");
|
||||
expect(levelOfWireEffort("minimal")).toBe("off");
|
||||
|
||||
@@ -11,8 +11,14 @@
|
||||
* and a GLM chat template reads an effort it does not know as its maximum
|
||||
* (vllm-project/vllm#54744). That is learned too (`REASONS_REGARDLESS`): later
|
||||
* calls go out as the call asks, with the thinking room #1276 keeps, and an
|
||||
* answerless reply is final. So one call costs two requests at most, and
|
||||
* after the first such call, one.
|
||||
* answerless reply is final. After the first such call, one request.
|
||||
*
|
||||
* An endpoint that refuses the reasoning field altogether can be asked
|
||||
* neither way: it learns `"none"` (no field), and once its model is seen
|
||||
* reasoning unasked, `FIELD_REFUSED_REASONS_REGARDLESS`. Its `none` repeat is
|
||||
* the refused field itself, so a refusal of that repeat is learned as both.
|
||||
* Every request of a call, these repeats included, counts against its budget
|
||||
* of two (`request-hops.ts`).
|
||||
*
|
||||
* What bounds the calls themselves is not here: the chain keeps background
|
||||
* work off a provider that keeps failing (`provider-runner.ts`), and each job
|
||||
@@ -21,9 +27,12 @@
|
||||
import { emitSignal } from "@/lib/logging/signal";
|
||||
import {
|
||||
ANSWERS_ONLY_UNREASONED,
|
||||
FIELD_REFUSED_REASONS_REGARDLESS,
|
||||
REASONS_REGARDLESS,
|
||||
annotateReasoningDowngrade,
|
||||
learnedReasoningDialect,
|
||||
reasoningSeen,
|
||||
refusesReasoningField,
|
||||
rememberReasoningDialect,
|
||||
type ReasoningDialectKey,
|
||||
} from "./dialect-cache";
|
||||
@@ -47,9 +56,19 @@ export function nextAnswerlessStep(
|
||||
sent: string | null,
|
||||
hadReasoning: boolean,
|
||||
): AnswerlessStep {
|
||||
if (learnedReasoningDialect(key) === REASONS_REGARDLESS) {
|
||||
const learned = learnedReasoningDialect(key);
|
||||
if (
|
||||
learned === REASONS_REGARDLESS ||
|
||||
learned === FIELD_REFUSED_REASONS_REGARDLESS
|
||||
) {
|
||||
return { kind: "final", learnedRegardless: false };
|
||||
}
|
||||
// The endpoint refuses the field: there is no `none` to ask with.
|
||||
if (learned === "none") {
|
||||
if (!hadReasoning) return { kind: "final", learnedRegardless: false };
|
||||
rememberReasoningDialect(key, FIELD_REFUSED_REASONS_REGARDLESS);
|
||||
return { kind: "final", learnedRegardless: true };
|
||||
}
|
||||
if (sent === "none") {
|
||||
if (!hadReasoning) return { kind: "final", learnedRegardless: false };
|
||||
rememberReasoningDialect(key, REASONS_REGARDLESS);
|
||||
@@ -111,7 +130,51 @@ export function askAgainUnreasoned(args: {
|
||||
*/
|
||||
export function reasonsUnasked(key: ReasoningDialectKey): boolean {
|
||||
const learned = learnedReasoningDialect(key);
|
||||
return learned === ANSWERS_ONLY_UNREASONED || learned === REASONS_REGARDLESS;
|
||||
return (
|
||||
learned === ANSWERS_ONLY_UNREASONED ||
|
||||
learned === REASONS_REGARDLESS ||
|
||||
learned === FIELD_REFUSED_REASONS_REGARDLESS
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* #1276 — true when the model has shown it reasons, for a request that did
|
||||
* (`asked`) or did not ask it to: a lesson that says it reasons unasked, or a
|
||||
* reply seen reasoning (`reasoningSeen`). Only then is unlabelled text that
|
||||
* ran into the ceiling read as reasoning (`answerOf`).
|
||||
*/
|
||||
export function knownReasoner(
|
||||
key: ReasoningDialectKey,
|
||||
asked: boolean,
|
||||
): boolean {
|
||||
return reasonsUnasked(key) || reasoningSeen(key, asked);
|
||||
}
|
||||
|
||||
/**
|
||||
* The endpoint refused the reasoning field `sent`: send none from now on. A
|
||||
* model already known to reason unasked keeps that lesson too, so it is never
|
||||
* asked with `none` again.
|
||||
*/
|
||||
export function learnReasoningFieldRefused(
|
||||
key: ReasoningDialectKey,
|
||||
provider: string,
|
||||
sent: string,
|
||||
): void {
|
||||
const before = learnedReasoningDialect(key);
|
||||
rememberReasoningDialect(
|
||||
key,
|
||||
reasonsUnasked(key) ? FIELD_REFUSED_REASONS_REGARDLESS : "none",
|
||||
);
|
||||
annotateReasoningDowngrade(provider, sent, "none");
|
||||
// One line per lesson, not per call: the operator learns that the
|
||||
// endpoint takes no reasoning field (an entry setting it refuses included).
|
||||
if (!refusesReasoningField(before)) {
|
||||
emitSignal({
|
||||
action: "ai.provider.reasoning_field_refused",
|
||||
level: "warn",
|
||||
meta: { provider, effort: sent },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -136,6 +199,27 @@ export async function withEarlierSpend<T extends { tokensUsed: number | null }>(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The `none` repeat after an answerless reply, carrying that reply's spend.
|
||||
* When the endpoint refused the `none` itself on the call's last request
|
||||
* (`reasoningRefused` on the error), the call ends with the first reply's
|
||||
* failure (`answerless`): the model reasoned without answering, and that is
|
||||
* what the call ran into, not the field the client then tried.
|
||||
*/
|
||||
export async function repeatUnreasoned<T extends { tokensUsed: number | null }>(
|
||||
repeat: Promise<T>,
|
||||
earlier: number | null,
|
||||
answerless: () => Error,
|
||||
): Promise<T> {
|
||||
try {
|
||||
return await withEarlierSpend(repeat, earlier);
|
||||
} catch (err) {
|
||||
const refused = (err as { reasoningRefused?: unknown } | null)
|
||||
?.reasoningRefused;
|
||||
throw refused === true ? answerless() : err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokens a failed call spent upstream, as its error carries them
|
||||
* (`tokensUsed`), or `null` when it carries none.
|
||||
|
||||
@@ -58,6 +58,21 @@ export const ANSWERS_ONLY_UNREASONED = "off";
|
||||
*/
|
||||
export const REASONS_REGARDLESS = "unswitchable";
|
||||
|
||||
/**
|
||||
* #1279 — the learned value for an endpoint that refuses the reasoning field
|
||||
* outright (LiteLLM in front of Ollama: "does not support parameters:
|
||||
* ['reasoning_effort']") in front of a model that reasons unasked. Both facts
|
||||
* at once: no field is sent, as with `"none"`, and the thinking room stays
|
||||
* and an answerless reply is final, as with `REASONS_REGARDLESS`. Held as one
|
||||
* value each, the two lessons overwrote each other on every request.
|
||||
*/
|
||||
export const FIELD_REFUSED_REASONS_REGARDLESS = "none-unswitchable";
|
||||
|
||||
/** True when `learned` says the endpoint refuses the reasoning field. */
|
||||
export function refusesReasoningField(learned: string | undefined): boolean {
|
||||
return learned === "none" || learned === FIELD_REFUSED_REASONS_REGARDLESS;
|
||||
}
|
||||
|
||||
/** How long a learned downgrade holds before the full request is tried again. */
|
||||
export const DIALECT_TTL_MS = 60 * 60 * 1000;
|
||||
|
||||
@@ -135,9 +150,58 @@ export function isReasoningParameterRejection(
|
||||
return params.test(text) && PARAMETER_REFUSAL.test(text);
|
||||
}
|
||||
|
||||
/**
|
||||
* #1276 — endpoints and models seen reasoning: a reasoning field or a think
|
||||
* tag in a reply. `unasked` when the request carried no effort (or `none`),
|
||||
* and it outlasts a later sighting on a request that asked. Held as long as a
|
||||
* lesson (`DIALECT_TTL_MS`) and on `globalThis` for the same reason.
|
||||
*/
|
||||
const SEEN_SLOT = Symbol.for("healthlog.ai.reasoningSeen");
|
||||
const seen: Map<string, { unasked: boolean; expiresAt: number }> = ((
|
||||
globalThis as Record<symbol, unknown>
|
||||
)[SEEN_SLOT] ??= new Map()) as Map<
|
||||
string,
|
||||
{ unasked: boolean; expiresAt: number }
|
||||
>;
|
||||
|
||||
function seenEntry(k: string) {
|
||||
const entry = seen.get(k);
|
||||
if (entry && Date.now() >= entry.expiresAt) {
|
||||
seen.delete(k);
|
||||
return undefined;
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
/** Record a reply of `key` that reasoned; `asked`: its request asked to. */
|
||||
export function noteReasoningSeen(
|
||||
key: ReasoningDialectKey,
|
||||
asked: boolean,
|
||||
): void {
|
||||
const k = cacheKey(key);
|
||||
seen.set(k, {
|
||||
unasked: !asked || (seenEntry(k)?.unasked ?? false),
|
||||
expiresAt: Date.now() + DIALECT_TTL_MS,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* True when `key` was seen reasoning in a way that speaks for a request that
|
||||
* did (`asked`) or did not ask to reason: a model that reasoned unasked
|
||||
* speaks for both, one seen reasoning only when asked for the first.
|
||||
*/
|
||||
export function reasoningSeen(
|
||||
key: ReasoningDialectKey,
|
||||
asked: boolean,
|
||||
): boolean {
|
||||
const entry = seenEntry(cacheKey(key));
|
||||
return entry !== undefined && (entry.unasked || asked);
|
||||
}
|
||||
|
||||
/** Test hook: forget everything learned. */
|
||||
export function resetReasoningDialectCache(): void {
|
||||
learned.clear();
|
||||
seen.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -275,16 +275,25 @@ export function reasoningTitleOf(text: string): string | null {
|
||||
*
|
||||
* #1276 — a reply can also carry only the closing tag: GLM, Qwen3 and
|
||||
* DeepSeek-R1 chat templates end the generation prompt with `<think>`, so the
|
||||
* model's output starts inside the block. Everything before the first
|
||||
* `</think>` of such a reply is reasoning.
|
||||
* model's output starts inside the block. Everything before that `</think>`
|
||||
* is reasoning when the tag opens a line (the templates close the block on a
|
||||
* line of its own, or at the very start), or anywhere once the model is known
|
||||
* to reason (`knownReasoner`). An ordinary answer that mentions the tag in a
|
||||
* sentence is left whole.
|
||||
*/
|
||||
export function splitThinkTags(content: string): {
|
||||
export function splitThinkTags(
|
||||
content: string,
|
||||
knownReasoner = false,
|
||||
): {
|
||||
content: string;
|
||||
reasoning: string | null;
|
||||
} {
|
||||
const m =
|
||||
/^\s*<think>([\s\S]*?)(<\/think>|$)/.exec(content) ??
|
||||
/^([\s\S]*?)(<\/think>)/.exec(content);
|
||||
(knownReasoner
|
||||
? /^([\s\S]*?)(<\/think>)/
|
||||
: /^((?:[\s\S]*?\n)?)[ \t]*(<\/think>)/
|
||||
).exec(content);
|
||||
if (!m) return { content, reasoning: null };
|
||||
const reasoning = m[1].trim();
|
||||
const rest = m[2] ? content.slice(m[0].length).replace(/^\s+/, "") : "";
|
||||
@@ -299,10 +308,12 @@ export function splitThinkTags(content: string): {
|
||||
* off by {@link splitThinkTags}.
|
||||
* - A reply that ran into a ceiling which held room for reasoning
|
||||
* (`finish_reason: "length"`, `reasoningRoom`), with no reasoning field
|
||||
* beside it, no tag and no tool call, is reasoning the server did not
|
||||
* separate: the budget was sized for thinking plus an answer, so text
|
||||
* that fills all of it unlabelled is the thought, cut off. It is never
|
||||
* the answer.
|
||||
* beside it, no tag and no tool call, from a model known to reason
|
||||
* (`knownReasoner`), is reasoning the server did not separate: the budget
|
||||
* was sized for thinking plus an answer, so text that fills all of it
|
||||
* unlabelled is the thought, cut off. From any other model it is the
|
||||
* answer, cut off, and the caller reads `finish_reason` (a model that
|
||||
* does not reason, asked with any level, still answers).
|
||||
*
|
||||
* `reasoning` is the text taken out of `content`, for the reasoning channel.
|
||||
* `unseparated` marks the second case so the caller can say so in its log.
|
||||
@@ -316,11 +327,13 @@ export function answerOf(
|
||||
hasToolCalls: boolean;
|
||||
/** The request's ceiling held room for reasoning on top of the answer. */
|
||||
reasoningRoom: boolean;
|
||||
/** This endpoint and model have shown they reason (`knownReasoner`). */
|
||||
knownReasoner: boolean;
|
||||
},
|
||||
): { content: string; reasoning: string | null; unseparated: boolean } {
|
||||
const text = content ?? "";
|
||||
if (!text) return { content: "", reasoning: null, unseparated: false };
|
||||
const split = splitThinkTags(text);
|
||||
const split = splitThinkTags(text, reply.knownReasoner);
|
||||
if (split.content !== text || split.reasoning !== null) {
|
||||
return {
|
||||
content: split.content,
|
||||
@@ -331,6 +344,7 @@ export function answerOf(
|
||||
if (
|
||||
reply.finishReason === "length" &&
|
||||
reply.reasoningRoom &&
|
||||
reply.knownReasoner &&
|
||||
!reply.fieldedReasoning &&
|
||||
!reply.hasToolCalls
|
||||
) {
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
/**
|
||||
* #1279 — the HTTP requests one completion call may send.
|
||||
*
|
||||
* The Local and the OpenAI-compatible gateway clients send a request again on
|
||||
* their own: without a reasoning field the endpoint refused, without the JSON
|
||||
* flag it refused, with `reasoning_effort: "none"` after a reply that
|
||||
* reasoned without answering, and the Local stream falls back to a buffered
|
||||
* request. Each repeat was bounded by what it had just learned, and two
|
||||
* lessons that undo each other were not bounded at all: a gateway that
|
||||
* refuses `reasoning_effort` in front of a model that reasons unasked made
|
||||
* one call send requests until something outside stopped it.
|
||||
*
|
||||
* So each call carries one budget through every repeat, and the budget bounds
|
||||
* it, not the lessons: at most `MAX_REQUESTS_PER_CALL` requests, after which
|
||||
* the call ends with the failure it has. What was learned on the way still
|
||||
* holds for the next call. `ai-request-hops-guard.test.ts` holds both clients
|
||||
* to sending only through here and to handing the budget to every repeat.
|
||||
*
|
||||
* Server-only.
|
||||
*/
|
||||
import { safeFetch } from "@/lib/safe-fetch";
|
||||
|
||||
/** The requests one call may send: the first, and one repeat. */
|
||||
export const MAX_REQUESTS_PER_CALL = 2;
|
||||
|
||||
export class RequestHops {
|
||||
private sent = 0;
|
||||
|
||||
/** True while the call may send one more request. */
|
||||
get canSend(): boolean {
|
||||
return this.sent < MAX_REQUESTS_PER_CALL;
|
||||
}
|
||||
|
||||
/**
|
||||
* `safeFetch`, counted. A request past the budget is not sent: every repeat
|
||||
* asks `canSend` first, so this throw marks a repeat that did not.
|
||||
*/
|
||||
fetch(...args: Parameters<typeof safeFetch>): Promise<Response> {
|
||||
if (!this.canSend) {
|
||||
return Promise.reject(
|
||||
Object.assign(new Error("AI call has no request left to send"), {
|
||||
httpStatus: 0,
|
||||
}),
|
||||
);
|
||||
}
|
||||
this.sent += 1;
|
||||
return safeFetch(...args);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
/**
|
||||
* What a refused request teaches the Local and the OpenAI-compatible gateway
|
||||
* clients, in one place for both.
|
||||
*
|
||||
* Only an HTTP 400 that names a field the request carried teaches anything:
|
||||
* the reasoning field (`isReasoningParameterRejection`, learned per endpoint
|
||||
* and model) or the JSON flag (`isResponseFormatRejection`, learned per
|
||||
* endpoint). The reasoning field is checked first, so a body that names it
|
||||
* never switches JSON mode off. Whether the call then sends the lesser request
|
||||
* is the client's, inside its request budget (`request-hops.ts`).
|
||||
*
|
||||
* Server-only.
|
||||
*/
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
|
||||
import {
|
||||
isResponseFormatRejection,
|
||||
rememberJsonModeDialect,
|
||||
} from "./json-dialect";
|
||||
import { learnReasoningFieldRefused } from "./reasoning/answerless";
|
||||
import {
|
||||
isReasoningParameterRejection,
|
||||
type ReasoningDialectKey,
|
||||
} from "./reasoning/dialect-cache";
|
||||
|
||||
export type RefusalLesson = "reasoning" | "json" | null;
|
||||
|
||||
export function learnFromRefusal(args: {
|
||||
status: number;
|
||||
body: string;
|
||||
model: string;
|
||||
/** The reasoning field the request carried, or `null` for none. */
|
||||
reasoning: {
|
||||
sent: string;
|
||||
key: ReasoningDialectKey;
|
||||
provider: string;
|
||||
/** The parameter names this endpoint's refusals use. */
|
||||
names: RegExp;
|
||||
} | null;
|
||||
/** The endpoint, when the request carried the JSON flag. */
|
||||
json: { endpoint: string; action: string } | null;
|
||||
}): RefusalLesson {
|
||||
const { reasoning, json } = args;
|
||||
if (
|
||||
reasoning &&
|
||||
isReasoningParameterRejection(
|
||||
args.status,
|
||||
args.body,
|
||||
reasoning.names,
|
||||
args.model,
|
||||
)
|
||||
) {
|
||||
learnReasoningFieldRefused(
|
||||
reasoning.key,
|
||||
reasoning.provider,
|
||||
reasoning.sent,
|
||||
);
|
||||
return "reasoning";
|
||||
}
|
||||
if (json && isResponseFormatRejection(args.status, args.body)) {
|
||||
rememberJsonModeDialect(json.endpoint, "none");
|
||||
annotate({
|
||||
action: { name: json.action },
|
||||
meta: { dialect: "none", httpStatus: args.status },
|
||||
});
|
||||
return "json";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -31,7 +31,14 @@ vi.mock("@/lib/rollups/mood-rollups", async () => {
|
||||
|
||||
import { buildMoodDailySeries } from "../mood-series";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { readMoodDayRollups } from "@/lib/rollups/mood-rollups";
|
||||
import {
|
||||
ensureUserMoodRollupsFresh,
|
||||
readMoodDayRollups,
|
||||
} from "@/lib/rollups/mood-rollups";
|
||||
import {
|
||||
pendingBackgroundTaskCount,
|
||||
settleBackgroundTasks,
|
||||
} from "@/lib/logging/background-tasks";
|
||||
|
||||
beforeEach(() => {
|
||||
vi.mocked(prisma.moodEntry.findMany).mockResolvedValue([] as never);
|
||||
@@ -104,4 +111,26 @@ describe("buildMoodDailySeries", () => {
|
||||
expect(series.source).toBe("rollup");
|
||||
expect(series.summary.count).toBe(0);
|
||||
});
|
||||
|
||||
it("files the rollup warm-up with the background registry", async () => {
|
||||
// The warm-up is not awaited by the read, so a caller that has to know
|
||||
// when the request's work is over (the statement counter, the suite's
|
||||
// settle hook) can only see it through the registry. Left unregistered,
|
||||
// a stale account's full recompute bled into whatever ran next.
|
||||
let release: () => void = () => {};
|
||||
vi.mocked(ensureUserMoodRollupsFresh).mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
release = () => resolve({ recomputed: true });
|
||||
}),
|
||||
);
|
||||
|
||||
await buildMoodDailySeries("user-1");
|
||||
|
||||
expect(ensureUserMoodRollupsFresh).toHaveBeenCalledWith("user-1");
|
||||
expect(pendingBackgroundTaskCount()).toBe(1);
|
||||
release();
|
||||
await settleBackgroundTasks();
|
||||
expect(pendingBackgroundTaskCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -12,6 +12,8 @@
|
||||
*/
|
||||
import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { runDetached } from "@/lib/logging/detached";
|
||||
import { trackBackgroundTask } from "@/lib/logging/background-tasks";
|
||||
import { summarize, type DataPoint } from "@/lib/analytics/trends";
|
||||
import {
|
||||
ensureUserMoodRollupsFresh,
|
||||
@@ -94,7 +96,9 @@ export async function buildMoodDailySeries(
|
||||
): Promise<MoodDailySeries> {
|
||||
// Fire-and-forget warm-up so the next cold mount for this user lands on
|
||||
// the rollup tier even when the boot-time backfill hasn't reached them.
|
||||
void ensureUserMoodRollupsFresh(userId);
|
||||
// Registered so a caller that must know when the request's work is over
|
||||
// (the suite's settle hook, the statement counter) can await it.
|
||||
trackBackgroundTask(runDetached(() => ensureUserMoodRollupsFresh(userId)));
|
||||
|
||||
const since = new Date(Date.now() - MOOD_SERIES_WINDOW_MS);
|
||||
const rollups = await readMoodDayRollups(userId, since);
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
* the sleep stages. The stages run in the order the route always ran them.
|
||||
*/
|
||||
import type { SessionUser } from "@/lib/auth/session-user";
|
||||
import { runDetached } from "@/lib/logging/detached";
|
||||
import { trackBackgroundTask } from "@/lib/logging/background-tasks";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import type { Locale } from "@/lib/i18n/config";
|
||||
import { DEFAULT_TIMEZONE } from "@/lib/tz/resolver";
|
||||
@@ -42,7 +44,7 @@ export async function buildAnalyticsResponse(
|
||||
// measurement lands, and the next request returns the up-to-date
|
||||
// value once the background refresh completes. `ensureUserRollupsFresh`
|
||||
// wraps its own try/catch, so the void call cannot reject.
|
||||
void ensureUserRollupsFresh(user.id);
|
||||
trackBackgroundTask(runDetached(() => ensureUserRollupsFresh(user.id)));
|
||||
|
||||
// Single per-type coverage probe shared by the
|
||||
// bp_in_target / healthScore / correlations branches below. The
|
||||
|
||||
@@ -59,6 +59,8 @@ import pLimit from "p-limit";
|
||||
import type { MeasurementType } from "@/generated/prisma/client";
|
||||
|
||||
import { prisma } from "@/lib/db";
|
||||
import { runDetached } from "@/lib/logging/detached";
|
||||
import { trackBackgroundTask } from "@/lib/logging/background-tasks";
|
||||
import {
|
||||
HOURLY_MEAN_DAY_TYPES,
|
||||
usesHourlyMeanDay,
|
||||
@@ -277,14 +279,12 @@ export async function computeSummariesSlice(
|
||||
// One clock for every window of the slice, JS and SQL alike (see
|
||||
// `request-clock-sql.ts`).
|
||||
const now = new Date();
|
||||
// v1.4.37.1 — fire-and-forget. See `src/app/api/analytics/route.ts`
|
||||
// for the full rationale: awaiting this on the read path can stall
|
||||
// the event loop for tens of seconds on tenants with large iOS step
|
||||
// sample sets that keep the 90-day window slightly stale. The downstream
|
||||
// coverage probe falls back to live SQL for uncovered types, so
|
||||
// correctness is preserved; the read just doesn't block waiting
|
||||
// for the background refresh.
|
||||
void ensureUserRollupsFresh(userId);
|
||||
// v1.4.37.1 — fire-and-forget (see `src/app/api/analytics/route.ts`):
|
||||
// awaiting it can stall the event loop for tens of seconds on accounts
|
||||
// whose iOS steps keep the 90-day window stale, and the coverage probe
|
||||
// falls back to live SQL meanwhile. Registered so the suite's settle hook
|
||||
// and the statement counter can await it.
|
||||
trackBackgroundTask(runDetached(() => ensureUserRollupsFresh(userId)));
|
||||
|
||||
// v1.4.36 — per-type coverage probe replaces the legacy global
|
||||
// COUNT. The previous gate returned true as soon as ANY type had at
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import type { AccountGrant } from "@/generated/prisma/client";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { runDetached } from "@/lib/logging/detached";
|
||||
import { trackBackgroundTask } from "@/lib/logging/background-tasks";
|
||||
import { SESSION_USER_SELECT, type SessionUser } from "@/lib/auth/session-user";
|
||||
import { annotate, getEvent } from "@/lib/logging/context";
|
||||
import { recordDelegatedAccess } from "@/lib/auth/audit";
|
||||
@@ -372,13 +374,15 @@ async function resolveSwitchedRecord(
|
||||
});
|
||||
// Fire-and-forget, the `ApiToken.lastUsedAt` posture: the read must not wait
|
||||
// on the bookkeeping, and the bookkeeping failing must not fail the read.
|
||||
void touchGrantUsage(grant.id);
|
||||
trackBackgroundTask(runDetached(() => touchGrantUsage(grant.id)));
|
||||
// The owner's copy of the same fact, and the one they can read: a day-
|
||||
// coalesced row on their own record saying who opened it. It lands here
|
||||
// rather than in the handlers because a route cannot forget what it never
|
||||
// had to remember — every delegated request in the product passes through
|
||||
// this line, and only requests that got past the grant check reach it.
|
||||
void recordDelegatedAccess(owner.id, auth.user.id);
|
||||
trackBackgroundTask(
|
||||
runDetached(() => recordDelegatedAccess(owner.id, auth.user.id)),
|
||||
);
|
||||
|
||||
return {
|
||||
session: auth.session,
|
||||
|
||||
@@ -24,6 +24,8 @@
|
||||
import type { SessionUser } from "@/lib/auth/session-user";
|
||||
|
||||
import { prisma } from "@/lib/db";
|
||||
import { runDetached } from "@/lib/logging/detached";
|
||||
import { trackBackgroundTask } from "@/lib/logging/background-tasks";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { cachedSwr, caches, type ServerCache } from "@/lib/cache/server-cache";
|
||||
import { dashboardSnapshotCacheKey } from "@/lib/cache/invalidate";
|
||||
@@ -201,14 +203,18 @@ export async function readDashboardSnapshotCached(
|
||||
// its claim, never thrown.
|
||||
if (briefingAi.available) {
|
||||
const lastSeen = cached.tiles?.lastSeenByType ?? {};
|
||||
void requestTodayBriefingWarm({
|
||||
userId: user.id,
|
||||
locale,
|
||||
timezone: user.timezone,
|
||||
generatedAt,
|
||||
lastSeenAt: (type) => lastSeen[type]?.lastSeenAt ?? null,
|
||||
now,
|
||||
}).catch(() => undefined);
|
||||
trackBackgroundTask(
|
||||
runDetached(() =>
|
||||
requestTodayBriefingWarm({
|
||||
userId: user.id,
|
||||
locale,
|
||||
timezone: user.timezone,
|
||||
generatedAt,
|
||||
lastSeenAt: (type) => lastSeen[type]?.lastSeenAt ?? null,
|
||||
now,
|
||||
}),
|
||||
).catch(() => undefined),
|
||||
);
|
||||
}
|
||||
|
||||
return { body, locale };
|
||||
|
||||
@@ -516,8 +516,6 @@ export const USER_KEPT_FIELDS: Readonly<Record<string, string>> = {
|
||||
oidcIssuer: "a sign-in credential — the external identity binding",
|
||||
oidcSub: "a sign-in credential — the external identity binding",
|
||||
createdAt: "when the account was created; the account is preserved",
|
||||
previewFeaturesJson:
|
||||
"which preview features the person opted into — how the interface is shown, not record content",
|
||||
updatedAt: "row bookkeeping maintained by Prisma",
|
||||
managedProfileAt:
|
||||
"whether this record is one somebody else runs — account shape, not record content. Clearing it would strip the guardian's reach from a record that has no credentials of its own, leaving an account nobody can open, administer or delete. Emptying a record is not emancipating the person in it.",
|
||||
|
||||
@@ -106,6 +106,7 @@ beforeEach(() => {
|
||||
result: { content: "ok", tokensUsed: 10, model: "m" },
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
} as never);
|
||||
});
|
||||
|
||||
@@ -194,6 +195,7 @@ describe("streamFencedReply — numeric grounding over the UNION", () => {
|
||||
},
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
} as never);
|
||||
const res = await streamFencedReply({
|
||||
...baseArgs,
|
||||
|
||||
@@ -42,6 +42,7 @@ import {
|
||||
AllProvidersFailedError,
|
||||
runStreamingRawCompletionWithFallback,
|
||||
} from "@/lib/ai/provider-runner";
|
||||
import { spentOnFailure } from "@/lib/ai/reasoning/answerless";
|
||||
import { singleUserTurn } from "@/lib/ai/types";
|
||||
import {
|
||||
buildDateKey,
|
||||
@@ -285,9 +286,8 @@ export async function streamFencedReply(
|
||||
args;
|
||||
|
||||
// ── Provider resolution: document order (local-first, codex last), SINGLE
|
||||
// pick, NO cascade — so the exact egress equals the checked target. The pick
|
||||
// re-checks `documentAi` for that provider, consent receipt included; a
|
||||
// refusal is thrown (403) before any egress. ──
|
||||
// pick, NO cascade, so the egress is the checked target. The pick re-checks
|
||||
// `documentAi` for it, receipt included, and throws 403 before egress. ──
|
||||
const { pick, withheld } = await resolveDocumentTextProvider(userId);
|
||||
if (withheld) throw withheld;
|
||||
if (!pick) {
|
||||
@@ -364,8 +364,7 @@ Reply now as the assistant, grounded ONLY in the documents above, in ${
|
||||
contractLocale === "de" ? "German" : "English"
|
||||
}.`;
|
||||
|
||||
// The authoritative grounding set: the LIVE attachments' texts only. Never
|
||||
// history, never a snapshot (none exists) — recomputed from `docs` this turn.
|
||||
// The grounding set: the LIVE attachments' texts only, recomputed this turn.
|
||||
const groundingSources = docs.map((d) => d.text);
|
||||
|
||||
type Outcome =
|
||||
@@ -397,7 +396,9 @@ Reply now as the assistant, grounded ONLY in the documents above, in ${
|
||||
});
|
||||
result = fallback.result;
|
||||
} catch (err) {
|
||||
await reconcileSpend(userId, reservation.reserved, 0, dateKey, 0, {
|
||||
// #1279 — what the failed call reported spending stays charged.
|
||||
const spent = spentOnFailure(err);
|
||||
await reconcileSpend(userId, reservation.reserved, spent, dateKey, 0, {
|
||||
servedBy: null,
|
||||
reservedOwner: reservation.owner,
|
||||
}).catch(() => {});
|
||||
|
||||
@@ -111,6 +111,21 @@ describe("admitAccountSettings", () => {
|
||||
expect(admitAccountSettings({}, ctx)).toEqual({ data: {}, refused: [] });
|
||||
});
|
||||
|
||||
it("accepts and ignores the retired preview-feature opt-ins of an older file", async () => {
|
||||
// Written by v1.44 and v1.45, whose column v1.46 dropped. The file must
|
||||
// still restore, so the field is neither refused nor written.
|
||||
const { accountSettingsBackupSchema } =
|
||||
await import("@/lib/export/account-settings-backup-schema");
|
||||
const entry = accountSettingsBackupSchema.parse({
|
||||
timezone: "Europe/Berlin",
|
||||
previewFeaturesJson: { topics: true },
|
||||
});
|
||||
expect(admitAccountSettings(entry, ctx)).toEqual({
|
||||
data: { timezone: "Europe/Berlin" },
|
||||
refused: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses a language, unit or time zone this release does not know", () => {
|
||||
const { data, refused } = admitAccountSettings(
|
||||
{
|
||||
|
||||
@@ -66,6 +66,9 @@ export const accountSettingsBackupSchema = z
|
||||
moodTagLayoutJson: z.unknown().optional(),
|
||||
documentsLayoutJson: z.unknown().optional(),
|
||||
moduleListLayoutJson: z.unknown().optional(),
|
||||
// Retired in v1.46.0 with its column (the preview-feature opt-ins). A file
|
||||
// written by v1.44 or v1.45 still carries it; it is accepted and ignored,
|
||||
// because no codec reads it, so such a file restores everything else.
|
||||
previewFeaturesJson: z.unknown().optional(),
|
||||
activityGoalsJson: z.unknown().optional(),
|
||||
reportSelectionJson: z.unknown().optional(),
|
||||
|
||||
@@ -539,7 +539,6 @@ const ACCOUNT_SETTING_CODECS = {
|
||||
moodTagLayoutJson: json("moodTagLayoutJson"),
|
||||
documentsLayoutJson: json("documentsLayoutJson"),
|
||||
moduleListLayoutJson: json("moduleListLayoutJson"),
|
||||
previewFeaturesJson: json("previewFeaturesJson"),
|
||||
activityGoalsJson: activityGoals,
|
||||
reportSelectionJson: json("reportSelectionJson"),
|
||||
globalExcludedInjectionSites: excludedSites,
|
||||
|
||||
@@ -160,6 +160,14 @@ export const RETIRED_ROUTES: readonly RetiredRoute[] = [
|
||||
"Research Mode was an opt-in for the estimated drug-level curve. The chart had stopped consulting the flag several releases earlier and painted for every account regardless, so the switch governed nothing. The curve is simply part of the medication page now and needs no preference read.",
|
||||
methods: ["GET", "POST", "DELETE"],
|
||||
},
|
||||
{
|
||||
path: "/api/settings/preview-features",
|
||||
removedIn: "1.46.0",
|
||||
replacedBy: null,
|
||||
reason:
|
||||
"Stored per-account opt-ins to preview features, of which there was one, the topic overview pages. Those pages were dropped before they shipped, so the switch governed nothing; the route and its column are gone, and no preview is gated per account.",
|
||||
methods: ["GET", "PATCH"],
|
||||
},
|
||||
];
|
||||
|
||||
const BY_PATH: ReadonlyMap<string, RetiredRoute> = new Map(
|
||||
|
||||
@@ -110,6 +110,7 @@ function mockProviderReply(
|
||||
},
|
||||
workingProvider: { providerType: "admin-openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -133,6 +134,43 @@ describe("runBriefingCompletion — ledger accounting", () => {
|
||||
expect(ledgerTotal).toBe(1234);
|
||||
});
|
||||
|
||||
it("charges what the hops that failed before the answer spent (#1279)", async () => {
|
||||
runRawCompletionWithFallback.mockResolvedValue({
|
||||
result: { content: "{}", model: "m", tokensUsed: 1234 },
|
||||
workingProvider: { providerType: "admin-openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 2700,
|
||||
});
|
||||
|
||||
await runBriefingCompletion(completionArgs());
|
||||
|
||||
expect(ledgerTotal).toBe(1234 + 2700);
|
||||
});
|
||||
|
||||
it("reserves and sends nothing once the caller's bound fired (#1279)", async () => {
|
||||
mockProviderReply(100);
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(
|
||||
runBriefingCompletion(completionArgs({ signal: controller.signal })),
|
||||
).rejects.toThrow();
|
||||
|
||||
expect(runRawCompletionWithFallback).not.toHaveBeenCalled();
|
||||
expect(ledgerTotal).toBe(0);
|
||||
});
|
||||
|
||||
it("hands the bound's signal to the provider call", async () => {
|
||||
mockProviderReply(100);
|
||||
const controller = new AbortController();
|
||||
|
||||
await runBriefingCompletion(completionArgs({ signal: controller.signal }));
|
||||
|
||||
expect(runRawCompletionWithFallback.mock.calls[0][0].params.signal).toBe(
|
||||
controller.signal,
|
||||
);
|
||||
});
|
||||
|
||||
it("charges the reservation when the provider reports no token count", async () => {
|
||||
mockProviderReply(null);
|
||||
|
||||
|
||||
@@ -165,6 +165,7 @@ function completion(content: string) {
|
||||
result: { content, tokensUsed: 10, model: "m" },
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -162,6 +162,7 @@ describe("daily-briefing re-roll on unchanged data", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const outcome = await generateComprehensiveInsight("u1", { locale: "de" });
|
||||
@@ -226,6 +227,7 @@ describe("daily-briefing re-roll on unchanged data", () => {
|
||||
result: { content: "not json at all", tokensUsed: 5, model: "m" },
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const outcome = await generateComprehensiveInsight("u1", { locale: "de" });
|
||||
|
||||
@@ -166,6 +166,7 @@ beforeEach(() => {
|
||||
result: { content: VALID, tokensUsed: 900, model: "m" },
|
||||
workingProvider: { providerType: "admin-openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -269,11 +270,13 @@ describe("generateComprehensiveInsight — daily token ceiling", () => {
|
||||
result: { content: "not json", tokensUsed: 400, model: "m" },
|
||||
workingProvider: { providerType: "admin-openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
result: { content: VALID, tokensUsed: 900, model: "m" },
|
||||
workingProvider: { providerType: "admin-openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const outcome = await generateComprehensiveInsight("u1", { locale: "de" });
|
||||
|
||||
@@ -301,6 +301,7 @@ describe("generateComprehensiveInsight — content-hash gate (v1.16.8)", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const outcome = await generateComprehensiveInsight("u1", {
|
||||
@@ -353,6 +354,7 @@ describe("generateComprehensiveInsight — content-hash gate (v1.16.8)", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
// The kilogram fingerprint no longer matches: a unit switch regenerates.
|
||||
@@ -429,6 +431,7 @@ describe("generateComprehensiveInsight — content-hash gate (v1.16.8)", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const outcome = await generateComprehensiveInsight("u1", {
|
||||
@@ -467,6 +470,7 @@ describe("generateComprehensiveInsight — content-hash gate (v1.16.8)", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const outcome = await generateComprehensiveInsight("u1", {
|
||||
@@ -503,6 +507,7 @@ describe("generateComprehensiveInsight — content-hash gate (v1.16.8)", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const outcome = await generateComprehensiveInsight("u1", {
|
||||
@@ -553,6 +558,7 @@ describe("generateComprehensiveInsight — usage level", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -130,11 +130,13 @@ describe("comprehensive JSON-retry", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
result: { content: VALID, tokensUsed: 10, model: "m" },
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const outcome = await generateComprehensiveInsight("u1", { locale: "de" });
|
||||
@@ -161,6 +163,7 @@ describe("comprehensive JSON-retry", () => {
|
||||
result: { content: "still not json", tokensUsed: 5, model: "m" },
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const outcome = await generateComprehensiveInsight("u1", { locale: "de" });
|
||||
@@ -174,6 +177,7 @@ describe("comprehensive JSON-retry", () => {
|
||||
result: { content: VALID, tokensUsed: 10, model: "m" },
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const outcome = await generateComprehensiveInsight("u1", { locale: "de" });
|
||||
|
||||
@@ -137,6 +137,7 @@ function reply(content: string) {
|
||||
result: { content, tokensUsed: 10, model: "m" },
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -128,6 +128,7 @@ beforeEach(() => {
|
||||
},
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -167,6 +167,7 @@ describe("generateComprehensiveInsight — consent withdrawn in flight", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
userUpdateMany.mockResolvedValue({ count: 1 });
|
||||
const { AiUnavailableError } =
|
||||
@@ -209,6 +210,7 @@ describe("generateComprehensiveInsight — scope/version commit guard", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
// Some User update landed: the guarded commit matches zero rows.
|
||||
userUpdateMany.mockResolvedValue({ count: 0 });
|
||||
@@ -258,6 +260,7 @@ describe("generateComprehensiveInsight — scope/version commit guard", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
// The real DB returns count: 1 here too, since the guarded predicate
|
||||
// never references `updatedAt` — the credential-refresh write cannot
|
||||
@@ -304,6 +307,7 @@ describe("generateComprehensiveInsight — scope/version commit guard", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
// Simulates a genuine concurrent settings/profile change (exclude list,
|
||||
// gender, or display name edited) between the read above and the
|
||||
@@ -349,6 +353,7 @@ describe("generateComprehensiveInsight — scope/version commit guard", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
userUpdateMany.mockResolvedValue({ count: 1 });
|
||||
|
||||
@@ -388,6 +393,7 @@ describe("generateComprehensiveInsight — scope/version commit guard", () => {
|
||||
},
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
userUpdateMany.mockResolvedValue({ count: 0 });
|
||||
|
||||
|
||||
@@ -135,6 +135,7 @@ describe("comprehensive timeout honouring", () => {
|
||||
result: { content: VALID, tokensUsed: 10, model: "m" },
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
await generateComprehensiveInsight("u1", { locale: "de" });
|
||||
@@ -150,6 +151,7 @@ describe("comprehensive timeout honouring", () => {
|
||||
result: { content: VALID, tokensUsed: 10, model: "m" },
|
||||
workingProvider: { providerType: "openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
await generateComprehensiveInsight("u1", { locale: "de" });
|
||||
|
||||
@@ -145,6 +145,7 @@ describe("comprehensive warm-path timeout + failure observability", () => {
|
||||
result: { content: VALID, tokensUsed: 10, model: "m" },
|
||||
workingProvider: { providerType: "codex" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const outcome = await generateComprehensiveInsight("u1", { locale: "de" });
|
||||
@@ -154,6 +155,34 @@ describe("comprehensive warm-path timeout + failure observability", () => {
|
||||
expect(params.timeoutMs).toBe(AI_BUDGETS.comprehensive.timeoutMs);
|
||||
});
|
||||
|
||||
it("sends nothing more once the caller's bound has given up (#1279)", async () => {
|
||||
// A slow model: its reply (not valid JSON, so the generator would ask
|
||||
// again) lands only after the warm's bound fired and aborted the run.
|
||||
const controller = new AbortController();
|
||||
runRawCompletionWithFallback.mockImplementation(async () => {
|
||||
controller.abort();
|
||||
return {
|
||||
result: { content: "not json", tokensUsed: 10, model: "m" },
|
||||
workingProvider: { providerType: "codex" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
};
|
||||
});
|
||||
|
||||
await generateComprehensiveInsight("u1", {
|
||||
locale: "de",
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
// The call in flight carried the bound's signal, so the client could tear
|
||||
// its request down, and no retry left after the bound.
|
||||
expect(runRawCompletionWithFallback).toHaveBeenCalledTimes(1);
|
||||
expect(runRawCompletionWithFallback.mock.calls[0][0].params.signal).toBe(
|
||||
controller.signal,
|
||||
);
|
||||
expect(userUpdate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not silently persist an empty block when every provider fails", async () => {
|
||||
// The mocked class ignores the argument; `[]` satisfies the real type.
|
||||
runRawCompletionWithFallback.mockRejectedValue(
|
||||
|
||||
@@ -22,6 +22,7 @@ vi.mock("@/lib/db", () => ({
|
||||
import {
|
||||
STATUS_NEGATIVE_WINDOW_BASE_MS,
|
||||
clearStatusNegativeWindows,
|
||||
holdStatusNoteWhileBenched,
|
||||
statusNegativeWindowMs,
|
||||
statusNoteCoolingDown,
|
||||
writeStatusNegativeWindow,
|
||||
@@ -116,6 +117,57 @@ describe("writeStatusNegativeWindow", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("holdStatusNoteWhileBenched", () => {
|
||||
// A run that sent nothing because every provider was benched says nothing
|
||||
// about the note: counting it left the note waiting up to two hours after
|
||||
// the provider recovered.
|
||||
it("opens the first window and leaves the streak as it was", async () => {
|
||||
findUnique.mockResolvedValue({ retryAt: new Date(Date.now() - MINUTE) });
|
||||
const before = Date.now();
|
||||
await holdStatusNoteWhileBenched({
|
||||
userId: "u1",
|
||||
cacheAction: "insights.weight-status.en",
|
||||
todayKey: TODAY,
|
||||
});
|
||||
const args = written();
|
||||
expect(args.update).not.toHaveProperty("failureStreak");
|
||||
expect(args.create).toMatchObject({ failureStreak: 0, dateKey: TODAY });
|
||||
const window = args.update.retryAt.getTime() - before;
|
||||
expect(window).toBeGreaterThanOrEqual(STATUS_NEGATIVE_WINDOW_BASE_MS);
|
||||
expect(window).toBeLessThan(STATUS_NEGATIVE_WINDOW_BASE_MS + MINUTE);
|
||||
});
|
||||
|
||||
it("leaves an open window alone, and the failure after it uncounted", async () => {
|
||||
findUnique.mockResolvedValue({
|
||||
retryAt: new Date(Date.now() + MINUTE),
|
||||
failureStreak: 3,
|
||||
});
|
||||
await holdStatusNoteWhileBenched({
|
||||
userId: "u1",
|
||||
cacheAction: "insights.weight-status.en",
|
||||
todayKey: TODAY,
|
||||
});
|
||||
await writeStatusNegativeWindow({
|
||||
userId: "u1",
|
||||
cacheAction: "insights.weight-status.en",
|
||||
todayKey: TODAY,
|
||||
reason: "error",
|
||||
});
|
||||
expect(upsert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("never throws: a missed hold only lets the next failure count", async () => {
|
||||
findUnique.mockRejectedValue(new Error("pool exhausted"));
|
||||
await expect(
|
||||
holdStatusNoteWhileBenched({
|
||||
userId: "u1",
|
||||
cacheAction: "insights.weight-status.en",
|
||||
todayKey: TODAY,
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("statusNegativeWindowMs", () => {
|
||||
it("doubles from five minutes to a two-hour ceiling", () => {
|
||||
expect([1, 2, 3, 4, 5, 6, 7, 30].map(statusNegativeWindowMs)).toEqual(
|
||||
@@ -157,4 +209,14 @@ describe("clearStatusNegativeWindows", () => {
|
||||
updateMany.mockRejectedValue(new Error("pool exhausted"));
|
||||
await expect(clearStatusNegativeWindows("u1")).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("closes them for every record when the operator's provider changed", async () => {
|
||||
await clearStatusNegativeWindows();
|
||||
expect(updateMany).toHaveBeenCalledWith({
|
||||
where: {
|
||||
OR: [{ retryAt: { not: null } }, { failureStreak: { gt: 0 } }],
|
||||
},
|
||||
data: { retryAt: null, negativeReason: null, failureStreak: 0 },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -60,6 +60,7 @@ vi.mock("@/lib/db", () => ({
|
||||
// #1279 — the note's negative window, read before any provider work.
|
||||
insightStatusCache: {
|
||||
findUnique: vi.fn(async () => noteWindow),
|
||||
upsert: vi.fn(async () => ({})),
|
||||
},
|
||||
},
|
||||
}));
|
||||
@@ -125,6 +126,8 @@ function mockProviderReply(tokensUsed: number | null) {
|
||||
runRawCompletionWithFallback.mockResolvedValue({
|
||||
result: { content: '{"summary":"ok"}', model: "m", tokensUsed },
|
||||
workingProvider: { providerType: "admin-openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 0,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -269,6 +272,19 @@ describe("runStatusCompletion — ledger accounting", () => {
|
||||
expect(chat.allowed).toBe(true);
|
||||
});
|
||||
|
||||
it("charges what the hops that failed before the answer spent (#1279)", async () => {
|
||||
runRawCompletionWithFallback.mockResolvedValue({
|
||||
result: { content: '{"summary":"ok"}', model: "m", tokensUsed: 100 },
|
||||
workingProvider: { providerType: "admin-openai" },
|
||||
fallbackHops: [],
|
||||
tokensSpent: 2700,
|
||||
});
|
||||
|
||||
await runStatusCompletion(completionArgs());
|
||||
|
||||
expect(ledgerTotal).toBe(100 + 2700);
|
||||
});
|
||||
|
||||
it("charges the reservation when the provider reports no token count", async () => {
|
||||
mockProviderReply(null);
|
||||
|
||||
@@ -320,6 +336,27 @@ describe("runStatusCompletion — ledger accounting", () => {
|
||||
expect(ledgerTotal).toBe(0);
|
||||
});
|
||||
|
||||
it("aborts the call its bound cut off, so nothing runs on detached (#1279)", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
let signal: AbortSignal | undefined;
|
||||
runRawCompletionWithFallback.mockImplementation(
|
||||
(args: { params: { signal?: AbortSignal } }) => {
|
||||
signal = args.params.signal;
|
||||
return new Promise(() => {});
|
||||
},
|
||||
);
|
||||
const pending = runStatusCompletion(completionArgs());
|
||||
await vi.waitFor(() => expect(signal).toBeDefined());
|
||||
expect(signal?.aborted).toBe(false);
|
||||
await vi.advanceTimersByTimeAsync(10 * 60_000);
|
||||
expect((await pending).kind).toBe("timeout");
|
||||
expect(signal?.aborted).toBe(true);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("asks no provider about a note inside its negative window (#1279)", async () => {
|
||||
noteWindow = { retryAt: new Date(Date.now() + 60_000) };
|
||||
const result = await runStatusCompletion(
|
||||
@@ -338,6 +375,36 @@ describe("runStatusCompletion — ledger accounting", () => {
|
||||
expect(again.kind).toBe("ok");
|
||||
});
|
||||
|
||||
it("holds the note without counting a run the bench sent nothing for (#1279)", async () => {
|
||||
runRawCompletionWithFallback.mockRejectedValue(
|
||||
Object.assign(new Error("All 1 configured providers failed"), {
|
||||
sentNothing: true,
|
||||
}),
|
||||
);
|
||||
const { prisma } = await import("@/lib/db");
|
||||
const result = await runStatusCompletion(
|
||||
completionArgs({ cacheAction: "insights.weight-status.en" }),
|
||||
);
|
||||
expect(result.kind).toBe("error");
|
||||
expect(prisma.insightStatusCache.upsert).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
create: expect.objectContaining({ failureStreak: 0 }),
|
||||
update: expect.not.objectContaining({
|
||||
failureStreak: expect.anything(),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves a run that reached a provider to the caller's counted window", async () => {
|
||||
runRawCompletionWithFallback.mockRejectedValue(new Error("upstream down"));
|
||||
const { prisma } = await import("@/lib/db");
|
||||
await runStatusCompletion(
|
||||
completionArgs({ cacheAction: "insights.weight-status.en" }),
|
||||
);
|
||||
expect(prisma.insightStatusCache.upsert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("charges what a failed chain reported spending (#1279)", async () => {
|
||||
// A model that reasons and never answers is billed for every call; a
|
||||
// full refund let background work spend without the ceiling seeing it.
|
||||
@@ -377,6 +444,7 @@ describe("runStatusCompletion — cost owner decides the ceiling", () => {
|
||||
runRawCompletionWithFallback.mockResolvedValue({
|
||||
result: { content: '{"summary":"ok"}', model: "m", tokensUsed: 100 },
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
tokensSpent: 0,
|
||||
});
|
||||
|
||||
const result = await runStatusCompletion(completionArgs());
|
||||
|
||||
@@ -82,6 +82,7 @@ function answersAfter(afterMs: number) {
|
||||
tokensUsed: 80,
|
||||
},
|
||||
workingProvider: { providerType: "local" },
|
||||
tokensSpent: 0,
|
||||
}),
|
||||
afterMs,
|
||||
),
|
||||
|
||||
@@ -104,6 +104,23 @@ export interface RunBriefingCompletionArgs {
|
||||
* budget; the client adds the thinking budget itself.
|
||||
*/
|
||||
reasoningJob?: BackgroundReasoningJob;
|
||||
/**
|
||||
* #1279 — a person is waiting on this call: the on-demand "Re-run
|
||||
* analysis". It keeps the background share of the day's ceiling, and walks
|
||||
* a provider the health ledger benched at the tail of the chain instead of
|
||||
* leaving it out, so the person gets that provider's own answer or refusal.
|
||||
* Absent for the scheduled generator.
|
||||
*/
|
||||
interactive?: boolean;
|
||||
/**
|
||||
* #1279 — the caller's outer bound (the nightly and forced warms, the
|
||||
* morning refresh). It reaches every provider call, so a call the bound
|
||||
* gave up on is torn down, and once it fired no further call is reserved
|
||||
* or sent: a briefing that outlives its bound does not keep running
|
||||
* detached. The on-demand route passes none on purpose: it finishes and
|
||||
* writes its briefing after the browser stopped waiting (#786).
|
||||
*/
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -123,6 +140,7 @@ export interface RunBriefingCompletionArgs {
|
||||
export async function runBriefingCompletion(
|
||||
args: RunBriefingCompletionArgs,
|
||||
): Promise<RunRawWithFallbackResult> {
|
||||
args.signal?.throwIfAborted();
|
||||
const dateKey = args.dateKey ?? (await buildDateKey(args.userId));
|
||||
|
||||
// v1.41 — one resolution per call: the operator's switch and cap, the payer
|
||||
@@ -175,9 +193,9 @@ export async function runBriefingCompletion(
|
||||
try {
|
||||
outcome = await runRawCompletionWithFallback({
|
||||
userId: args.userId,
|
||||
// A background generator: an operator-funded fallback hop is rationed at
|
||||
// the job share, exactly as the reservation above was.
|
||||
// Rationed at the job share, exactly as the reservation above was.
|
||||
surface: "job",
|
||||
...(args.interactive ? { interactive: true } : {}),
|
||||
providers: args.chain,
|
||||
params: {
|
||||
...singleUserTurn({
|
||||
@@ -187,6 +205,7 @@ export async function runBriefingCompletion(
|
||||
maxTokens: args.maxTokens,
|
||||
seed: args.seed,
|
||||
timeoutMs: args.timeoutMs,
|
||||
signal: args.signal,
|
||||
// Every briefing call parses its reply with `JSON.parse`, so opt the
|
||||
// chains with a native JSON mode into it.
|
||||
responseFormat: "json",
|
||||
@@ -219,7 +238,9 @@ export async function runBriefingCompletion(
|
||||
throw err;
|
||||
}
|
||||
|
||||
const actualTokens = outcome.result.tokensUsed ?? reservation.reserved;
|
||||
// #1279 — plus what the hops that failed before it spent.
|
||||
const actualTokens =
|
||||
(outcome.result.tokensUsed ?? reservation.reserved) + outcome.tokensSpent;
|
||||
await reconcileSpend(
|
||||
args.userId,
|
||||
reservation.reserved,
|
||||
|
||||
@@ -62,6 +62,8 @@ export function runBriefingCall(
|
||||
// the client's 60 s default on large accounts.
|
||||
timeoutMs: ctx.effectiveTimeoutMs,
|
||||
stage: options.stage,
|
||||
// The person who pressed "Re-run analysis" is waiting on this call.
|
||||
interactive: true,
|
||||
// A briefing asked for by hand reasons exactly like the scheduled one;
|
||||
// the retries repair a reply and do not.
|
||||
...(options.reasoning ? { reasoningJob: "daily_briefing" as const } : {}),
|
||||
|
||||
@@ -98,6 +98,8 @@
|
||||
* shape `summarize()` would return.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { runDetached } from "@/lib/logging/detached";
|
||||
import { trackBackgroundTask } from "@/lib/logging/background-tasks";
|
||||
import type { DataSummary } from "@/lib/analytics/trends";
|
||||
import {
|
||||
liveTrend,
|
||||
@@ -254,14 +256,12 @@ export async function buildComprehensiveAggregate(
|
||||
// bp-raw, first-at, coverage-probe) without re-instrumenting.
|
||||
const timings: Record<string, number> = {};
|
||||
|
||||
// v1.4.37.1 — fire-and-forget. See `src/app/api/analytics/route.ts`
|
||||
// for the full rationale: awaiting this on the read path can stall
|
||||
// the event loop for 30–60 s on power-user accounts whose iOS step
|
||||
// samples keep the 90-day window slightly stale. The downstream
|
||||
// coverage probe falls back to live SQL for uncovered types, so
|
||||
// correctness is preserved; the read just doesn't block waiting
|
||||
// for the background refresh.
|
||||
void ensureUserRollupsFresh(userId);
|
||||
// v1.4.37.1 — fire-and-forget (see `src/app/api/analytics/route.ts`):
|
||||
// awaiting it can stall the event loop for 30–60 s on accounts whose iOS
|
||||
// steps keep the 90-day window stale, and the coverage probe falls back to
|
||||
// live SQL meanwhile. Registered so the suite's settle hook and the
|
||||
// statement counter can await it.
|
||||
trackBackgroundTask(runDetached(() => ensureUserRollupsFresh(userId)));
|
||||
|
||||
// v1.4.36 — per-type coverage probe. The legacy global COUNT
|
||||
// returned true as soon as any one type had a DAY bucket, which made
|
||||
|
||||
@@ -244,6 +244,8 @@ async function rerollBriefingParagraph(args: {
|
||||
* slow-backend allowance the main generation does.
|
||||
*/
|
||||
effectiveTimeoutMs: number;
|
||||
/** The caller's bound: a re-roll it gave up on sends nothing more. */
|
||||
signal?: AbortSignal;
|
||||
}): Promise<{ text: string; providerType: string } | null> {
|
||||
let cached: Record<string, unknown>;
|
||||
try {
|
||||
@@ -277,6 +279,7 @@ async function rerollBriefingParagraph(args: {
|
||||
// generation is not clipped at the client's 60 s default mid-stream.
|
||||
// v1.25 — honours the per-user response-timeout setting (see caller).
|
||||
timeoutMs: args.effectiveTimeoutMs,
|
||||
signal: args.signal,
|
||||
stage: "reroll",
|
||||
// v1.41 — the re-roll is a fresh pass at the paragraph, so it may think
|
||||
// like the generation it replaces; the operator's switch still wins.
|
||||
@@ -716,13 +719,10 @@ interface GenerateOptions {
|
||||
/** Skip the 24 h cache short-circuit and force a fresh generation. */
|
||||
force?: boolean;
|
||||
/**
|
||||
* v1.9.0 — abort the generation when the caller has given up on it.
|
||||
*
|
||||
* The on-demand `forceWarmUser` path bounds the comprehensive step with a
|
||||
* timeout and then proceeds to warm the per-status caches itself. Without
|
||||
* this signal a comprehensive that resolves AFTER the timeout would still
|
||||
* write a cache row + timestamp the caller no longer expects. The signal
|
||||
* lets the caller cut the generation off before the cache write.
|
||||
* The caller's bound gave up on the generation (the warms bound it, then
|
||||
* move on). Every provider call carries it, so the call in flight is torn
|
||||
* down and no later one is sent (#1279), and a generation that still
|
||||
* resolves writes no cache row the caller no longer expects.
|
||||
*/
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
@@ -1141,6 +1141,7 @@ export async function generateComprehensiveInsight(
|
||||
features,
|
||||
comparison: groundingExtra,
|
||||
effectiveTimeoutMs,
|
||||
signal,
|
||||
});
|
||||
if (rerolled) {
|
||||
// The reroll rewrites content derived from the scope captured before
|
||||
@@ -1232,6 +1233,7 @@ export async function generateComprehensiveInsight(
|
||||
// generation is not clipped at the client's 60 s default mid-stream.
|
||||
// v1.25 — honours the per-user response-timeout setting.
|
||||
timeoutMs: effectiveTimeoutMs,
|
||||
signal,
|
||||
stage: "generate",
|
||||
// v1.41 — the briefing is one of the few background jobs that may
|
||||
// reason (medium on the person's own provider, low on the operator's,
|
||||
@@ -1328,6 +1330,7 @@ export async function generateComprehensiveInsight(
|
||||
// v1.21.5 — wider upstream budget; see the first generation call.
|
||||
// v1.25 — honours the per-user response-timeout setting.
|
||||
timeoutMs: effectiveTimeoutMs,
|
||||
signal,
|
||||
stage: "json-retry",
|
||||
});
|
||||
result = retry.result;
|
||||
@@ -1421,6 +1424,7 @@ export async function generateComprehensiveInsight(
|
||||
// v1.21.5 — wider upstream budget; see the first generation call.
|
||||
// v1.25 — honours the per-user response-timeout setting.
|
||||
timeoutMs: effectiveTimeoutMs,
|
||||
signal,
|
||||
stage: "grounding-retry",
|
||||
});
|
||||
const retryInsights = parseComprehensiveResult(retry.result.content);
|
||||
@@ -1553,6 +1557,7 @@ export async function generateComprehensiveInsight(
|
||||
temperature: AI_BUDGETS.comprehensive.temperature,
|
||||
maxTokens: resolveInsightsMaxTokens(),
|
||||
timeoutMs: effectiveTimeoutMs,
|
||||
signal,
|
||||
stage: "screen-retry",
|
||||
});
|
||||
const retryInsights = parseComprehensiveResult(retry.result.content);
|
||||
@@ -1594,15 +1599,10 @@ export async function generateComprehensiveInsight(
|
||||
}
|
||||
}
|
||||
|
||||
// v1.9.0 — the caller abandoned this generation (its bounded timeout
|
||||
// fired and it has moved on to warm the per-status caches itself). Bail
|
||||
// out before the cache write so a late-resolving comprehensive cannot
|
||||
// stamp a timestamp/text the caller no longer expects. The provider call
|
||||
// already cost what it cost; what we must NOT do now is touch the cache.
|
||||
// The caller abandoned this generation (its bound fired). Bail out before
|
||||
// the cache write, annotated and marked like every other failure: an
|
||||
// abandoned generation is a real "no fresh briefing tonight" outcome.
|
||||
if (signal?.aborted) {
|
||||
// v1.28.30 — annotated + marked like every other failure: an abandoned
|
||||
// generation is a real "no fresh briefing tonight" outcome, and the read
|
||||
// path needs the marker to render an honest state instead of silence.
|
||||
return failBeforeProvider(userId, locale, "aborted");
|
||||
}
|
||||
|
||||
|
||||
@@ -22,6 +22,8 @@
|
||||
* left on it deliberately (see their call sites in `build.ts`).
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { runDetached } from "@/lib/logging/detached";
|
||||
import { trackBackgroundTask } from "@/lib/logging/background-tasks";
|
||||
import type { DataPoint } from "@/lib/analytics/trends";
|
||||
import {
|
||||
ensureUserMoodRollupsFresh,
|
||||
@@ -41,7 +43,7 @@ export async function readOverviewMoodSeries(
|
||||
userId: string,
|
||||
since: Date,
|
||||
): Promise<OverviewMoodSeries> {
|
||||
void ensureUserMoodRollupsFresh(userId);
|
||||
trackBackgroundTask(runDetached(() => ensureUserMoodRollupsFresh(userId)));
|
||||
const moodRollupDayRows = await readMoodDayRollups(userId, since);
|
||||
if (moodRollupDayRows.length > 0) {
|
||||
return {
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
* Split out of `features.ts`, which builds the block from them.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { runDetached } from "@/lib/logging/detached";
|
||||
import { trackBackgroundTask } from "@/lib/logging/background-tasks";
|
||||
import {
|
||||
ensureUserMoodRollupsFresh,
|
||||
readMoodDayRollups,
|
||||
@@ -39,7 +41,7 @@ export async function readMoodDailyPoints(
|
||||
totalEntries: number;
|
||||
latestScore: number | null;
|
||||
}> {
|
||||
void ensureUserMoodRollupsFresh(userId);
|
||||
trackBackgroundTask(runDetached(() => ensureUserMoodRollupsFresh(userId)));
|
||||
// Five-year window mirrors the mood-rollup writer default; covers
|
||||
// every realistic user history span without an unbounded scan.
|
||||
const fiveYearsAgo = new Date(Date.now() - 5 * 365 * 24 * 60 * 60 * 1000);
|
||||
|
||||
@@ -72,6 +72,48 @@ export async function writeStatusNegativeWindow(args: {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* #1279 — hold a note while its chain sent nothing: every provider was
|
||||
* benched for background work (or refused by the operator cost cap). Such a
|
||||
* run says nothing about the note, and counting it left the note waiting up
|
||||
* to two hours after the provider recovered. Opens the first window unless
|
||||
* one is open and leaves the streak as it was, so the caller's own failure
|
||||
* write lands inside it and neither moves nor counts. Best-effort: a missed
|
||||
* hold only lets that write count.
|
||||
*/
|
||||
export async function holdStatusNoteWhileBenched(args: {
|
||||
userId: string;
|
||||
cacheAction: string;
|
||||
todayKey: string;
|
||||
}): Promise<void> {
|
||||
const key = statusCacheKey(args.cacheAction);
|
||||
if (!key) return;
|
||||
const where = { userId_metric_locale: { userId: args.userId, ...key } };
|
||||
try {
|
||||
const row = await prisma.insightStatusCache.findUnique({
|
||||
where,
|
||||
select: { retryAt: true },
|
||||
});
|
||||
const now = Date.now();
|
||||
if (row?.retryAt && row.retryAt.getTime() > now) return;
|
||||
const retryAt = new Date(now + STATUS_NEGATIVE_WINDOW_BASE_MS);
|
||||
await prisma.insightStatusCache.upsert({
|
||||
where,
|
||||
create: {
|
||||
userId: args.userId,
|
||||
...key,
|
||||
dateKey: args.todayKey,
|
||||
retryAt,
|
||||
negativeReason: "error",
|
||||
failureStreak: 0,
|
||||
},
|
||||
update: { retryAt, negativeReason: "error" },
|
||||
});
|
||||
} catch {
|
||||
// The caller's own write still opens a window.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True while the note's negative-cache window is open. #1279 — checked right
|
||||
* before a status generation reaches a provider (`runStatusCompletion`), so
|
||||
@@ -94,15 +136,17 @@ export async function statusNoteCoolingDown(
|
||||
/**
|
||||
* Close every open window on one record's notes, streaks included: the person
|
||||
* asked for fresh notes (the regenerate action) or changed what answers them.
|
||||
* Best-effort: a failed write leaves windows that lapse on their own.
|
||||
* Without `userId`, on every record's notes: the operator changed the
|
||||
* provider every account may fall back on. Best-effort: a failed write leaves
|
||||
* windows that lapse on their own.
|
||||
*/
|
||||
export async function clearStatusNegativeWindows(
|
||||
userId: string,
|
||||
userId?: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await prisma.insightStatusCache.updateMany({
|
||||
where: {
|
||||
userId,
|
||||
...(userId ? { userId } : {}),
|
||||
OR: [{ retryAt: { not: null } }, { failureStreak: { gt: 0 } }],
|
||||
},
|
||||
data: { retryAt: null, negativeReason: null, failureStreak: 0 },
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
reserveBudget,
|
||||
resolveCostOwner,
|
||||
resolveDailyCapFor,
|
||||
type BudgetCostOwner,
|
||||
} from "@/lib/ai/coach/budget";
|
||||
import {
|
||||
AI_BUDGETS,
|
||||
@@ -36,7 +37,10 @@ import {
|
||||
type BackgroundReasoningJob,
|
||||
} from "@/lib/ai/reasoning/levels";
|
||||
import { resolveJobReasoning } from "@/lib/ai/reasoning/controls";
|
||||
import { statusNoteCoolingDown } from "./status-negative-window";
|
||||
import {
|
||||
holdStatusNoteWhileBenched,
|
||||
statusNoteCoolingDown,
|
||||
} from "./status-negative-window";
|
||||
|
||||
/**
|
||||
* v1.41 — the fallback ceiling for a status-path call that reasons. Thinking
|
||||
@@ -228,15 +232,11 @@ export async function runStatusCompletion(
|
||||
return { kind: "none", reason: refusal.reason };
|
||||
}
|
||||
|
||||
// Honour the per-user response-timeout setting the operator dials in for a
|
||||
// slow self-hosted / local backend (Settings → AI). This is the single
|
||||
// chokepoint every status / reference surface funnels through — per-metric
|
||||
// cards, the batched assessment, the derived assessments, and the period
|
||||
// narratives — so resolving it here threads the setting onto all of them at
|
||||
// once. A positive stored value wins (seconds → ms); unset falls back to the
|
||||
// status-path budget. Applied to the upstream call's own `timeoutMs` and,
|
||||
// times the calls a run may make, to the outer `withTimeout` cap, so a
|
||||
// raised value is not silently clipped by a wall-clock sized for less.
|
||||
// The per-user response-timeout setting (Settings → AI), threaded here onto
|
||||
// every status / reference surface at once. A positive stored value wins
|
||||
// (seconds → ms); unset falls back to the status-path budget. Applied to the
|
||||
// call's own `timeoutMs` and, times the calls a run may make, to the outer
|
||||
// `withTimeout` cap, so a raised value is not clipped by a shorter bound.
|
||||
const settingsRow = await prisma.user.findUnique({
|
||||
where: { id: userId },
|
||||
select: { aiResponseTimeoutSeconds: true },
|
||||
@@ -260,12 +260,9 @@ export async function runStatusCompletion(
|
||||
// The answer budget. Each client adds the thinking budget on top itself.
|
||||
const maxTokens = args.maxTokens ?? AI_BUDGETS.status.maxTokens;
|
||||
|
||||
// The day's token ledger. Until now this chokepoint — the provider entry for
|
||||
// EVERY status/reference family (the specialised cards, the generic metric
|
||||
// cards, biomarker cards, the batched assessment, the derived scores, the
|
||||
// period narrative, and the off-request Coach memory workers: rolling
|
||||
// summary, fact extraction, plan proposals) — ran with no accounting at all,
|
||||
// so none of that spend appeared in `coach_usage` and no ceiling applied.
|
||||
// The day's token ledger, for EVERY status/reference family (the cards, the
|
||||
// batched assessment, the derived scores, the period narrative, and the
|
||||
// off-request Coach memory workers).
|
||||
//
|
||||
// The reservation is atomic (single upsert-increment), matching the Coach and
|
||||
// document paths: a read-then-write check would let concurrent generations
|
||||
@@ -302,6 +299,10 @@ export async function runStatusCompletion(
|
||||
return { kind: "error" };
|
||||
}
|
||||
|
||||
// #1279 — the run's bound aborts the call it cuts off, so a provider call
|
||||
// the bound gave up on (and any repeat inside it) stops instead of running
|
||||
// on detached, with its spend reported nowhere.
|
||||
const abort = new AbortController();
|
||||
const raced = await withTimeout(
|
||||
() =>
|
||||
runRawCompletionWithFallback({
|
||||
@@ -324,34 +325,26 @@ export async function runStatusCompletion(
|
||||
responseFormat: args.responseFormat === "text" ? undefined : "json",
|
||||
timeoutMs: effectiveTimeoutMs,
|
||||
}),
|
||||
signal: abort.signal,
|
||||
// v1.41 — only a named job ever carries this; see `reasoningJob`.
|
||||
...(reasoning ? { reasoning } : {}),
|
||||
},
|
||||
}),
|
||||
statusRunBoundMs(effectiveTimeoutMs),
|
||||
null,
|
||||
() => abort.abort(),
|
||||
);
|
||||
|
||||
if (raced.timedOut || raced.errored || raced.value === null) {
|
||||
// A timeout or failure: charge only what a reply reported spending, never
|
||||
// an invented figure. #1279 — a failed chain's report counts: a model
|
||||
// that reasons without answering bills every call, and a full refund let
|
||||
// background work spend without the day's ceiling ever seeing it.
|
||||
const spent = spentOnFailure(raced.error);
|
||||
await reconcileSpend(userId, reservation.reserved, spent, dateKey, 0, {
|
||||
servedBy: null,
|
||||
reservedOwner: reservation.owner,
|
||||
});
|
||||
return { kind: raced.timedOut ? "timeout" : "error" };
|
||||
return failedRun(raced, { userId, cacheAction, dateKey, reservation });
|
||||
}
|
||||
|
||||
const { result, workingProvider } = raced.value;
|
||||
// Reconcile against what the provider actually reported. This runs for the
|
||||
// empty-content branch too: those tokens were burned upstream even though the
|
||||
// reply was unusable, so they stay on the ledger rather than being refunded
|
||||
// into a free retry loop. Falls back to the reservation when the provider
|
||||
// reports no count, so an unreported generation is never billed as zero.
|
||||
const actualTokens = result.tokensUsed ?? reservation.reserved;
|
||||
const { result, workingProvider, tokensSpent } = raced.value;
|
||||
// What the provider reported, plus (#1279) what the hops that failed before
|
||||
// it spent. The empty-content branch is charged too: those tokens were
|
||||
// burned upstream. No count falls back to the reservation, never to zero.
|
||||
const actualTokens =
|
||||
(result.tokensUsed ?? reservation.reserved) + tokensSpent;
|
||||
await reconcileSpend(userId, reservation.reserved, actualTokens, dateKey, 0, {
|
||||
servedBy: workingProvider.providerType,
|
||||
reservedOwner: reservation.owner,
|
||||
@@ -397,6 +390,44 @@ export async function runStatusCompletion(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A run that timed out or failed. It charges only what a reply reported
|
||||
* spending, never an invented figure; #1279 — a failed chain's report counts,
|
||||
* since a model that reasons without answering bills every call, and a full
|
||||
* refund let background work spend without the day's ceiling seeing it. A
|
||||
* chain that sent nothing (the bench) holds the note without counting it.
|
||||
*/
|
||||
async function failedRun(
|
||||
raced: { timedOut: boolean; error?: unknown },
|
||||
run: {
|
||||
userId: string;
|
||||
cacheAction: string;
|
||||
dateKey: string;
|
||||
reservation: { reserved: number; owner: BudgetCostOwner };
|
||||
},
|
||||
): Promise<StatusProviderResult> {
|
||||
const spent = spentOnFailure(raced.error);
|
||||
await reconcileSpend(
|
||||
run.userId,
|
||||
run.reservation.reserved,
|
||||
spent,
|
||||
run.dateKey,
|
||||
0,
|
||||
{
|
||||
servedBy: null,
|
||||
reservedOwner: run.reservation.owner,
|
||||
},
|
||||
);
|
||||
if ((raced.error as { sentNothing?: unknown } | null)?.sentNothing === true) {
|
||||
await holdStatusNoteWhileBenched({
|
||||
userId: run.userId,
|
||||
cacheAction: run.cacheAction,
|
||||
todayKey: run.dateKey,
|
||||
});
|
||||
}
|
||||
return { kind: raced.timedOut ? "timeout" : "error" };
|
||||
}
|
||||
|
||||
/**
|
||||
* Over the day's ceiling. Reported as `error` — a TRANSIENT miss the caller
|
||||
* serves the fallback for without persisting it — deliberately NOT `none`,
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
/**
|
||||
* The end of a provider's full-history backfill marks the read caches stale.
|
||||
*
|
||||
* The per-write tail always did (through `afterMeasurementMutation`), but the
|
||||
* backfill tail refolded the rollups and re-warmed the status cards without
|
||||
* touching the analytics caches, so the dashboard and the analytics reads went
|
||||
* on serving the pre-backfill body until its TTL ran out.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@/lib/rollups/measurement-rollups", () => ({
|
||||
collapseToTypeDayKeys: (keys: Array<{ type: string; measuredAt: Date }>) =>
|
||||
keys,
|
||||
recomputeUserRollups: vi.fn(async () => undefined),
|
||||
}));
|
||||
vi.mock("@/lib/cache/invalidate", () => ({
|
||||
invalidateUserMeasurements: vi.fn(),
|
||||
invalidateUserDashboardSnapshot: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/insights/status-invalidation", () => ({
|
||||
invalidateStatusInsightsForTypes: vi.fn(async () => undefined),
|
||||
}));
|
||||
vi.mock("@/lib/rollups/after-measurement-mutation", () => ({
|
||||
afterMeasurementMutation: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/arrivals/measurement-emit", () => ({
|
||||
emitInsertedMeasurementArrivals: vi.fn(async () => undefined),
|
||||
}));
|
||||
|
||||
import { invalidateUserMeasurements } from "@/lib/cache/invalidate";
|
||||
import { invalidateStatusInsightsForTypes } from "@/lib/insights/status-invalidation";
|
||||
import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups";
|
||||
|
||||
import { afterProviderBackfill } from "../after-write";
|
||||
|
||||
const keys = [
|
||||
{ type: "WEIGHT" as const, measuredAt: new Date("2026-03-05T00:00:00Z") },
|
||||
{ type: "PULSE" as const, measuredAt: new Date("2026-01-10T00:00:00Z") },
|
||||
{ type: "WEIGHT" as const, measuredAt: new Date("2026-02-01T00:00:00Z") },
|
||||
];
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("afterProviderBackfill", () => {
|
||||
it("marks the user's read caches stale back to the first touched day", async () => {
|
||||
await afterProviderBackfill({ userId: "u1", provider: "fitbit", keys });
|
||||
|
||||
expect(recomputeUserRollups).toHaveBeenCalledWith("u1", {
|
||||
types: ["WEIGHT", "PULSE"],
|
||||
from: new Date("2026-01-10T00:00:00Z"),
|
||||
to: new Date("2026-03-06T00:00:00Z"),
|
||||
});
|
||||
expect(invalidateUserMeasurements).toHaveBeenCalledTimes(1);
|
||||
expect(invalidateUserMeasurements).toHaveBeenCalledWith("u1", {
|
||||
touchedFrom: new Date("2026-01-10T00:00:00Z"),
|
||||
});
|
||||
// After the fold, so a reader rebuilds against the new rollups.
|
||||
expect(
|
||||
vi.mocked(invalidateUserMeasurements).mock.invocationCallOrder[0],
|
||||
).toBeGreaterThan(
|
||||
vi.mocked(recomputeUserRollups).mock.invocationCallOrder[0]!,
|
||||
);
|
||||
expect(invalidateStatusInsightsForTypes).toHaveBeenCalledWith(
|
||||
"u1",
|
||||
["WEIGHT", "PULSE"],
|
||||
{ source: "sync" },
|
||||
);
|
||||
});
|
||||
|
||||
it("marks them stale even when the fold fails, and still throws for the caller", async () => {
|
||||
vi.mocked(recomputeUserRollups).mockRejectedValueOnce(new Error("fold"));
|
||||
|
||||
await expect(
|
||||
afterProviderBackfill({ userId: "u1", provider: "fitbit", keys }),
|
||||
).rejects.toThrow("fold");
|
||||
expect(invalidateUserMeasurements).toHaveBeenCalledWith("u1", {
|
||||
touchedFrom: new Date("2026-01-10T00:00:00Z"),
|
||||
});
|
||||
});
|
||||
|
||||
it("does nothing for a backfill that touched nothing", async () => {
|
||||
await afterProviderBackfill({ userId: "u1", provider: "fitbit", keys: [] });
|
||||
expect(recomputeUserRollups).not.toHaveBeenCalled();
|
||||
expect(invalidateUserMeasurements).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -29,7 +29,10 @@ import {
|
||||
emitInsertedMeasurementArrivals,
|
||||
type InsertedMeasurementArrivalRow,
|
||||
} from "@/lib/arrivals/measurement-emit";
|
||||
import { invalidateUserDashboardSnapshot } from "@/lib/cache/invalidate";
|
||||
import {
|
||||
invalidateUserDashboardSnapshot,
|
||||
invalidateUserMeasurements,
|
||||
} from "@/lib/cache/invalidate";
|
||||
import { invalidateStatusInsightsForTypes } from "@/lib/insights/status-invalidation";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation";
|
||||
@@ -116,8 +119,14 @@ export function invalidateProviderStatus(
|
||||
|
||||
/**
|
||||
* The end of a full-history backfill: refold the whole touched range once
|
||||
* (from the first touched day to the end of the last), then re-warm the
|
||||
* touched status cards. Throws when the fold fails, for the caller to count.
|
||||
* (from the first touched day to the end of the last), mark the read caches
|
||||
* stale back to the first touched day, then re-warm the touched status cards.
|
||||
* Throws when the fold fails, for the caller to count.
|
||||
*
|
||||
* The caches are marked even when the fold fails: the rows changed either
|
||||
* way, and a cached analytics body built before the backfill would otherwise
|
||||
* be served until its TTL ran out. That is the same order and the same
|
||||
* posture as the per-write tail (`afterMeasurementMutation`).
|
||||
*/
|
||||
export async function afterProviderBackfill(input: {
|
||||
userId: string;
|
||||
@@ -135,6 +144,10 @@ export async function afterProviderBackfill(input: {
|
||||
// The keys are UTC day starts; the end is past the last touched day so the
|
||||
// aggregator's `< to` bound covers it.
|
||||
const to = new Date(sorted[sorted.length - 1]! + 24 * 60 * 60 * 1000);
|
||||
await recomputeUserRollups(input.userId, { types, from, to });
|
||||
try {
|
||||
await recomputeUserRollups(input.userId, { types, from, to });
|
||||
} finally {
|
||||
invalidateUserMeasurements(input.userId, { touchedFrom: from });
|
||||
}
|
||||
invalidateProviderStatus(input.userId, input.provider, types);
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* The worker's boot preload of every job handler module.
|
||||
*
|
||||
* Handlers load on their queue's first job, so a module that cannot load
|
||||
* stopped failing the boot and started failing jobs instead: the queue burned
|
||||
* its retries, and a one-shot job (a restore, an import tail) ended failed
|
||||
* with nobody told at deploy time. The worker now loads every handler once in
|
||||
* the background after binding and reports each failure by queue name.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { Job, PgBoss } from "pg-boss";
|
||||
|
||||
const bound = vi.hoisted(
|
||||
() => new Map<string, (jobs: unknown[]) => Promise<unknown>>(),
|
||||
);
|
||||
|
||||
vi.mock("@/lib/jobs/reminder/registrar-shared", () => ({
|
||||
createAndSchedule: vi.fn(async () => undefined),
|
||||
retireQueues: vi.fn(async () => undefined),
|
||||
createAndWork: vi.fn(
|
||||
async (
|
||||
_boss: unknown,
|
||||
queue: string,
|
||||
_opts: unknown,
|
||||
handler: (jobs: unknown[]) => Promise<unknown>,
|
||||
) => {
|
||||
bound.set(queue, handler);
|
||||
},
|
||||
),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/jobs/reminder/shared", () => ({ workerLog: vi.fn() }));
|
||||
|
||||
vi.mock("@/lib/logging/signal", () => ({ emitSignal: vi.fn() }));
|
||||
vi.mock("@/lib/jobs/report-worker-error", () => ({
|
||||
reportWorkerError: vi.fn(async () => undefined),
|
||||
}));
|
||||
vi.mock("@/lib/jobs/job-failures", () => ({
|
||||
readConsecutiveRunFailures: vi.fn(async () => 0),
|
||||
}));
|
||||
|
||||
import { workerLog } from "@/lib/jobs/reminder/shared";
|
||||
import { emitSignal } from "@/lib/logging/signal";
|
||||
import { reportWorkerError } from "@/lib/jobs/report-worker-error";
|
||||
import { reportHandlerLoadFailure } from "@/lib/jobs/run-job";
|
||||
|
||||
import { readSource } from "@/__tests__/helpers/source-files";
|
||||
|
||||
import type { JobDefinition, JobGroup } from "../define-job";
|
||||
import {
|
||||
preloadHandlers,
|
||||
registerJobGroup,
|
||||
type BoundLoader,
|
||||
} from "../register";
|
||||
|
||||
const boss = {} as PgBoss;
|
||||
|
||||
function definition(
|
||||
queue: string,
|
||||
load: () => Promise<unknown>,
|
||||
): JobDefinition {
|
||||
return { queue, work: { batchSize: 1 }, load } as unknown as JobDefinition;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
bound.clear();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("preloading the job handlers at worker boot", () => {
|
||||
it("reports a module that fails to load, by queue, and loads the rest", async () => {
|
||||
const ok = vi.fn(async () => async () => ({ ok: true as const }));
|
||||
const broken = vi.fn(async () => {
|
||||
throw new Error("Cannot find module './gone'");
|
||||
});
|
||||
const group = {
|
||||
name: "maintenance",
|
||||
jobs: [
|
||||
definition("first-ok", ok),
|
||||
definition("broken-module", broken),
|
||||
definition("second-ok", ok),
|
||||
],
|
||||
} as unknown as JobGroup;
|
||||
|
||||
const loaders: BoundLoader[] = [];
|
||||
await registerJobGroup(boss, group, (b) => loaders.push(b));
|
||||
expect(loaders.map((b) => b.queue)).toEqual([
|
||||
"first-ok",
|
||||
"broken-module",
|
||||
"second-ok",
|
||||
]);
|
||||
|
||||
const report = vi.fn(async () => undefined);
|
||||
const failed = await preloadHandlers(loaders, report);
|
||||
|
||||
expect(failed).toEqual(["broken-module"]);
|
||||
expect(report).toHaveBeenCalledTimes(1);
|
||||
expect(report).toHaveBeenCalledWith(
|
||||
"broken-module",
|
||||
expect.objectContaining({ message: "Cannot find module './gone'" }),
|
||||
);
|
||||
expect(ok).toHaveBeenCalledTimes(2);
|
||||
expect(workerLog).toHaveBeenCalledWith(
|
||||
"error",
|
||||
expect.stringContaining("failed to load: broken-module"),
|
||||
);
|
||||
});
|
||||
|
||||
it("preloads through the loader the queue is bound to, so the first job does not import again", async () => {
|
||||
const handler = vi.fn(async () => ({ ok: true as const }));
|
||||
const load = vi.fn(async () => handler);
|
||||
const group = {
|
||||
name: "maintenance",
|
||||
jobs: [definition("warm", load)],
|
||||
} as unknown as JobGroup;
|
||||
|
||||
const loaders: BoundLoader[] = [];
|
||||
await registerJobGroup(boss, group, (b) => loaders.push(b));
|
||||
await preloadHandlers(loaders, vi.fn());
|
||||
await bound.get("warm")!([{ id: "j1", data: {} } as Job<unknown>]);
|
||||
|
||||
expect(load).toHaveBeenCalledTimes(1);
|
||||
expect(handler).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not stop the sweep when a report itself fails", async () => {
|
||||
const loaders: BoundLoader[] = [
|
||||
{ queue: "a", load: () => Promise.reject(new Error("a")) },
|
||||
{ queue: "b", load: () => Promise.reject(new Error("b")) },
|
||||
];
|
||||
const report = vi.fn(async () => {
|
||||
throw new Error("sink down");
|
||||
});
|
||||
await expect(preloadHandlers(loaders, report)).resolves.toEqual(["a", "b"]);
|
||||
expect(report).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("reports a load failure on the job.run.failed line, at error, naming the queue", async () => {
|
||||
const error = new Error("SyntaxError in handler");
|
||||
await reportHandlerLoadFailure("broken-module", error);
|
||||
|
||||
expect(emitSignal).toHaveBeenCalledWith({
|
||||
action: "job.run.failed",
|
||||
level: "error",
|
||||
meta: {
|
||||
queue: "broken-module",
|
||||
reason: "handler_load_failed",
|
||||
consecutiveFailures: 0,
|
||||
},
|
||||
error,
|
||||
});
|
||||
expect(reportWorkerError).toHaveBeenCalledWith("broken-module", error, {
|
||||
phase: "handler_preload",
|
||||
});
|
||||
});
|
||||
|
||||
it("is started by the worker after binding, without holding up the boot", () => {
|
||||
const source = readSource("src/lib/jobs/reminder-worker.ts", [
|
||||
"registry.registerAllJobGroups(",
|
||||
"void registry.preloadHandlers(bound)",
|
||||
]);
|
||||
const register = source.indexOf("registry.registerAllJobGroups(");
|
||||
const preload = source.indexOf("void registry.preloadHandlers(bound)");
|
||||
expect(register).toBeGreaterThan(-1);
|
||||
expect(preload).toBeGreaterThan(register);
|
||||
});
|
||||
});
|
||||
@@ -10,7 +10,13 @@ import type { JobDefinition, JobGroup } from "./define-job";
|
||||
import { INTEGRATION_SYNC_GROUP } from "./integration-sync";
|
||||
import { MAINTENANCE_GROUP } from "./maintenance";
|
||||
import { REMINDERS_GROUP } from "./reminders";
|
||||
import { registerJobGroup, runBootDiscovery } from "./register";
|
||||
import {
|
||||
registerJobGroup,
|
||||
runBootDiscovery,
|
||||
type BoundLoader,
|
||||
} from "./register";
|
||||
|
||||
export { preloadHandlers, type BoundLoader } from "./register";
|
||||
import { ROLLUP_GROUP } from "./rollup";
|
||||
import { STATUS_GROUP } from "./status";
|
||||
|
||||
@@ -29,16 +35,18 @@ export function allJobDefinitions(): readonly JobDefinition[] {
|
||||
|
||||
/**
|
||||
* Register every group in order. `onGroup` runs before each one, so the boot
|
||||
* can time the groups as steps. Returns every queue name created.
|
||||
* can time the groups as steps; `onBound` receives every queue's handler
|
||||
* loader as it is bound. Returns every queue name created.
|
||||
*/
|
||||
export async function registerAllJobGroups(
|
||||
boss: PgBoss,
|
||||
onGroup: (group: JobGroup) => void = () => {},
|
||||
onBound: (bound: BoundLoader) => void = () => {},
|
||||
): Promise<string[]> {
|
||||
const queues: string[] = [];
|
||||
for (const group of JOB_GROUPS) {
|
||||
onGroup(group);
|
||||
queues.push(...(await registerJobGroup(boss, group)));
|
||||
queues.push(...(await registerJobGroup(boss, group, onBound)));
|
||||
}
|
||||
return queues;
|
||||
}
|
||||
|
||||
@@ -13,6 +13,14 @@
|
||||
* queue's protections do not depend on whether its module has loaded yet. A
|
||||
* module that fails to load fails the job that asked for it, with the queue's
|
||||
* name on the error, and the next job tries the import again.
|
||||
*
|
||||
* Loading lazily moved a broken module from a failed boot to a failed job: the
|
||||
* worker comes up healthy, and the queue burns its retries the first time it
|
||||
* has work, which for a one-shot job (a restore, an import tail) means the
|
||||
* job ends failed with nobody told at deploy time. `preloadHandlers` closes
|
||||
* that gap. The worker runs it once after binding, in the background, through
|
||||
* the same memoised loader the queue uses, and reports every module that does
|
||||
* not load before any job needs it.
|
||||
*/
|
||||
import type { Job, PgBoss } from "pg-boss";
|
||||
|
||||
@@ -26,6 +34,7 @@ import {
|
||||
type ScheduleEntry,
|
||||
} from "@/lib/jobs/reminder/registrar-shared";
|
||||
import { workerLog } from "@/lib/jobs/reminder/shared";
|
||||
import { reportHandlerLoadFailure } from "@/lib/jobs/run-job";
|
||||
|
||||
import type {
|
||||
BatchHandler,
|
||||
@@ -35,6 +44,34 @@ import type {
|
||||
JobGroup,
|
||||
} from "./define-job";
|
||||
|
||||
/** A queue's handler loader, memoised: one import, retried after a failure. */
|
||||
export type HandlerLoader = () => Promise<BatchHandler<Job<unknown>>>;
|
||||
|
||||
/** One bound queue and the loader its handler goes through. */
|
||||
export interface BoundLoader {
|
||||
readonly queue: string;
|
||||
readonly load: HandlerLoader;
|
||||
}
|
||||
|
||||
/**
|
||||
* The loader a definition's binding shares: the first call imports, every
|
||||
* later call answers from the same promise, and a failed import is forgotten
|
||||
* so the next call tries again.
|
||||
*/
|
||||
export function handlerLoader(
|
||||
definition: JobDefinition,
|
||||
ctx: JobContext,
|
||||
): HandlerLoader {
|
||||
let loading: Promise<BatchHandler<Job<unknown>>> | null = null;
|
||||
return () => {
|
||||
loading ??= definition.load(ctx).catch((err: unknown) => {
|
||||
loading = null;
|
||||
throw err;
|
||||
});
|
||||
return loading;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The handler a definition binds: its loaded module behind the lock and the
|
||||
* lane it declares.
|
||||
@@ -42,15 +79,8 @@ import type {
|
||||
export function bindableHandler(
|
||||
definition: JobDefinition,
|
||||
ctx: JobContext,
|
||||
load: HandlerLoader = handlerLoader(definition, ctx),
|
||||
): BatchHandler<Job<unknown>> {
|
||||
let loading: Promise<BatchHandler<Job<unknown>>> | null = null;
|
||||
const load = (): Promise<BatchHandler<Job<unknown>>> => {
|
||||
loading ??= definition.load(ctx).catch((err: unknown) => {
|
||||
loading = null;
|
||||
throw err;
|
||||
});
|
||||
return loading;
|
||||
};
|
||||
let handler: BatchHandler<Job<unknown>> = async (jobs) =>
|
||||
(await load())(jobs);
|
||||
if (definition.lane) {
|
||||
@@ -88,11 +118,13 @@ export function policiesOf(jobs: readonly JobDefinition[]): QueuePolicyTable {
|
||||
|
||||
/**
|
||||
* Register one group: create, schedule, retire, bind. Returns the queue names
|
||||
* it created.
|
||||
* it created. `onBound` is handed each queue's loader as it is bound, so the
|
||||
* boot can preload through the very loader the queue will use.
|
||||
*/
|
||||
export async function registerJobGroup(
|
||||
boss: PgBoss,
|
||||
group: JobGroup,
|
||||
onBound: (bound: BoundLoader) => void = () => {},
|
||||
): Promise<readonly string[]> {
|
||||
const queues = group.jobs.map((job) => job.queue);
|
||||
await createAndSchedule(
|
||||
@@ -104,7 +136,14 @@ export async function registerJobGroup(
|
||||
if (group.retired) await retireQueues(boss, group.retired);
|
||||
const ctx: JobContext = { boss };
|
||||
for (const job of group.jobs) {
|
||||
await createAndWork(boss, job.queue, job.work, bindableHandler(job, ctx));
|
||||
const load = handlerLoader(job, ctx);
|
||||
await createAndWork(
|
||||
boss,
|
||||
job.queue,
|
||||
job.work,
|
||||
bindableHandler(job, ctx, load),
|
||||
);
|
||||
onBound({ queue: job.queue, load });
|
||||
}
|
||||
if (group.afterBind) await group.afterBind(ctx);
|
||||
return queues;
|
||||
@@ -144,3 +183,45 @@ export async function runBootDiscovery(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load every bound handler once, in declaration order, and report each one
|
||||
* that fails: `job.run.failed` at error with the queue's name and the reason
|
||||
* `handler_load_failed`, the line the repeated-failure alert already keys on,
|
||||
* plus the worker error report (stderr and GlitchTip). Never throws, and the
|
||||
* caller does not wait for it, so a broken module costs the boot nothing but
|
||||
* the report. A module that loads stays loaded, and the queue's first job
|
||||
* finds it in the loader's memo. Returns the queues that failed.
|
||||
*/
|
||||
export async function preloadHandlers(
|
||||
bound: readonly BoundLoader[],
|
||||
report: (
|
||||
queue: string,
|
||||
error: unknown,
|
||||
) => Promise<void> = reportHandlerLoadFailure,
|
||||
): Promise<string[]> {
|
||||
const failed: string[] = [];
|
||||
for (const { queue, load } of bound) {
|
||||
try {
|
||||
await load();
|
||||
} catch (error) {
|
||||
failed.push(queue);
|
||||
try {
|
||||
await report(queue, error);
|
||||
} catch (reportError) {
|
||||
// The report failing must not stop the sweep; say so on stderr.
|
||||
workerLog(
|
||||
"error",
|
||||
`[registry] reporting the load failure of ${queue} failed`,
|
||||
reportError,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
workerLog(
|
||||
failed.length === 0 ? "info" : "error",
|
||||
`[registry] preloaded ${bound.length - failed.length}/${bound.length} job handlers` +
|
||||
(failed.length === 0 ? "" : `; failed to load: ${failed.join(", ")}`),
|
||||
);
|
||||
return failed;
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import { reportJobsCutOffAtBoot } from "@/lib/jobs/job-observer";
|
||||
import { beginBootStep, reportBootSteps } from "@/lib/boot/boot-steps";
|
||||
import { assertSubsystemEnabled } from "@/lib/process-type";
|
||||
import { DATABASE_URL, workerLog } from "./reminder/shared";
|
||||
import type { BoundLoader } from "./registry/index";
|
||||
|
||||
/**
|
||||
* Give the event loop a turn around a module load, so requests and the
|
||||
@@ -76,8 +77,11 @@ export async function startReminderWorker() {
|
||||
"registry",
|
||||
() => import("./registry/index"),
|
||||
);
|
||||
const registeredQueues = await registry.registerAllJobGroups(boss, (group) =>
|
||||
beginBootStep(`register_${group.name}`),
|
||||
const bound: BoundLoader[] = [];
|
||||
const registeredQueues = await registry.registerAllJobGroups(
|
||||
boss,
|
||||
(group) => beginBootStep(`register_${group.name}`),
|
||||
(loader) => bound.push(loader),
|
||||
);
|
||||
workerLog(
|
||||
"info",
|
||||
@@ -100,6 +104,11 @@ export async function startReminderWorker() {
|
||||
await registry.runAllBootDiscovery();
|
||||
|
||||
await reportBootSteps({ boot_queue_count: registeredQueues.length });
|
||||
|
||||
// Every handler module is loaded once now, in the background, so one that
|
||||
// cannot load is reported at deploy time rather than on its queue's first
|
||||
// job. Not awaited: readiness does not wait on it, and it never throws.
|
||||
void registry.preloadHandlers(bound);
|
||||
return boss;
|
||||
}
|
||||
|
||||
|
||||
@@ -90,6 +90,20 @@ async function signalRunFailed(
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* A queue's handler module that failed to load before any job asked for it
|
||||
* (the worker's boot preload, `preloadHandlers`). Reported the way a failed
|
||||
* run is, so the operator hears about it at deploy time on the line the
|
||||
* repeated-failure alert already reads, instead of on the queue's first job.
|
||||
*/
|
||||
export async function reportHandlerLoadFailure(
|
||||
queue: string,
|
||||
error: unknown,
|
||||
): Promise<void> {
|
||||
await reportWorkerError(queue, error, { phase: "handler_preload" });
|
||||
await signalRunFailed(queue, "handler_load_failed", error);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap a handler for `boss.work`. Resolves with the bounded, redacted
|
||||
* persistence representation on `ok: true`; reports and rethrows on
|
||||
|
||||
@@ -56,9 +56,14 @@ const inFlight = new Set<Promise<unknown>>();
|
||||
export function trackBackgroundTask(settled: Promise<unknown>): void {
|
||||
if (!TRACK_IN_FLIGHT) return;
|
||||
|
||||
inFlight.add(settled);
|
||||
void settled.finally(() => {
|
||||
inFlight.delete(settled);
|
||||
// A test double that stands in for the background work often returns
|
||||
// nothing at all. Filed as is, `undefined` would never leave the set and
|
||||
// `settleBackgroundTasks` would spin on it forever at full CPU. A real
|
||||
// promise passes through `Promise.resolve` as the same object.
|
||||
const handle = Promise.resolve(settled);
|
||||
inFlight.add(handle);
|
||||
void handle.finally(() => {
|
||||
inFlight.delete(handle);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
Loaded 100 of 134 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user