diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 54946f446..efb542eeb 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -15429,45 +15429,6 @@ paths: "403": *a9 "422": *a4 "429": *a5 - /api/settings/preview-features: - get: - tags: - - Meta - summary: Get the preview-feature opt-ins - description: Every flag, off unless the caller switched it on. - responses: - "200": - description: The opt-ins. - content: - application/json: - schema: - $ref: "#/components/schemas/PreviewFeaturesEnvelope" - "401": *a1 - "403": *a9 - "422": *a4 - "429": *a5 - patch: - tags: - - Meta - summary: Switch preview features on or off - description: Sets the flags the body names and keeps the rest. An unknown flag or an empty body is refused with 422. - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/PreviewFeaturesPatchBody" - responses: - "200": - description: The opt-ins as stored. - content: - application/json: - schema: - $ref: "#/components/schemas/PreviewFeaturesSavedEnvelope" - "401": *a1 - "403": *a9 - "422": *a4 - "429": *a5 /api/settings/privacy-summary: get: tags: @@ -27216,6 +27177,61 @@ paths: application/json: schema: $ref: "#/components/schemas/ErrorEnvelope" + /api/settings/preview-features: + get: + security: [] + deprecated: true + tags: + - Retired + summary: GET /api/settings/preview-features — removed in v1.46.0 + description: >- + Removed in v1.46.0. Every request to this path answers 410 Gone, whatever the method and whatever credential it + carries. There is no replacement; the capability is gone rather than moved. + + + Stored per-account opt-ins to preview features, of which there was one, the topic overview pages. Those pages + were dropped before they shipped, so the switch governed nothing; the route and its column are gone, and no + preview is gated per account. + + + Retiring a path is deliberate, so the answer says so rather than leaving a client to read a 404 as a broken + deployment and retry forever. Treat 410 as terminal: drop the call, degrade the surface that depended on it, and + do not offer a retry. + responses: + "410": + description: The path was removed in v1.46.0 and will not answer again. `meta.errorCode` = `route.retired`, + `meta.removedIn` carries the version, and `meta.replacedBy` carries the replacement path or null. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" + patch: + security: [] + deprecated: true + tags: + - Retired + summary: PATCH /api/settings/preview-features — removed in v1.46.0 + description: >- + Removed in v1.46.0. Every request to this path answers 410 Gone, whatever the method and whatever credential it + carries. There is no replacement; the capability is gone rather than moved. + + + Stored per-account opt-ins to preview features, of which there was one, the topic overview pages. Those pages + were dropped before they shipped, so the switch governed nothing; the route and its column are gone, and no + preview is gated per account. + + + Retiring a path is deliberate, so the answer says so rather than leaving a client to read a 404 as a broken + deployment and retry forever. Treat 410 as terminal: drop the call, degrade the surface that depended on it, and + do not offer a retry. + responses: + "410": + description: The path was removed in v1.46.0 and will not answer again. `meta.errorCode` = `route.retired`, + `meta.removedIn` carries the version, and `meta.replacedBy` carries the replacement path or null. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" components: schemas: CreateCycleCustomSymptomRequest: @@ -33268,12 +33284,6 @@ components: - version - list additionalProperties: false - PreviewFeaturesPatchBody: - type: object - properties: - topics: - type: boolean - additionalProperties: false ConsentPostBody: type: object properties: @@ -73603,50 +73613,6 @@ components: - data - error additionalProperties: false - PreviewFeaturesEnvelope: - type: object - properties: - data: - $ref: "#/components/schemas/PreviewFeatures" - error: - type: "null" - meta: - type: object - properties: - requestId: - type: string - additionalProperties: false - required: - - data - - error - additionalProperties: false - PreviewFeatures: - type: object - properties: - topics: - type: boolean - description: Topic overview pages above the single-metric Insights pages. Off by default. - required: - - topics - additionalProperties: false - description: Per-user opt-in to preview features, one boolean per flag, every flag off by default. - PreviewFeaturesSavedEnvelope: - type: object - properties: - data: - $ref: "#/components/schemas/PreviewFeatures" - error: - type: "null" - meta: - type: object - properties: - requestId: - type: string - additionalProperties: false - required: - - data - - error - additionalProperties: false PrivacySummaryEnvelope: type: object properties: diff --git a/docs/ops/deploy.md b/docs/ops/deploy.md index 4f44fa1f3..ce8c89dbc 100644 --- a/docs/ops/deploy.md +++ b/docs/ops/deploy.md @@ -251,6 +251,21 @@ Pass `--only=prod` / `--only=demo` to check a single target, or `--attempts=N --interval=ms` to widen the retry window while a slow pull settles. +The worker loads each job's handler module on that job's first run, so a +module the image cannot load does not stop the worker from starting. Once +it has bound its queues, the worker loads every handler a single time in +the background and reports each one that fails, normally within a minute +of start. Each failure is a `job.run.failed` event at `error` with +`reason: "handler_load_failed"` and the queue's name, and stderr also gets +`[pg-boss] [registry] preloaded N/M job handlers; failed to load: `. +If either shows up after a deploy, the image is broken for those queues, +and their jobs will fail when they run. Roll back (see Rollback above) +instead of waiting for the retries to run out: + +```sh +docker compose logs --since 5m app 2>&1 | grep -E 'handler_load_failed|failed to load:' +``` + ## Walk the synthetic journey The version check proves the instance serves the right build. It does not diff --git a/prisma/migrations/0485_drop_user_preview_features/migration.sql b/prisma/migrations/0485_drop_user_preview_features/migration.sql new file mode 100644 index 000000000..93c04cbf8 --- /dev/null +++ b/prisma/migrations/0485_drop_user_preview_features/migration.sql @@ -0,0 +1,27 @@ +-- Drop the per-user preview-feature opt-ins. +-- +-- `users.preview_features_json` (0440) stored one flag, `topics`, for the +-- topic overview pages. Those pages were dropped before they shipped, so no +-- surface ever read the flag; `/api/settings/preview-features` is retired +-- with this release and answers 410. +-- +-- What is lost is a switch that switched nothing. A backup written by a +-- release that still had the column carries the field, and the restore +-- accepts and ignores it. +-- +-- Dropping a column takes an ACCESS EXCLUSIVE lock on `users` for a catalogue +-- update only (no rewrite). The lock wait is bounded so a long-running reader +-- cannot queue every sign-in behind this statement; the deploy retries a +-- timed-out attempt. +-- +-- Reversibility: re-adding the nullable column restores the shape; every +-- account then reads as all flags off, which is what every account had. +-- +-- @retry-on-lock-timeout + +DO $migration$ +BEGIN + PERFORM set_config('lock_timeout', '5s', true); + ALTER TABLE "users" DROP COLUMN IF EXISTS "preview_features_json"; +END +$migration$; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 3312190bd..8397e0cb5 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -387,11 +387,6 @@ model User { // copies a stored browser value up once when the server has none for that list. // Shape: { "version": 1, "labs": { "view": "list", "order": [""], "sortDir": "manual" }, ... } moduleListLayoutJson Json? @map("module_list_layout_json") - /// Per-user opt-in to preview features, one boolean per flag, all off by - /// default (`src/lib/settings/preview-features.ts`). Read and written - /// through `/api/settings/preview-features` and carried by the backup. - /// @pending-consumer(topics preview) the `topics` flag is stored, but no surface reads it and the Settings > System switch is unlisted until the topic overview pages ship. - previewFeaturesJson Json? @map("preview_features_json") /// Optional activity goals the person set for themselves: workouts, active /// minutes and steps per week, steps per month. Null or an absent key = no /// goal. Parsed by `src/lib/validations/activity-goals.ts`, written through diff --git a/src/__tests__/ai-call-timeout-guard.test.ts b/src/__tests__/ai-call-timeout-guard.test.ts index 452926bcd..6e205ce40 100644 --- a/src/__tests__/ai-call-timeout-guard.test.ts +++ b/src/__tests__/ai-call-timeout-guard.test.ts @@ -56,12 +56,15 @@ function code(rel: string): string { return stripComments(readFileSync(join(SRC, rel), "utf8")); } -/** The wire clients: every `*-client.ts` that dials through `safeFetch`. */ +/** + * The wire clients: every `*-client.ts` that dials through `safeFetch`, or + * through the call's request budget (`hops.fetch`, `request-hops.ts`). + */ function wireClients(): string[] { return readdirSync(AI_DIR) .filter((f) => f.endsWith("-client.ts")) .map((f) => `lib/ai/${f}`) - .filter((rel) => /\bsafeFetch\s*\(/.test(code(rel))) + .filter((rel) => /\b(?:safeFetch|hops\.fetch)\s*\(/.test(code(rel))) .sort(); } diff --git a/src/__tests__/ai-request-hops-guard.test.ts b/src/__tests__/ai-request-hops-guard.test.ts new file mode 100644 index 000000000..d770c3261 --- /dev/null +++ b/src/__tests__/ai-request-hops-guard.test.ts @@ -0,0 +1,127 @@ +/** + * Structural guard (#1279): a provider client that sends a request again on + * its own sends every request through the call's budget, and hands that + * budget to every repeat. + * + * A client that repeats a request (`this.generateCompletion(` or + * `this.generateCompletionStream(` inside its own class) decided each repeat + * from what it had just learned, and two lessons that undid each other made + * one call send requests until something outside stopped it. The budget in + * `src/lib/ai/request-hops.ts` refuses the request past two; it can only do + * that if no request goes around it and no repeat starts a fresh one. So, in + * every such client: + * + * - no request goes out through `safeFetch(` directly, only `hops.fetch(`; + * - every repeat passes `hops` on; + * - each public entry takes the budget as its last parameter, defaulting + * to a fresh one, so a caller outside starts one per call. + * + * `request-hops.test.ts` walks the refusal and answerless combinations + * against a loopback server and counts the requests. + * + * Its limit: it reads source text. A client that repeats through a helper + * that is not a method named `generateCompletion*`, or sends through another + * transport, is not found here. + * + * Mutation check: drop `hops` from one repeat, or put one `safeFetch(` back + * into either client, and this goes red. + */ +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + readSourceText, + requireFloor, + stripComments, + walkSourceFiles, +} from "./helpers/source-files"; + +const AI = join(process.cwd(), "src/lib/ai"); + +/** The text of the call that starts at `open` (the index of its `(`). */ +function callText(source: string, open: number): string { + let depth = 0; + for (let i = open; i < source.length; i += 1) { + if (source[i] === "(") depth += 1; + else if (source[i] === ")") { + depth -= 1; + if (depth === 0) return source.slice(open, i + 1); + } + } + return source.slice(open); +} + +const REPEAT = /\bthis\.generateCompletion(?:Stream)?\s*\(/g; + +function repeatingClients(): Array<{ rel: string; source: string }> { + const out: Array<{ rel: string; source: string }> = []; + for (const rel of walkSourceFiles(AI, { floor: 40 })) { + if (rel.includes("__tests__/")) continue; + const source = stripComments(readSourceText(join(AI, rel))); + if (REPEAT.test(source)) out.push({ rel, source }); + REPEAT.lastIndex = 0; + } + return out; +} + +describe("a client's own repeats stay inside the call's request budget", () => { + const clients = repeatingClients(); + + it("finds the clients that repeat a request", () => { + // The Local client and the OpenAI-compatible gateway client today. + requireFloor("repeating clients", clients, 2); + expect(clients.map((c) => c.rel).sort()).toEqual([ + "local-client.ts", + "openai-client.ts", + ]); + }); + + it("sends every request through the budget", () => { + for (const { rel, source } of clients) { + expect({ rel, direct: /\bsafeFetch\s*\(/.test(source) }).toEqual({ + rel, + direct: false, + }); + expect({ rel, counted: /\bhops\.fetch\s*\(/.test(source) }).toEqual({ + rel, + counted: true, + }); + } + }); + + it("hands the budget to every repeat", () => { + const repeats: Array<{ rel: string; text: string }> = []; + for (const { rel, source } of clients) { + for (const m of source.matchAll(REPEAT)) { + const open = (m.index ?? 0) + m[0].length - 1; + repeats.push({ rel, text: callText(source, open) }); + } + } + requireFloor("repeats", repeats, 6); + expect( + repeats.filter(({ text }) => !/,\s*hops\s*,?\s*\)$/.test(text)), + ).toEqual([]); + }); + + it("starts one budget per call at each public entry", () => { + const entries: Array<{ rel: string; text: string }> = []; + for (const { rel, source } of clients) { + for (const m of source.matchAll( + /\basync\s+generateCompletion(?:Stream)?\s*\(/g, + )) { + const open = (m.index ?? 0) + m[0].length - 1; + entries.push({ rel, text: callText(source, open) }); + } + } + requireFloor("public entries", entries, 3); + expect( + entries.filter( + ({ text }) => + !/hops:\s*RequestHops\s*=\s*new RequestHops\(\)\s*,?\s*\)$/.test( + text, + ), + ), + ).toEqual([]); + }); +}); diff --git a/src/__tests__/background-task-settle-guard.test.ts b/src/__tests__/background-task-settle-guard.test.ts index e6b564813..dad2f0e28 100644 --- a/src/__tests__/background-task-settle-guard.test.ts +++ b/src/__tests__/background-task-settle-guard.test.ts @@ -142,6 +142,14 @@ describe("background-task settle contract", () => { expect(pendingBackgroundTaskCount()).toBe(0); }); + it("drains a handle that is not a promise instead of spinning on it", async () => { + // A mocked background starter that returns nothing hands the registry + // `undefined`; left in the set, settle would loop on it forever. + trackBackgroundTask(undefined as unknown as Promise); + await settleBackgroundTasks(); + expect(pendingBackgroundTaskCount()).toBe(0); + }); + it("registers what fireAndForget starts", async () => { let release: () => void = () => {}; fireAndForget( diff --git a/src/__tests__/baselines/source-file-lines.json b/src/__tests__/baselines/source-file-lines.json index 4b1bff5d6..d57271904 100644 --- a/src/__tests__/baselines/source-file-lines.json +++ b/src/__tests__/baselines/source-file-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", - "count": 87, - "total": 97455, + "count": 86, + "total": 96631, "entries": { "app/medications/page-client.tsx": 839, "components/charts/mood-chart.tsx": 1166, @@ -29,7 +29,6 @@ "lib/ai/coach/stream-events.ts": 817, "lib/ai/coach/types.ts": 1011, "lib/ai/codex-client.ts": 1045, - "lib/ai/local-client.ts": 824, "lib/ai/prompts/insight-generator.ts": 1110, "lib/ai/provider.ts": 1632, "lib/analytics/score/reader.ts": 998, diff --git a/src/__tests__/baselines/source-function-lines.json b/src/__tests__/baselines/source-function-lines.json index dbf457494..e29dc0c14 100644 --- a/src/__tests__/baselines/source-function-lines.json +++ b/src/__tests__/baselines/source-function-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", - "count": 350, - "total": 113403, + "count": 349, + "total": 113048, "entries": { "app/achievements/page.tsx :: AchievementsPage": 206, "app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 314, @@ -31,7 +31,6 @@ "app/api/medications/[id]/dose-history/route.ts :: apiHandler(arg)": 256, "app/api/medications/[id]/inventory/[itemId]/route.ts :: apiHandler(arg)": 204, "app/api/medications/[id]/schedule-revisions/[revisionId]/route.ts :: apiHandler(arg)": 250, - "app/api/medications/extract/route.ts :: handleExtract": 201, "app/api/mental-health/assessments/route.ts :: postAssessment": 249, "app/api/mood-entries/[id]/route.ts :: apiHandler(arg) #2": 296, "app/api/mood-entries/route.ts :: postMoodEntry": 324, @@ -250,12 +249,12 @@ "lib/ai/coach/glp1-snapshot.ts :: buildGlp1SnapshotBlock": 218, "lib/ai/coach/snapshot.ts :: buildCoachSnapshotImpl": 881, "lib/ai/coach/tools/loop.ts :: runCoachToolLoop": 438, - "lib/ai/coach/turn/model.ts :: runTurnModel": 540, + "lib/ai/coach/turn/model.ts :: runTurnModel": 538, "lib/ai/coach/turn/pipeline.ts :: runTurn": 274, "lib/ai/coach/turn/reply-guards.ts :: guardReply": 331, "lib/ai/codex-client.ts :: parseResponse": 302, - "lib/ai/local-client.ts :: generateCompletionStream": 294, - "lib/ai/openai-client.ts :: generateCompletion": 335, + "lib/ai/local-client.ts :: generateCompletionStream": 222, + "lib/ai/openai-client.ts :: generateCompletion": 266, "lib/analytics/bp-in-target-fast-path.ts :: computeFromRollups": 203, "lib/analytics/correlations-fast-path.ts :: computeCorrelationHypothesesFastPath": 256, "lib/analytics/intraday-pulse-io.ts :: loadIntradayPulse": 206, @@ -297,7 +296,7 @@ "lib/insights/comprehensive-generate.ts :: generateComprehensiveInsight": 876, "lib/insights/features.ts :: extractFeatures": 796, "lib/insights/mood-aggregates.ts :: fetchMoodAggregates": 246, - "lib/insights/status-provider.ts :: runStatusCompletion": 226, + "lib/insights/status-provider.ts :: runStatusCompletion": 215, "lib/integrations/connection-health-load.ts :: loadConnectionFacts": 227, "lib/integrations/status.ts :: recordSyncFailure": 292, "lib/jobs/apple-health-import-worker.ts :: handleAppleHealthImport": 300, diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index 9bb4f7a09..d329e4c03 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", "count": 2897, - "total": 9270, + "total": 9249, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -175,9 +175,9 @@ "app/api/auth/refresh/route.ts": 2, "app/api/auth/register/__tests__/route.test.ts": 5, "app/api/auth/register/route.ts": 10, - "app/api/biomarkers/__tests__/error-contract.test.ts": 1, "app/api/biomarkers/[id]/__tests__/route.test.ts": 2, "app/api/biomarkers/[id]/route.ts": 6, + "app/api/biomarkers/__tests__/error-contract.test.ts": 1, "app/api/biomarkers/route.ts": 4, "app/api/c/[token]/unlock/__tests__/route.test.ts": 1, "app/api/c/[token]/unlock/route.ts": 1, @@ -187,12 +187,12 @@ "app/api/coach/about-me/questions/__tests__/route-module-gate.test.ts": 1, "app/api/coach/about-me/questions/route.ts": 3, "app/api/coach/about-me/route.ts": 4, - "app/api/coach/plans/__tests__/route.test.ts": 1, "app/api/coach/plans/[id]/route.ts": 2, + "app/api/coach/plans/__tests__/route.test.ts": 1, "app/api/coach/plans/route.ts": 1, "app/api/coach/reminder-suggestions/route.ts": 1, - "app/api/coach/reminders/__tests__/route.test.ts": 1, "app/api/coach/reminders/[id]/route.ts": 1, + "app/api/coach/reminders/__tests__/route.test.ts": 1, "app/api/coach/reminders/route.ts": 1, "app/api/coach/suggested-actions/__tests__/route.test.ts": 1, "app/api/coach/suggested-actions/route.ts": 1, @@ -200,9 +200,9 @@ "app/api/consent/ai/latest/__tests__/route.test.ts": 3, "app/api/consent/ai/latest/route.ts": 3, "app/api/consent/ai/route.ts": 3, + "app/api/custom-metrics/[id]/entries/[entryId]/route.ts": 2, "app/api/custom-metrics/[id]/entries/__tests__/note-encryption.test.ts": 1, "app/api/custom-metrics/[id]/entries/__tests__/soft-delete-restore.test.ts": 1, - "app/api/custom-metrics/[id]/entries/[entryId]/route.ts": 2, "app/api/custom-metrics/[id]/entries/route.ts": 4, "app/api/custom-metrics/[id]/route.ts": 1, "app/api/custom-metrics/route.ts": 1, @@ -212,8 +212,8 @@ "app/api/cycle/day-logs/[id]/route.ts": 4, "app/api/cycle/day-logs/route.ts": 1, "app/api/cycle/period/route.ts": 4, - "app/api/cycle/symptoms/custom/__tests__/custom-crud.test.ts": 1, "app/api/cycle/symptoms/custom/[key]/route.ts": 1, + "app/api/cycle/symptoms/custom/__tests__/custom-crud.test.ts": 1, "app/api/cycle/symptoms/custom/route.ts": 3, "app/api/dashboard/chart-overlay-prefs/__tests__/route.test.ts": 1, "app/api/dashboard/chart-overlay-prefs/route.ts": 2, @@ -223,11 +223,10 @@ "app/api/dashboard/summary/route.ts": 5, "app/api/dashboard/widgets/__tests__/route.test.ts": 4, "app/api/dashboard/widgets/route.ts": 14, - "app/api/devices/__tests__/route.test.ts": 1, "app/api/devices/[id]/__tests__/route.test.ts": 1, "app/api/devices/[id]/route.ts": 1, + "app/api/devices/__tests__/route.test.ts": 1, "app/api/devices/route.ts": 4, - "app/api/documents/inbound/__tests__/route.test.ts": 1, "app/api/documents/inbound/[id]/__tests__/route.test.ts": 1, "app/api/documents/inbound/[id]/chat/__tests__/route.test.ts": 2, "app/api/documents/inbound/[id]/chat/route.ts": 1, @@ -242,6 +241,7 @@ "app/api/documents/inbound/[id]/suggest/route.ts": 2, "app/api/documents/inbound/[id]/summary/route.ts": 2, "app/api/documents/inbound/[id]/thumbnail/route.ts": 1, + "app/api/documents/inbound/__tests__/route.test.ts": 1, "app/api/documents/inbound/reindex/route.ts": 1, "app/api/documents/inbound/route.ts": 2, "app/api/documents/inbound/usage/route.ts": 1, @@ -260,10 +260,10 @@ "app/api/export/mood/route.ts": 2, "app/api/family-history/[id]/route.ts": 2, "app/api/family-history/route.ts": 1, + "app/api/fhir/Patient/$everything/route.ts": 1, "app/api/fhir/__tests__/observation-series.test.ts": 1, "app/api/fhir/__tests__/route.test.ts": 1, "app/api/fhir/metadata/route.ts": 1, - "app/api/fhir/Patient/$everything/route.ts": 1, "app/api/fitbit/callback/route.ts": 1, "app/api/fitbit/connect/route.ts": 1, "app/api/fitbit/credentials/route.ts": 1, @@ -292,6 +292,11 @@ "app/api/insights/breathing-screening/route.ts": 2, "app/api/insights/cards/__tests__/route.test.ts": 2, "app/api/insights/cards/route.ts": 9, + "app/api/insights/chat/[id]/attachments/[documentId]/route.ts": 1, + "app/api/insights/chat/[id]/attachments/__tests__/route.test.ts": 1, + "app/api/insights/chat/[id]/attachments/route.ts": 1, + "app/api/insights/chat/[id]/messages/[messageId]/trail/__tests__/route.test.ts": 1, + "app/api/insights/chat/[id]/route.ts": 1, "app/api/insights/chat/__tests__/dialog-harness.ts": 1, "app/api/insights/chat/__tests__/golden-transcripts.test.ts": 1, "app/api/insights/chat/__tests__/route-activity.test.ts": 1, @@ -303,19 +308,14 @@ "app/api/insights/chat/__tests__/route-snapshot-once.test.ts": 2, "app/api/insights/chat/__tests__/route-tool-mode.test.ts": 4, "app/api/insights/chat/__tests__/route-workout-scope.test.ts": 1, - "app/api/insights/chat/[id]/attachments/__tests__/route.test.ts": 1, - "app/api/insights/chat/[id]/attachments/[documentId]/route.ts": 1, - "app/api/insights/chat/[id]/attachments/route.ts": 1, - "app/api/insights/chat/[id]/messages/[messageId]/trail/__tests__/route.test.ts": 1, - "app/api/insights/chat/[id]/route.ts": 1, "app/api/insights/chat/fenced/__tests__/route.test.ts": 1, "app/api/insights/chat/fenced/route.ts": 1, "app/api/insights/chat/messages/[id]/feedback/route.ts": 3, "app/api/insights/chat/route.ts": 9, "app/api/insights/coach-read/route.ts": 2, + "app/api/insights/coach/facts/[id]/route.ts": 2, "app/api/insights/coach/facts/__tests__/route-memory.test.ts": 1, "app/api/insights/coach/facts/__tests__/route.test.ts": 1, - "app/api/insights/coach/facts/[id]/route.ts": 2, "app/api/insights/coach/facts/route.ts": 1, "app/api/insights/coach/nudge-status/route.ts": 1, "app/api/insights/coach/seen/route.ts": 1, @@ -326,9 +326,9 @@ "app/api/insights/derived/batch/__tests__/route.test.ts": 3, "app/api/insights/derived/batch/route.ts": 6, "app/api/insights/derived/route.ts": 5, - "app/api/insights/ecg/__tests__/route.test.ts": 1, "app/api/insights/ecg/[id]/__tests__/route.test.ts": 1, "app/api/insights/ecg/[id]/route.ts": 2, + "app/api/insights/ecg/__tests__/route.test.ts": 1, "app/api/insights/ecg/route.ts": 1, "app/api/insights/feedback/route.ts": 2, "app/api/insights/generate/__tests__/route.test.ts": 12, @@ -367,8 +367,8 @@ "app/api/internal/deploy-webhook/route.ts": 1, "app/api/internal/web-vitals/__tests__/route.test.ts": 1, "app/api/internal/web-vitals/route.ts": 1, - "app/api/labs/__tests__/error-contract.test.ts": 1, "app/api/labs/[id]/route.ts": 7, + "app/api/labs/__tests__/error-contract.test.ts": 1, "app/api/labs/ocr/capability/route.ts": 1, "app/api/labs/ocr/commit/__tests__/route.test.ts": 2, "app/api/labs/ocr/commit/route.ts": 4, @@ -388,13 +388,13 @@ "app/api/measurement-reminders/[id]/skip/route.ts": 1, "app/api/measurement-reminders/[id]/snooze/route.ts": 1, "app/api/measurement-reminders/route.ts": 3, + "app/api/measurements/[id]/__tests__/route.test.ts": 2, + "app/api/measurements/[id]/route.ts": 16, "app/api/measurements/__tests__/arrival-wiring.test.ts": 1, "app/api/measurements/__tests__/group-by-day.test.ts": 8, "app/api/measurements/__tests__/put-duplicate-timestamp.test.ts": 1, "app/api/measurements/__tests__/range-aggregation-route.test.ts": 3, "app/api/measurements/__tests__/zod-multi-issue.test.ts": 4, - "app/api/measurements/[id]/__tests__/route.test.ts": 2, - "app/api/measurements/[id]/route.ts": 16, "app/api/measurements/batch/__tests__/aggregated-hr-wire.test.ts": 1, "app/api/measurements/batch/__tests__/duplicate-prefilter.test.ts": 2, "app/api/measurements/batch/__tests__/hr-bucket-observability.test.ts": 1, @@ -411,7 +411,6 @@ "app/api/measurements/series-batch/route.ts": 1, "app/api/measurements/series/__tests__/route.test.ts": 5, "app/api/measurements/series/route.ts": 6, - "app/api/medications/__tests__/route.test.ts": 10, "app/api/medications/[id]/__tests__/route.test.ts": 8, "app/api/medications/[id]/api-endpoint/route.ts": 2, "app/api/medications/[id]/cadence/route.ts": 10, @@ -423,10 +422,10 @@ "app/api/medications/[id]/dose-history/route.ts": 14, "app/api/medications/[id]/glp1/__tests__/route.test.ts": 3, "app/api/medications/[id]/glp1/route.ts": 8, - "app/api/medications/[id]/intake/__tests__/route.test.ts": 11, - "app/api/medications/[id]/intake/__tests__/source-provenance.test.ts": 1, "app/api/medications/[id]/intake/[eventId]/__tests__/route.test.ts": 9, "app/api/medications/[id]/intake/[eventId]/route.ts": 4, + "app/api/medications/[id]/intake/__tests__/route.test.ts": 11, + "app/api/medications/[id]/intake/__tests__/source-provenance.test.ts": 1, "app/api/medications/[id]/intake/bulk-delete/route.ts": 7, "app/api/medications/[id]/intake/import/__tests__/route.test.ts": 2, "app/api/medications/[id]/intake/import/route.ts": 6, @@ -434,11 +433,12 @@ "app/api/medications/[id]/inventory/[itemId]/route.ts": 8, "app/api/medications/[id]/inventory/route.ts": 4, "app/api/medications/[id]/phase-config/route.ts": 4, - "app/api/medications/[id]/schedule-revisions/__tests__/route.test.ts": 1, "app/api/medications/[id]/schedule-revisions/[revisionId]/route.ts": 9, + "app/api/medications/[id]/schedule-revisions/__tests__/route.test.ts": 1, "app/api/medications/[id]/schedule-revisions/route.ts": 3, "app/api/medications/[id]/side-effects/[logId]/route.ts": 2, "app/api/medications/[id]/side-effects/route.ts": 5, + "app/api/medications/__tests__/route.test.ts": 10, "app/api/medications/categories/[key]/route.ts": 1, "app/api/medications/categories/route.ts": 1, "app/api/medications/compliance/__tests__/route.test.ts": 2, @@ -455,9 +455,9 @@ "app/api/medications/route.ts": 2, "app/api/mental-health/assessments/__tests__/route.test.ts": 2, "app/api/mental-health/assessments/route.ts": 6, - "app/api/mood-entries/__tests__/route.test.ts": 7, "app/api/mood-entries/[id]/__tests__/route.test.ts": 4, "app/api/mood-entries/[id]/route.ts": 22, + "app/api/mood-entries/__tests__/route.test.ts": 7, "app/api/mood-entries/bulk-delete/__tests__/route.test.ts": 1, "app/api/mood-entries/bulk-delete/route.ts": 4, "app/api/mood-entries/bulk/__tests__/route.test.ts": 5, @@ -547,12 +547,12 @@ "app/api/withings/connect/__tests__/route.test.ts": 1, "app/api/withings/connect/route.ts": 3, "app/api/withings/status/route.ts": 1, - "app/api/withings/webhook/__tests__/route.test.ts": 1, "app/api/withings/webhook/[token]/__tests__/route.test.ts": 1, + "app/api/withings/webhook/__tests__/route.test.ts": 1, "app/api/withings/webhook/route.ts": 2, + "app/api/workouts/[id]/route.ts": 2, "app/api/workouts/__tests__/batch-create.test.ts": 3, "app/api/workouts/__tests__/canonical-dedup.test.ts": 3, - "app/api/workouts/[id]/route.ts": 2, "app/api/workouts/batch/route.ts": 3, "app/api/workouts/route.ts": 2, "app/auth/claim/page.tsx": 1, @@ -654,20 +654,20 @@ "app/loading.tsx": 2, "app/mcp/__tests__/route.test.ts": 5, "app/measurements/page.tsx": 5, + "app/medications/[id]/history/page.tsx": 1, + "app/medications/[id]/page.tsx": 2, "app/medications/__tests__/category-filter.test.tsx": 2, "app/medications/__tests__/detail-page-history-surface.test.ts": 1, "app/medications/__tests__/list-view-toggle.test.tsx": 5, "app/medications/__tests__/page-prefetch.test.tsx": 1, "app/medications/__tests__/take-all-due-affordance.test.tsx": 1, - "app/medications/[id]/history/page.tsx": 1, - "app/medications/[id]/page.tsx": 2, "app/medications/new/page.tsx": 1, "app/medications/page-client.tsx": 50, "app/mental-wellbeing/page.tsx": 1, "app/mood/page-client.tsx": 2, "app/not-found.tsx": 2, - "app/onboarding/__tests__/page.test.tsx": 1, "app/onboarding/[step]/page.tsx": 1, + "app/onboarding/__tests__/page.test.tsx": 1, "app/onboarding/handover/page.tsx": 1, "app/onboarding/page.tsx": 2, "app/privacy/__tests__/page.test.tsx": 1, @@ -1748,10 +1748,10 @@ "lib/ai/insight-interpretation.ts": 1, "lib/ai/legacy-payload.ts": 2, "lib/ai/legacy-private-host-report.ts": 1, - "lib/ai/local-client.ts": 18, + "lib/ai/local-client.ts": 12, "lib/ai/medical-references.ts": 1, "lib/ai/mock-client.ts": 5, - "lib/ai/openai-client.ts": 16, + "lib/ai/openai-client.ts": 9, "lib/ai/prompts/__tests__/assessment-prompt-contract.test.ts": 5, "lib/ai/prompts/__tests__/briefing-personalisation.test.ts": 1, "lib/ai/prompts/__tests__/coach-prompt-locale.test.ts": 1, @@ -1783,7 +1783,7 @@ "lib/ai/prompts/weight.ts": 2, "lib/ai/provider-chain.ts": 1, "lib/ai/provider-health-ledger.ts": 3, - "lib/ai/provider-runner.ts": 15, + "lib/ai/provider-runner.ts": 11, "lib/ai/provider.ts": 10, "lib/ai/reasoning/__tests__/controls.test.ts": 1, "lib/ai/reasoning/controls.ts": 2, @@ -2158,7 +2158,7 @@ "lib/insights/chart-tokens.ts": 24, "lib/insights/coach-launch-context.tsx": 10, "lib/insights/comprehensive-aggregator.ts": 27, - "lib/insights/comprehensive-generate.ts": 48, + "lib/insights/comprehensive-generate.ts": 45, "lib/insights/correlation-channel-series.ts": 16, "lib/insights/correlation-discovery.ts": 30, "lib/insights/correlation-series-builders.ts": 1, @@ -2752,7 +2752,7 @@ "lib/rollups/mood-rollups.ts": 4, "lib/rollups/tiered-context.ts": 6, "lib/score-config/rows.ts": 1, - "lib/settings/section-slugs.ts": 20, + "lib/settings/section-slugs.ts": 19, "lib/sharing/__tests__/account-access-parse.test.ts": 3, "lib/sharing/__tests__/account-access.test.ts": 2, "lib/sharing/__tests__/account-label.test.ts": 1, diff --git a/src/__tests__/detached-work-guard.test.ts b/src/__tests__/detached-work-guard.test.ts index 8f3992048..c4903abf0 100644 --- a/src/__tests__/detached-work-guard.test.ts +++ b/src/__tests__/detached-work-guard.test.ts @@ -91,3 +91,56 @@ describe("detached work cannot rename the request's event", () => { expect(startsDetached(detached)).toBe(true); }); }); + +/** + * Read-path work a request starts and never awaits. Each of these runs real + * statements against Postgres after the response is already on its way, so a + * bare `void` hides it from `settleBackgroundTasks()`: the statement counter + * then closes its window while a stale account's rollup recompute is still + * writing, and the next test inherits the tail. The dashboard summary budget + * read 30 against 28 that way, depending on which side of the window a full + * mood recompute landed. + */ +const READ_PATH_BACKGROUND_STARTERS = [ + "ensureUserRollupsFresh", + "ensureUserMoodRollupsFresh", + "requestTodayBriefingWarm", + "touchGrantUsage", + "recordDelegatedAccess", +] as const; + +describe("read-path background work is registered", () => { + const files = readSourceFolder("src", { + floor: 3000, + include: (rel) => + !rel.startsWith("generated/") && + !rel.includes("__tests__/") && + !/\.test\.tsx?$/.test(rel), + }); + const bare = new RegExp( + String.raw`\bvoid\s+(${READ_PATH_BACKGROUND_STARTERS.join("|")})\s*\(`, + ); + + it("starts none of them with a bare void", () => { + const offenders = files.flatMap(([rel, text]) => + stripComments(text) + .split("\n") + .flatMap((line, i) => (bare.test(line) ? [`src/${rel}:${i + 1}`] : [])), + ); + expect( + offenders, + "wrap the call in trackBackgroundTask(runDetached(() => ...)) or " + + "fireAndForget(...) so the suite can await it", + ).toEqual([]); + }); + + it("still finds each starter at a detached call site", () => { + const joined = files.map(([, text]) => stripComments(text)).join("\n"); + for (const name of READ_PATH_BACKGROUND_STARTERS) { + expect( + new RegExp(String.raw`=>\s*${name}\s*\(`).test(joined), + `${name} is no longer started detached anywhere; drop it from the list`, + ).toBe(true); + } + }); +}); diff --git a/src/__tests__/sharing-surface-guard.test.ts b/src/__tests__/sharing-surface-guard.test.ts index 827b63bfd..458f6a282 100644 --- a/src/__tests__/sharing-surface-guard.test.ts +++ b/src/__tests__/sharing-surface-guard.test.ts @@ -2172,8 +2172,6 @@ const ACTOR_ROUTES: Record = { "The UI language belongs to the person reading the screen, never to the record on it. Under a record scope this would transplant a delegate's choice onto the owner's row and send the owner's cron mail in a language they may not read. It has to keep working rather than merely refuse safely: the switcher's mount-time backfill fires on every page load, including the shared dashboard.", "app/api/settings/module-list-layout/route.ts": "How the person's own labs, illness and checkups lists are laid out: card or list view, manual order, sort. It reads and writes the caller's own row only. Inside another record the ids of a saved order simply match nothing and the list keeps the server order, so serving the person's own choice there reveals nothing of the owner's and grants nothing; refusing it under a switch broke the list controls in exactly the context they are used in.", - "app/api/settings/preview-features/route.ts": - "The person's own opt-in to preview features. A preview entry is chrome of the person reading the screen, never a property of the record on it; it reads and writes the caller's own row and nothing of the owner's.", "app/api/feature-flags/route.ts": "Reads the `AppSettings` singleton and no user row at all, so there is no record for a switch to substitute. It answers about the deployment, which for this purpose is the caller's side of the request; the shell gates the Coach launcher and the assistant surfaces on it, so a 403 here is a piece of chrome deciding it does not exist.", }; @@ -2283,11 +2281,14 @@ const ACTOR_ROUTES: Record = { * owner-only, and a delegate sees the goals only as the calendar reports them. * 270 -> 273. * + * The preview-feature switch went with its column in the next release; the + * topic pages it was kept for were dropped. 273 -> 272. + * * Ending a running course (#1280) joins the record list and the * manage literal: it was refused under a switch while the medication's PUT, - * which used to carry the end, admitted a manager. 273 -> 275. + * which used to carry the end, admitted a manager. 272 -> 274. */ -const FROZEN_ENTRY_COUNT = 275; +const FROZEN_ENTRY_COUNT = 274; /** * The two surfaces that authenticate a Bearer token outside `requireAuth` — diff --git a/src/app/api/admin/ai-settings/__tests__/route.test.ts b/src/app/api/admin/ai-settings/__tests__/route.test.ts index f776098b5..2a64f5d60 100644 --- a/src/app/api/admin/ai-settings/__tests__/route.test.ts +++ b/src/app/api/admin/ai-settings/__tests__/route.test.ts @@ -16,6 +16,10 @@ vi.mock("@/lib/db", () => ({ upsert: vi.fn(), }, user: { findMany: vi.fn().mockResolvedValue([]) }, + providerHealth: { deleteMany: vi.fn().mockResolvedValue({ count: 0 }) }, + insightStatusCache: { + updateMany: vi.fn().mockResolvedValue({ count: 0 }), + }, }, })); @@ -217,6 +221,31 @@ describe("PUT /api/admin/ai-settings", () => { statusCode: 429, }); }); + + // #1279 — background work leaves a provider the health ledger benched out + // of every account's chain. A fixed operator provider must not sit out a + // pause the broken one earned, so a saved change forgets what the ledger + // recorded for it on every account, and lets their notes try again. + it("forgets the operator provider's recorded failures on every account", async () => { + upsertEcho(); + await PUT(jsonReq({ model: "gpt-4o-mini" })); + expect(prisma.providerHealth.deleteMany).toHaveBeenCalledWith({ + where: { providerType: "admin-openai" }, + }); + expect(prisma.insightStatusCache.updateMany).toHaveBeenCalledWith({ + where: { + OR: [{ retryAt: { not: null } }, { failureStreak: { gt: 0 } }], + }, + data: { retryAt: null, negativeReason: null, failureStreak: 0 }, + }); + }); + + it("forgets nothing when the change is refused", async () => { + await expect( + PUT(jsonReq({ baseUrl: "https://attacker.example/v1" })), + ).rejects.toMatchObject({ statusCode: 422 }); + expect(prisma.providerHealth.deleteMany).not.toHaveBeenCalled(); + }); }); /** diff --git a/src/app/api/admin/ai-settings/route.ts b/src/app/api/admin/ai-settings/route.ts index aafbd9b7a..a413d0bb9 100644 --- a/src/app/api/admin/ai-settings/route.ts +++ b/src/app/api/admin/ai-settings/route.ts @@ -8,6 +8,8 @@ import { encrypt, decrypt } from "@/lib/crypto"; import { checkRateLimit } from "@/lib/rate-limit"; import { annotate } from "@/lib/logging/context"; import { legacyPrivateHostReport } from "@/lib/ai/legacy-private-host-report"; +import { forgetProviderHealth } from "@/lib/ai/provider-health-ledger"; +import { clearStatusNegativeWindows } from "@/lib/insights/status-negative-window"; export const dynamic = "force-dynamic"; @@ -134,6 +136,16 @@ export const PUT = apiHandler(async (request: NextRequest) => { details: auditDetails, }); + // #1279 — background work leaves out a provider the health ledger benched, + // on every account whose chain falls back on this one, and a status note + // waits out its negative window. A fixed operator provider must not sit out + // a pause the broken one earned, so what was recorded against it is + // forgotten on every account, as a person's own save does for theirs. + await Promise.all([ + forgetProviderHealth({ providerType: "admin-openai" }), + clearStatusNegativeWindows(), + ]); + const hasKey = Boolean(settings.adminAiKeyEncrypted); return apiSuccess({ diff --git a/src/app/api/admin/central-codex/device-poll/route.ts b/src/app/api/admin/central-codex/device-poll/route.ts index 86aab4eed..62d9df175 100644 --- a/src/app/api/admin/central-codex/device-poll/route.ts +++ b/src/app/api/admin/central-codex/device-poll/route.ts @@ -9,6 +9,7 @@ import { auditLog } from "@/lib/auth/audit"; import { getClientIp, apiSuccess, apiError } from "@/lib/api-response"; import { annotate } from "@/lib/logging/context"; import { forgetProviderHealth } from "@/lib/ai/provider-health-ledger"; +import { clearStatusNegativeWindows } from "@/lib/insights/status-negative-window"; /** * Operator-shared central Codex — device-code poll (admin only). @@ -94,7 +95,12 @@ export const POST = apiHandler(async (request: NextRequest) => { cookieStore.delete("central_codex_device"); // A new central sign-in: every account's ledger rows describe the refused // one, and the central connection is one credential shared by all of them. - await forgetProviderHealth({ providerType: "admin-codex" }); + // #1279 — and every record's note windows close, as on the admin AI + // settings save, so no note sits out a window the broken one earned. + await Promise.all([ + forgetProviderHealth({ providerType: "admin-codex" }), + clearStatusNegativeWindows(), + ]); await auditLog("admin.central-codex.connected", { userId: user.id, diff --git a/src/app/api/auth/codex/device-poll/route.ts b/src/app/api/auth/codex/device-poll/route.ts index 47f9987b0..e4fb52758 100644 --- a/src/app/api/auth/codex/device-poll/route.ts +++ b/src/app/api/auth/codex/device-poll/route.ts @@ -10,6 +10,7 @@ import { getClientIp, apiSuccess, apiError } from "@/lib/api-response"; import { annotate } from "@/lib/logging/context"; import { INSIGHTS_CACHE_CLEARED } from "@/lib/insights/cached-text-cipher"; import { forgetProviderHealth } from "@/lib/ai/provider-health-ledger"; +import { clearStatusNegativeWindows } from "@/lib/insights/status-negative-window"; /** * v1.4.7.1: Device-code flow polling endpoint. @@ -90,8 +91,12 @@ export const POST = apiHandler(async (request: NextRequest) => { cookieStore.delete("codex_device"); // A new sign-in: whatever the ledger held against the old one no longer - // describes this credential. - await forgetProviderHealth({ userId: user.id, providerType: "codex" }); + // describes this credential, and (#1279) the person's notes stop waiting + // out windows it earned, as on a save of their provider settings. + await Promise.all([ + forgetProviderHealth({ userId: user.id, providerType: "codex" }), + clearStatusNegativeWindows(user.id), + ]); await auditLog("codex.oauth.connected", { userId: user.id, diff --git a/src/app/api/documents/inbound/[id]/chat/__tests__/route.test.ts b/src/app/api/documents/inbound/[id]/chat/__tests__/route.test.ts index 1cee38bac..823dd0fd6 100644 --- a/src/app/api/documents/inbound/[id]/chat/__tests__/route.test.ts +++ b/src/app/api/documents/inbound/[id]/chat/__tests__/route.test.ts @@ -210,6 +210,7 @@ beforeEach(() => { }, workingProvider: { providerType: "anthropic", instance: {} }, fallbackHops: [], + tokensSpent: 0, } as never); }); @@ -356,6 +357,7 @@ describe("POST — numeric grounding", () => { }, workingProvider: { providerType: "anthropic", instance: {} }, fallbackHops: [], + tokensSpent: 0, } as never); const res = await POST( req("doc-1", { message: "what is my ldl?" }) as never, @@ -393,6 +395,7 @@ describe("POST — numeric grounding", () => { }, workingProvider: { providerType: "anthropic", instance: {} }, fallbackHops: [], + tokensSpent: 0, } as never); const res = await POST( req("doc-1", { message: "how is my ldl?" }) as never, diff --git a/src/app/api/insights/chat/__tests__/dialog-harness.ts b/src/app/api/insights/chat/__tests__/dialog-harness.ts index dff956574..178ef3964 100644 --- a/src/app/api/insights/chat/__tests__/dialog-harness.ts +++ b/src/app/api/insights/chat/__tests__/dialog-harness.ts @@ -325,6 +325,7 @@ async function runRawCompletionWithFallback(args: { }, workingProvider: { providerType: "anthropic" }, fallbackHops: [], + tokensSpent: 0, }; } diff --git a/src/app/api/insights/chat/__tests__/golden-harness.ts b/src/app/api/insights/chat/__tests__/golden-harness.ts index 64324be38..e08e7c9aa 100644 --- a/src/app/api/insights/chat/__tests__/golden-harness.ts +++ b/src/app/api/insights/chat/__tests__/golden-harness.ts @@ -349,6 +349,7 @@ export function streamingResult(content: string) { model: "m-local", }, workingProvider: { providerType: "local" }, + tokensSpent: 0, }; }; } diff --git a/src/app/api/insights/chat/__tests__/route-guard.test.ts b/src/app/api/insights/chat/__tests__/route-guard.test.ts index 6499eb771..33603cc8e 100644 --- a/src/app/api/insights/chat/__tests__/route-guard.test.ts +++ b/src/app/api/insights/chat/__tests__/route-guard.test.ts @@ -264,6 +264,7 @@ async function noToolTurn(content: string): Promise { runStreamingRawCompletionWithFallback.mockResolvedValue({ result: { content, tokensUsed: 42, model: "m" }, workingProvider: { providerType: "anthropic" }, + tokensSpent: 0, }); const res = await post(chatReq({ message: "How is my BP?" })); await sse.done; diff --git a/src/app/api/insights/chat/__tests__/route-snapshot-once.test.ts b/src/app/api/insights/chat/__tests__/route-snapshot-once.test.ts index 401598337..613398663 100644 --- a/src/app/api/insights/chat/__tests__/route-snapshot-once.test.ts +++ b/src/app/api/insights/chat/__tests__/route-snapshot-once.test.ts @@ -220,6 +220,7 @@ describe("coach chat — stateless no-tools snapshot grounding", () => { runStreamingRawCompletionWithFallback.mockResolvedValue({ result: { content: "Your BP looks stable.", tokensUsed: 42, model: "m" }, workingProvider: { providerType: "admin-openai" }, + tokensSpent: 0, }); }); @@ -276,6 +277,7 @@ describe("coach chat — snapshot stays grounded across transcript elision", () runStreamingRawCompletionWithFallback.mockResolvedValue({ result: { content: "Your BP looks stable.", tokensUsed: 42, model: "m" }, workingProvider: { providerType: "admin-openai" }, + tokensSpent: 0, }); }); diff --git a/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts b/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts index 7f25ae425..947964daa 100644 --- a/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts +++ b/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts @@ -569,6 +569,7 @@ describe("coach chat — tool-mode routing (F1)", () => { runStreamingRawCompletionWithFallback.mockResolvedValue({ result: { content: "Your BP is steady.", tokensUsed: 42, model: "m" }, workingProvider: { providerType: "anthropic" }, + tokensSpent: 0, }); await postAndDrain({ message: "How is my BP?" }); diff --git a/src/app/api/insights/chat/__tests__/route-workout-scope.test.ts b/src/app/api/insights/chat/__tests__/route-workout-scope.test.ts index 64fad5815..11e0041ac 100644 --- a/src/app/api/insights/chat/__tests__/route-workout-scope.test.ts +++ b/src/app/api/insights/chat/__tests__/route-workout-scope.test.ts @@ -292,6 +292,7 @@ beforeEach(() => { model: "m", }, workingProvider: { providerType: "admin-openai" }, + tokensSpent: 0, }); isModuleEnabled.mockResolvedValue(true); }); diff --git a/src/app/api/insights/generate/__tests__/route.test.ts b/src/app/api/insights/generate/__tests__/route.test.ts index bab35438f..22eb6abba 100644 --- a/src/app/api/insights/generate/__tests__/route.test.ts +++ b/src/app/api/insights/generate/__tests__/route.test.ts @@ -71,6 +71,9 @@ vi.mock("@/lib/db", () => ({ consentReceipt: { findFirst: vi.fn(async () => ({ id: "receipt-1" })), }, + // The provider health ledger the chain walk reads. Nothing benched by + // default; the bench case below seeds a row. + providerHealth: { findMany: vi.fn(async () => []) }, }, })); @@ -160,6 +163,8 @@ vi.mock("@/lib/i18n/server-locale", () => ({ resolveServerLocale: vi.fn(async () => "en"), })); +import { NextRequest } from "next/server"; + import { GET, POST, resolveInsightsRateLimit } from "../route"; import { enqueueStatusRefillForUser } from "@/lib/insights/comprehensive-generate"; import { resolveProvider } from "@/lib/ai/provider"; @@ -284,6 +289,51 @@ describe("POST /api/insights/generate — daily token ceiling", () => { }); }); +describe("POST /api/insights/generate — a provider the health ledger benched", () => { + // #1279 — "Re-run analysis" is a person waiting on the answer. Background + // work leaves a benched provider out; this call walks it at the tail of + // the chain, so it reaches the provider and reports what the provider + // said, not a connection failure that never happened. + it("still asks the benched provider and reports its own refusal", async () => { + const now = Date.now(); + vi.mocked(prisma.providerHealth.findMany).mockResolvedValueOnce([ + { + id: "ph-1", + userId: "u-1", + providerType: "admin-openai", + lastResult: "auth_failed", + lastStatus: 401, + consecutiveFailures: 1, + lastOkAt: null, + lastFailureAt: new Date(now), + nextRetryAt: new Date(now + 60 * 60 * 1000), + updatedAt: new Date(now), + }, + ]); + const err = Object.assign(new Error("OpenAI request failed (401)"), { + httpStatus: 401, + }); + makeProviderThatThrows(err); + const provider = await vi.mocked(resolveProvider)("u-1"); + const body = JSON.stringify({ force: true }); + const request = new NextRequest("http://localhost/api/insights/generate", { + method: "POST", + headers: { + "content-type": "application/json", + "content-length": String(body.length), + }, + body, + }); + + const res = await POST(request); + + expect(provider.generateCompletion).toHaveBeenCalledTimes(1); + expect(res.status).toBe(422); + const envelope = (await res.json()) as ApiErrorEnvelope; + expect(envelope.error).toMatch(/AI provider rejected/i); + }); +}); + describe("POST /api/insights/generate — provider error mapping", () => { it("rejects a body over the 16 KB cap with 413 before parsing", async () => { const res = await POST( diff --git a/src/app/api/medications/extract/__tests__/route.test.ts b/src/app/api/medications/extract/__tests__/route.test.ts index f13877232..81d06c85b 100644 --- a/src/app/api/medications/extract/__tests__/route.test.ts +++ b/src/app/api/medications/extract/__tests__/route.test.ts @@ -192,6 +192,7 @@ describe("POST /api/medications/extract — happy path", () => { }, workingProvider: { providerType: "openai", instance: {} as never }, fallbackHops: [], + tokensSpent: 0, }); const res = await POST( @@ -237,6 +238,7 @@ describe("POST /api/medications/extract — happy path", () => { }, workingProvider: { providerType: "openai", instance: {} as never }, fallbackHops: [], + tokensSpent: 0, }); const res = await POST( @@ -261,6 +263,7 @@ describe("POST /api/medications/extract — daily budget", () => { }, workingProvider: { providerType: "openai", instance: {} as never }, fallbackHops: [], + tokensSpent: 0, }; } @@ -409,6 +412,7 @@ describe("POST /api/medications/extract — validation + provider errors", () => }, workingProvider: { providerType: "openai", instance: {} as never }, fallbackHops: [], + tokensSpent: 0, }); const res = await POST(postReq({ text: "Mounjaro 5mg weekly" }) as never); diff --git a/src/app/api/medications/extract/route.ts b/src/app/api/medications/extract/route.ts index 4764d4d08..b0086519f 100644 --- a/src/app/api/medications/extract/route.ts +++ b/src/app/api/medications/extract/route.ts @@ -71,6 +71,7 @@ import { AllProvidersFailedError, runRawCompletionWithFallback, } from "@/lib/ai/provider-runner"; +import { spentOnFailure } from "@/lib/ai/reasoning/answerless"; import { resolveProvider, resolveProviderChain } from "@/lib/ai/provider"; import { singleUserTurn } from "@/lib/ai/types"; @@ -230,13 +231,9 @@ async function handleExtract(request: NextRequest): Promise { locale, }); - // Atomically reserve the day's budget BEFORE the provider call. The old - // `enforceBudget` read-then-write left a TOCTOU window (concurrent calls - // could each observe a sub-cap total and all proceed) AND passed no cap, so - // it always measured against `OPERATOR_COST_CAP` — rationing a self-hoster - // paying for their own key by the operator's ceiling. `resolveDailyCap` - // charges the operator ceiling only when the chain's primary is the - // operator's own credential; every other chain gets the user-plan ceiling. + // Atomically reserve the day's budget BEFORE the provider call, against the + // cost owner's ceiling: the operator's only when the chain's primary is the + // operator's own credential, the user-plan ceiling for every other chain. const dateKey = accountToday; const estimatedTokens = MAX_OUTPUT_TOKENS + @@ -268,11 +265,9 @@ async function handleExtract(request: NextRequest): Promise { params: singleUserTurn({ system: systemPrompt, user: userPrompt, - // Low temperature because we want a deterministic field - // extractor, not a creative rephrase. + // A deterministic field extractor, not a creative rephrase. temperature: 0.1, - // The reply body is a single small JSON object; 600 is well - // over the empirical max we have seen on a 2k-char input. + // One small JSON object; well over the largest seen on 2k chars. maxTokens: MAX_OUTPUT_TOKENS, // Parsed with `parseModelJson` (JSON.parse) below — opt the OpenAI / // Codex chains into their strict JSON-object mode (gated on this flag). @@ -280,12 +275,17 @@ async function handleExtract(request: NextRequest): Promise { }), }); } catch (err) { - // Nothing was generated — refund the whole reservation rather than bill an - // invented figure. Best-effort: a failed refund must not mask the 503. - await reconcileSpend(userId, reservation.reserved, 0, dateKey, 0, { - servedBy: null, - reservedOwner: reservation.owner, - }).catch(() => {}); + // Nothing was generated — refund the reservation rather than bill an + // invented figure, keeping only what the failed hops reported spending + // (#1279). Best-effort: a failed refund must not mask the 503. + await reconcileSpend( + userId, + reservation.reserved, + spentOnFailure(err), + dateKey, + 0, + { servedBy: null, reservedOwner: reservation.owner }, + ).catch(() => {}); if (err instanceof AllProvidersFailedError) { annotate({ action: { name: "medications.extract.provider-failed" }, @@ -296,12 +296,12 @@ async function handleExtract(request: NextRequest): Promise { throw err; } - // Reconcile against what the provider actually reported. This runs for the - // unusable-reply branches too: those tokens were burned upstream even though - // the body could not be used, so they stay on the ledger rather than funding - // a free retry loop. Falls back to the reservation when the provider reports - // no count, so an unreported generation is never billed as zero. - const actualTokens = completion.result.tokensUsed ?? reservation.reserved; + // What the provider reported, plus (#1279) what the hops that failed before + // it spent; for the unusable-reply branches too, since those tokens were + // burned upstream. No count falls back to the reservation, never to zero. + const actualTokens = + (completion.result.tokensUsed ?? reservation.reserved) + + completion.tokensSpent; await reconcileSpend(userId, reservation.reserved, actualTokens, dateKey, 0, { servedBy: completion.workingProvider.providerType, reservedOwner: reservation.owner, diff --git a/src/app/api/settings/preview-features/route.ts b/src/app/api/settings/preview-features/route.ts deleted file mode 100644 index 8a6859f9a..000000000 --- a/src/app/api/settings/preview-features/route.ts +++ /dev/null @@ -1,80 +0,0 @@ -/** - * GET / PATCH the caller's preview-feature opt-ins. - * - * GET answers every flag, off unless the caller switched it on. PATCH sets - * the flags the body names and keeps the rest; unknown flags are refused - * with the multi-issue 422 so a typo cannot silently do nothing. The blob - * lives on its own `User` column (`preview_features_json`). - * - * The opt-ins are the caller's own, so this is an actor surface - * (`requireActorAuth`): a browser acting for another record keeps reaching - * the person's own switch. The PATCH reads and writes under a row lock, so two - * flags switched at once both stay switched. - */ -import { apiHandler, requireActorAuth } from "@/lib/api-handler"; -import { apiSuccess } from "@/lib/api-response"; -import { parseJsonBody } from "@/lib/api/body"; -import { prisma, toJson } from "@/lib/db"; -import { annotate } from "@/lib/logging/context"; -import { - PREVIEW_FEATURE_KEYS, - resolvePreviewFeatures, - type PreviewFeatures, -} from "@/lib/settings/preview-features"; -import { previewFeaturesPatchSchema } from "@/lib/validations/settings-prefs"; - -export const dynamic = "force-dynamic"; - -async function readStored( - userId: string, - db: Pick = prisma, -): Promise { - const row = await db.user.findUnique({ - where: { id: userId }, - select: { previewFeaturesJson: true }, - }); - return row?.previewFeaturesJson; -} - -function enabledFlags(features: PreviewFeatures): string[] { - return PREVIEW_FEATURE_KEYS.filter((key) => features[key]); -} - -export const GET = apiHandler(async () => { - const { user } = await requireActorAuth(); - const features = resolvePreviewFeatures(await readStored(user.id)); - annotate({ - action: { name: "settings.preview-features.read" }, - meta: { enabled: enabledFlags(features) }, - }); - return apiSuccess(features); -}); - -export const PATCH = apiHandler(async (request: Request) => { - const { user } = await requireActorAuth(); - - const parsed = await parseJsonBody(request, previewFeaturesPatchSchema, { - maxBytes: 1024, - action: "settings.preview-features.validation-failed", - }); - if (!parsed.ok) return parsed.error; - - const next = await prisma.$transaction(async (tx) => { - await tx.$queryRaw`SELECT 1 FROM "users" WHERE "id" = ${user.id} FOR UPDATE`; - const current = resolvePreviewFeatures(await readStored(user.id, tx)); - const merged: PreviewFeatures = { - topics: parsed.data.topics ?? current.topics, - }; - await tx.user.update({ - where: { id: user.id }, - data: { previewFeaturesJson: toJson(merged) }, - }); - return merged; - }); - - annotate({ - action: { name: "settings.preview-features.update" }, - meta: { enabled: enabledFlags(next) }, - }); - return apiSuccess(next); -}); diff --git a/src/app/api/sync/changes/__tests__/module-gate.test.ts b/src/app/api/sync/changes/__tests__/module-gate.test.ts index e75cd7145..9640f30c1 100644 --- a/src/app/api/sync/changes/__tests__/module-gate.test.ts +++ b/src/app/api/sync/changes/__tests__/module-gate.test.ts @@ -22,6 +22,8 @@ vi.mock("@/lib/modules/gate", () => ({ isModuleEnabled: vi.fn() })); vi.mock("@/lib/db", () => ({ prisma: { measurement: { findMany: vi.fn() }, + // The mirror walk's token lookup: no token switched, so no walk. + apiToken: { findMany: vi.fn(async () => []) }, moodEntry: { findMany: vi.fn() }, medicationIntakeEvent: { findMany: vi.fn() }, cycleDayLog: { findMany: vi.fn() }, diff --git a/src/lib/ai/__tests__/entry-effort-refused.test.ts b/src/lib/ai/__tests__/entry-effort-refused.test.ts new file mode 100644 index 000000000..caa14e151 --- /dev/null +++ b/src/lib/ai/__tests__/entry-effort-refused.test.ts @@ -0,0 +1,157 @@ +/** + * #1126 / #1279 — a gateway or local entry whose own reasoning setting the + * endpoint refuses. + * + * The entry's setting (`reasoningEffort`, stamped by the resolver) rides a + * call that carries no level of its own. An endpoint that refuses the field + * (LiteLLM in front of Ollama) answered every such call with a 400 that was + * never learned, because only the call's own level counted as "sent": every + * call failed. It is learned now like any refused field: the call is asked + * once more without it, inside its budget of two requests, later calls send + * no field, and the lesson is reported once. + * + * Driven through the real clients against a loopback server. + */ +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +const emitSignal = vi.hoisted(() => vi.fn()); +vi.mock("@/lib/logging/signal", () => ({ + emitSignal: (...a: unknown[]) => emitSignal(...a), +})); + +import { LocalOpenAICompatibleClient } from "../local-client"; +import { OpenAIClient } from "../openai-client"; +import { resetJsonModeDialectCache } from "../json-dialect"; +import { resetReasoningDialectCache } from "../reasoning/dialect-cache"; +import { singleUserTurn } from "../types"; +import { _resetPinnedDispatcherForTests } from "@/lib/safe-fetch-dispatcher"; + +const ANSWER = '{"summary":"Steady."}'; + +let server: http.Server; +let origin = ""; +let seen: unknown[] = []; + +beforeAll(async () => { + server = http.createServer((req, res) => { + let raw = ""; + req.on("data", (c) => (raw += c)); + req.on("end", () => { + const body = JSON.parse(raw) as Record; + seen.push(body.reasoning_effort); + if (body.reasoning_effort !== undefined) { + res.writeHead(400, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + error: { + message: + "litellm.UnsupportedParamsError: ollama does not support parameters: ['reasoning_effort']", + }, + }), + ); + return; + } + res.writeHead(200, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + choices: [ + { + index: 0, + message: { role: "assistant", content: ANSWER }, + finish_reason: "stop", + }, + ], + usage: { total_tokens: 400 }, + }), + ); + }); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +}); + +afterAll(async () => { + _resetPinnedDispatcherForTests(); + await new Promise((r) => server.close(() => r())); +}); + +const savedOrigins = process.env.AI_PRIVATE_ORIGINS; + +beforeEach(() => { + process.env.AI_PRIVATE_ORIGINS = origin; + seen = []; + emitSignal.mockClear(); + resetReasoningDialectCache(); + resetJsonModeDialectCache(); +}); + +afterEach(() => { + if (savedOrigins === undefined) delete process.env.AI_PRIVATE_ORIGINS; + else process.env.AI_PRIVATE_ORIGINS = savedOrigins; +}); + +function gateway() { + const client = new OpenAIClient({ + apiKey: "", + model: "qwen3", + baseUrl: `${origin}/v1`, + providerType: "openai-compatible", + }); + client.reasoningEffort = "high"; + return client; +} + +function local() { + const client = new LocalOpenAICompatibleClient({ + baseUrl: `${origin}/v1`, + model: "qwen3", + }); + client.reasoningEffort = "high"; + return client; +} + +/** A status card call: no level of its own. */ +const params = () => + singleUserTurn({ + system: "s", + user: "u", + maxTokens: 300, + timeoutMs: 30_000, + responseFormat: "json", + }); + +describe.each([ + ["the gateway", gateway], + ["the Local client", local], +] as const)("%s, an entry setting the endpoint refuses", (_, make) => { + it("answers on the second request, and later calls send no field", async () => { + const first = await make().generateCompletion(params()); + expect(first.content).toBe(ANSWER); + expect(seen).toEqual(["high", undefined]); + + seen = []; + for (let i = 0; i < 3; i += 1) { + expect((await make().generateCompletion(params())).content).toBe(ANSWER); + } + expect(seen).toEqual([undefined, undefined, undefined]); + }); + + it("says so once, not once per call", async () => { + for (let i = 0; i < 3; i += 1) await make().generateCompletion(params()); + const lines = emitSignal.mock.calls + .map((c) => c[0] as { action: string }) + .filter((s) => s.action === "ai.provider.reasoning_field_refused"); + expect(lines).toHaveLength(1); + }); +}); diff --git a/src/lib/ai/__tests__/local-reasoning.test.ts b/src/lib/ai/__tests__/local-reasoning.test.ts index 50565fe3c..30e5f091f 100644 --- a/src/lib/ai/__tests__/local-reasoning.test.ts +++ b/src/lib/ai/__tests__/local-reasoning.test.ts @@ -232,7 +232,7 @@ describe("Local reasoning replies", () => { expect(result.reasoning?.summary).toEqual(["The user asks about sleep."]); }); - it("never answers with unlabelled text that filled the reasoning ceiling", async () => { + it("never answers with unlabelled text that filled the reasoning ceiling, once the model reasons", async () => { const cut = async () => new Response( JSON.stringify({ @@ -245,15 +245,47 @@ describe("Local reasoning replies", () => { }), { status: 200 }, ); - const mock = vi.fn().mockImplementation(cut); + // The model has answered with a reasoning field before. + const mock = vi + .fn() + .mockImplementationOnce( + buffered({ content: "ok", reasoning: "Thinking first" }), + ) + .mockImplementation(cut); vi.stubGlobal("fetch", mock); - await expect( - client().generateCompletion( - params({ reasoning: { effort: "medium", summaries: true } }), - ), - ).rejects.toMatchObject({ httpStatus: 0, kind: "reasoning_only" }); + const c = client(); + const wanted = params({ reasoning: { effort: "medium", summaries: true } }); + expect((await c.generateCompletion(wanted)).content).toBe("ok"); + await expect(c.generateCompletion(wanted)).rejects.toMatchObject({ + httpStatus: 0, + kind: "reasoning_only", + }); // Asked once more with `none` (#1267), which this model ignores. - expect(mock).toHaveBeenCalledTimes(2); + expect(mock).toHaveBeenCalledTimes(3); + }); + + it("keeps a cut answer from a model never seen reasoning, whatever the level", async () => { + const mock = vi.fn().mockImplementation( + async () => + new Response( + JSON.stringify({ + choices: [ + { + message: { content: '{"summary": "Steady and' }, + finish_reason: "length", + }, + ], + }), + { status: 200 }, + ), + ); + vi.stubGlobal("fetch", mock); + const result = await client().generateCompletion( + params({ reasoning: { effort: "medium", summaries: true } }), + ); + expect(result.content).toBe('{"summary": "Steady and'); + expect(result.finishReason).toBe("length"); + expect(mock).toHaveBeenCalledTimes(1); }); it("keeps a cut answer when the ceiling held no room for reasoning", async () => { diff --git a/src/lib/ai/__tests__/provider-runner-background-bench.test.ts b/src/lib/ai/__tests__/provider-runner-background-bench.test.ts index 18b29e814..e17e9c558 100644 --- a/src/lib/ai/__tests__/provider-runner-background-bench.test.ts +++ b/src/lib/ai/__tests__/provider-runner-background-bench.test.ts @@ -111,6 +111,19 @@ describe("#1279: background work and a benched provider", () => { ]); } expect(gateway.calls).toBe(HARD_FAILURE_SKIP_THRESHOLD); + // The bench sent nothing; a hop that reached the provider did. + const benched = await walk( + "job", + "u-bench", + "openai-compatible", + gateway, + ledger, + ).then( + () => null, + (e: unknown) => e, + ); + expect(benched).toBeInstanceOf(AllProvidersFailedError); + expect((benched as AllProvidersFailedError).sentNothing).toBe(true); // One line for the window, not one per job. expect(signals("ai.chain.background_paused")).toHaveLength(1); expect(signals("ai.chain.background_paused")[0].meta).toMatchObject({ @@ -139,6 +152,30 @@ describe("#1279: background work and a benched provider", () => { expect((err as AllProvidersFailedError).attempts[0].failureKind).toBe( "reasoning_only", ); + expect((err as AllProvidersFailedError).sentNothing).toBe(false); + }); + + it("walks the benched provider for a job-rationed call a person waits on", async () => { + const ledger = createInMemoryProviderHealthLedger(); + const gateway = new AnswerlessProvider(); + for (let i = 0; i < PROVIDER_PAUSE_THRESHOLD; i += 1) { + await walk("job", "u-rerun", "openai-compatible", gateway, ledger).catch( + () => undefined, + ); + } + const before = gateway.calls; + const err = await runRawCompletionWithFallback({ + userId: "u-rerun", + surface: "job", + interactive: true, + providers: [{ providerType: "openai-compatible", instance: gateway }], + params: singleUserTurn({ system: "s", user: "u" }), + ledger, + }).catch((e) => e); + expect(gateway.calls).toBe(before + 1); + expect((err as AllProvidersFailedError).attempts[0].failureKind).toBe( + "reasoning_only", + ); }); it("logs once more when a new window opens", async () => { @@ -187,6 +224,36 @@ describe("#1279: background work and a benched provider", () => { expect(local.calls).toBe(PROVIDER_PAUSE_THRESHOLD); }); + it("reports what the hops before the answering one spent", async () => { + const ledger = createInMemoryProviderHealthLedger(); + const out = await runRawCompletionWithFallback({ + userId: "u-spent-then-answered", + surface: "job", + providers: [ + { + providerType: "openai-compatible", + instance: new AnswerlessProvider(), + }, + { + providerType: "anthropic", + instance: { + type: "anthropic", + generateCompletion: async () => ({ + content: "ok", + tokensUsed: 100, + model: "m", + providerType: "anthropic", + }), + } as AIProvider, + }, + ], + params: singleUserTurn({ system: "s", user: "u" }), + ledger, + }); + expect(out.result.tokensUsed).toBe(100); + expect(out.tokensSpent).toBe(2700); + }); + it("sums what the failed hops reported spending", async () => { const ledger = createInMemoryProviderHealthLedger(); const err = await runRawCompletionWithFallback({ diff --git a/src/lib/ai/__tests__/request-hops.test.ts b/src/lib/ai/__tests__/request-hops.test.ts new file mode 100644 index 000000000..673d09bf3 --- /dev/null +++ b/src/lib/ai/__tests__/request-hops.test.ts @@ -0,0 +1,281 @@ +/** + * #1279 — one completion call sends at most two requests, whatever the + * endpoint answers. + * + * A gateway in front of Ollama (LiteLLM: "ollama does not support + * parameters: ['reasoning_effort']") refuses the reasoning field outright, + * and the model behind it reasons unasked and answers nothing. The answerless + * rule asked again with `none`, the refusal rule wrote "send no field" over + * that lesson, the next request was answerless again, and so on: one call + * sent requests until something outside stopped it. + * + * Driven through the real clients, the real `safeFetch` and the pinned + * dispatcher against a loopback server that answers each request by a + * script, so every combination of refusal and answerless reply is walked. + */ +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +vi.mock("@/lib/logging/signal", () => ({ emitSignal: () => {} })); + +import { LocalOpenAICompatibleClient } from "../local-client"; +import { OpenAIClient } from "../openai-client"; +import { resetJsonModeDialectCache } from "../json-dialect"; +import { resetReasoningDialectCache } from "../reasoning/dialect-cache"; +import { MAX_REQUESTS_PER_CALL } from "../request-hops"; +import { singleUserTurn, type CompletionParams } from "../types"; +import { _resetPinnedDispatcherForTests } from "@/lib/safe-fetch-dispatcher"; + +interface Seen { + effort: unknown; + json: boolean; + stream: boolean; +} + +/** Decides one reply from what the request carried. */ +type Script = (req: Seen) => { status: number; body: unknown }; + +const REFUSE_REASONING = { + status: 400, + body: { + error: { + message: + "litellm.UnsupportedParamsError: ollama does not support parameters: ['reasoning_effort']", + }, + }, +}; + +const REFUSE_JSON = { + status: 400, + body: { error: { message: "Unsupported parameter: response_format" } }, +}; + +function answerless() { + return { + status: 200, + body: { + choices: [ + { + index: 0, + message: { + role: "assistant", + content: null, + reasoning_content: "thinking ".repeat(50), + }, + finish_reason: "length", + }, + ], + usage: { total_tokens: 1000 }, + }, + }; +} + +/** Refuses the reasoning field, reasons without answering otherwise. */ +const gatewayRefusingTheField: Script = (req) => + req.effort !== undefined ? REFUSE_REASONING : answerless(); + +/** Refuses the JSON flag and the reasoning field, answerless otherwise. */ +const refusesEverything: Script = (req) => + req.json + ? REFUSE_JSON + : req.effort !== undefined + ? REFUSE_REASONING + : answerless(); + +/** Refuses every streamed request and the reasoning field. */ +const refusesStreams: Script = (req) => + req.stream + ? { status: 400, body: { error: { message: "no stream" } } } + : gatewayRefusingTheField(req); + +let server: http.Server; +let origin = ""; +let script: Script = gatewayRefusingTheField; +let seen: Seen[] = []; + +beforeAll(async () => { + server = http.createServer((req, res) => { + let raw = ""; + req.on("data", (c) => (raw += c)); + req.on("end", () => { + if (!raw) { + res.writeHead(404); + res.end(); + return; + } + const body = JSON.parse(raw) as Record; + const request: Seen = { + effort: body.reasoning_effort, + json: body.response_format !== undefined, + stream: body.stream === true, + }; + seen.push(request); + // A backstop for the unbounded case, so a failing run ends. + const reply = + seen.length > 40 + ? { status: 500, body: { error: { message: "stop" } } } + : script(request); + res.writeHead(reply.status, { "Content-Type": "application/json" }); + res.end(JSON.stringify(reply.body)); + }); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +}); + +afterAll(async () => { + _resetPinnedDispatcherForTests(); + await new Promise((r) => server.close(() => r())); +}); + +const savedOrigins = process.env.AI_PRIVATE_ORIGINS; + +beforeEach(() => { + process.env.AI_PRIVATE_ORIGINS = origin; + seen = []; + resetReasoningDialectCache(); + resetJsonModeDialectCache(); +}); + +afterEach(() => { + if (savedOrigins === undefined) delete process.env.AI_PRIVATE_ORIGINS; + else process.env.AI_PRIVATE_ORIGINS = savedOrigins; +}); + +function gateway() { + return new OpenAIClient({ + apiKey: "", + model: "qwen3", + baseUrl: `${origin}/v1`, + providerType: "openai-compatible", + }); +} + +function local() { + return new LocalOpenAICompatibleClient({ + baseUrl: `${origin}/v1`, + model: "qwen3", + }); +} + +function params(extra: Partial = {}): CompletionParams { + return { + ...singleUserTurn({ + system: "s", + user: "u", + maxTokens: 300, + timeoutMs: 30_000, + responseFormat: "json", + }), + ...extra, + }; +} + +type Call = (p: CompletionParams) => Promise; + +const CALLERS: ReadonlyArray<[string, () => Call]> = [ + ["the gateway", () => (p) => gateway().generateCompletion(p)], + ["the Local client", () => (p) => local().generateCompletion(p)], + [ + "the Local client's stream", + () => (p) => local().generateCompletionStream(p, () => {}), + ], +]; + +/** Run `calls` calls in a row; the requests each one sent. */ +async function requestsPerCall( + call: Call, + calls: number, + extra: Partial = {}, +) { + const out: Array<{ + requests: Seen[]; + error: Record | null; + }> = []; + for (let i = 0; i < calls; i += 1) { + const before = seen.length; + const error = await call(params(extra)).then( + () => null, + (e: unknown) => e as Record, + ); + out.push({ requests: seen.slice(before), error }); + } + return out; +} + +describe("one call, at most two requests", () => { + it("the budget is two", () => { + expect(MAX_REQUESTS_PER_CALL).toBe(2); + }); + + describe.each(CALLERS)("%s", (_, make) => { + it.each([ + ["a gateway that refuses the reasoning field", gatewayRefusingTheField], + ["an endpoint that refuses the JSON flag too", refusesEverything], + ["a server that refuses streams", refusesStreams], + ] as const)("%s", async (_name, s) => { + script = s; + for (const extra of [ + {}, + { reasoning: { effort: "low" as const, summaries: false } }, + { reasoning: { effort: "off" as const, summaries: false } }, + ]) { + resetReasoningDialectCache(); + resetJsonModeDialectCache(); + const calls = await requestsPerCall(make(), 6, extra); + for (const { requests } of calls) { + expect(requests.length).toBeLessThanOrEqual(MAX_REQUESTS_PER_CALL); + expect(requests.length).toBeGreaterThan(0); + } + } + }); + + it("stops asking with `none` once the endpoint refused it, and fails as the reasoning it is", async () => { + script = gatewayRefusingTheField; + const calls = await requestsPerCall(make(), 4); + // The first call: no field, answerless; asked again with `none`, + // refused. Every later call sends no field and one request. + expect(calls[0].requests.map((r) => r.effort)).toEqual([ + undefined, + "none", + ]); + for (const later of calls.slice(1)) { + expect(later.requests.map((r) => r.effort)).toEqual([undefined]); + } + for (const { error } of calls) { + expect(error?.kind).toBe("reasoning_only"); + expect(error?.httpStatus).toBe(0); + } + // The first call's answerless request is what it cost. + expect(calls[0].error?.tokensUsed).toBe(1000); + }); + + it("never asks a model with `none` behind an endpoint that refused the field", async () => { + script = gatewayRefusingTheField; + // A call with a level learns that the field is refused first. + const first = await requestsPerCall(make(), 1, { + reasoning: { effort: "low", summaries: false }, + }); + expect(first[0].requests.map((r) => r.effort)).toEqual([ + "low", + undefined, + ]); + // Its answerless repeat taught that the model reasons unasked; no call + // after it sends a reasoning field again. + const later = await requestsPerCall(make(), 3); + for (const { requests } of later) { + expect(requests.map((r) => r.effort)).toEqual([undefined]); + } + }); + }); +}); diff --git a/src/lib/ai/__tests__/truncated-plain-answer.test.ts b/src/lib/ai/__tests__/truncated-plain-answer.test.ts new file mode 100644 index 000000000..4ab9ea976 --- /dev/null +++ b/src/lib/ai/__tests__/truncated-plain-answer.test.ts @@ -0,0 +1,178 @@ +/** + * #1276 — a reply that ran into the token ceiling with unlabelled text is the + * model's answer, cut off, unless this endpoint and model have shown that + * they reason. + * + * The rule read every such reply as reasoning the server did not separate, + * whenever the ceiling held room for reasoning: a call carrying any level + * (Off included) to a model that does not reason at all (llama3.1) lost its + * own truncated answer. The briefing never reached its JSON-shape retry, the + * Coach named the wrong error, the failure counted toward the bench, and the + * client learned to ask with `none` for an hour. Now the text is reasoning + * only once the model has shown it reasons (a reasoning field, a think tag, + * or a lesson that says so); otherwise it is the answer with + * `finishReason: "length"`. + * + * Driven through the real clients against a loopback server. + */ +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +vi.mock("@/lib/logging/signal", () => ({ emitSignal: () => {} })); + +import { LocalOpenAICompatibleClient } from "../local-client"; +import { OpenAIClient } from "../openai-client"; +import { resetJsonModeDialectCache } from "../json-dialect"; +import { + learnedReasoningDialect, + resetReasoningDialectCache, +} from "../reasoning/dialect-cache"; +import { singleUserTurn, type CompletionParams } from "../types"; +import { _resetPinnedDispatcherForTests } from "@/lib/safe-fetch-dispatcher"; + +const CUT = '{"summary": "Your weight has been stable over the last weeks and'; + +/** What the next replies carry, one per request; the last one repeats. */ +let replies: Array> = []; +let seen: unknown[] = []; +let server: http.Server; +let origin = ""; + +function cut() { + return { message: { role: "assistant", content: CUT }, finish: "length" }; +} + +function reasonedAnswer() { + return { + message: { + role: "assistant", + content: '{"summary":"Steady."}', + reasoning_content: "The readings are flat.", + }, + finish: "stop", + }; +} + +beforeAll(async () => { + server = http.createServer((req, res) => { + let raw = ""; + req.on("data", (c) => (raw += c)); + req.on("end", () => { + const body = JSON.parse(raw) as Record; + seen.push(body.reasoning_effort); + const next = (replies.length > 1 ? replies.shift() : replies[0]) as { + message: unknown; + finish: string; + }; + res.writeHead(200, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + choices: [ + { index: 0, message: next.message, finish_reason: next.finish }, + ], + usage: { total_tokens: 3000 }, + }), + ); + }); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +}); + +afterAll(async () => { + _resetPinnedDispatcherForTests(); + await new Promise((r) => server.close(() => r())); +}); + +const savedOrigins = process.env.AI_PRIVATE_ORIGINS; + +beforeEach(() => { + process.env.AI_PRIVATE_ORIGINS = origin; + seen = []; + replies = []; + resetReasoningDialectCache(); + resetJsonModeDialectCache(); +}); + +afterEach(() => { + if (savedOrigins === undefined) delete process.env.AI_PRIVATE_ORIGINS; + else process.env.AI_PRIVATE_ORIGINS = savedOrigins; +}); + +function params(effort: "off" | "low"): CompletionParams { + return { + ...singleUserTurn({ + system: "s", + user: "u", + maxTokens: 2500, + timeoutMs: 30_000, + responseFormat: "json", + }), + reasoning: { effort, summaries: false }, + }; +} + +describe.each([ + [ + "the gateway", + () => + new OpenAIClient({ + apiKey: "", + model: "llama3.1", + baseUrl: `${origin}/v1`, + providerType: "openai-compatible", + }), + "openai", + ], + [ + "the Local client", + () => + new LocalOpenAICompatibleClient({ + baseUrl: `${origin}/v1`, + model: "llama3.1", + }), + "local", + ], +] as const)("%s", (_, make, provider) => { + const key = () => ({ + provider, + endpoint: `${origin}/v1`, + model: "llama3.1", + }); + + it.each(["off", "low"] as const)( + "returns a model's cut answer as the answer (level %s), and learns nothing", + async (effort) => { + replies = [cut()]; + for (let i = 0; i < 2; i += 1) { + const result = await make().generateCompletion(params(effort)); + expect(result.content).toBe(CUT); + expect(result.finishReason).toBe("length"); + } + // One request per call, each sent as the call asked. + const asked = effort === "off" ? "none" : "low"; + expect(seen).toEqual([asked, asked]); + expect(learnedReasoningDialect(key())).toBeUndefined(); + }, + ); + + it("reads the same cut text as reasoning once the model has shown it reasons", async () => { + replies = [reasonedAnswer(), cut()]; + const first = await make().generateCompletion(params("low")); + expect(first.content).toBe('{"summary":"Steady."}'); + const err = await make() + .generateCompletion(params("low")) + .catch((e: unknown) => e as Record); + expect(err).toMatchObject({ httpStatus: 0, kind: "reasoning_only" }); + }); +}); diff --git a/src/lib/ai/background-bench.ts b/src/lib/ai/background-bench.ts index 0a5fb5614..a2b8d594f 100644 --- a/src/lib/ai/background-bench.ts +++ b/src/lib/ai/background-bench.ts @@ -7,7 +7,9 @@ * times in a row: a chain whose only entry answered every call with reasoning * and no answer was called by every background job, all day, because the tail * is still walked. So a job leaves every benched provider out until its - * window lifts, and the window grows while the failures continue + * window lifts (a job-rationed call a person waits on, the on-demand + * briefing, passes `interactive` and walks it at the tail like any + * interactive call), and the window grows while the failures continue * (`hardFailureCooldownMs` in `provider-health-ledger.ts`). The ledger is * durable, so the bench holds across workers and restarts. * diff --git a/src/lib/ai/coach/__tests__/tools-loop-always-reasoning-gateway.test.ts b/src/lib/ai/coach/__tests__/tools-loop-always-reasoning-gateway.test.ts index e66828695..f1be3a3be 100644 --- a/src/lib/ai/coach/__tests__/tools-loop-always-reasoning-gateway.test.ts +++ b/src/lib/ai/coach/__tests__/tools-loop-always-reasoning-gateway.test.ts @@ -209,6 +209,7 @@ async function turn(level: ReasoningLevel) { result: await client.generateCompletion(params), workingProvider: { providerType: "openai-compatible" }, fallbackHops: [], + tokensSpent: 0, }), ); return runCoachToolLoop({ @@ -268,11 +269,20 @@ describe("#1276: a gateway model that always reasons", () => { reasoning: { effort: "medium", summaries: true }, }) .then( - (r) => ({ ok: true as const, content: r.content }), + (r) => ({ ok: true as const, finishReason: r.finishReason }), (e: { kind?: string }) => ({ ok: false as const, kind: e.kind }), ); - // Never the thought as an answer. - expect(first).toEqual({ ok: false, kind: "reasoning_only" }); + // Never the thought as an answer, once it is labelled as one. The + // template-opened reply carries no label at all (no field, no tag) and + // comes from a model not yet seen reasoning: text cut off at the + // ceiling from such a model is its answer, cut off (a model that does + // not reason, asked with any level, must keep its answer), so it comes + // back as the answer with `finishReason: "length"`. + expect(first).toEqual( + mode === "template-opened" + ? { ok: true, finishReason: "length" } + : { ok: false, kind: "reasoning_only" }, + ); // The next turn reasons a normal amount. reasoningWords = 1200; diff --git a/src/lib/ai/coach/__tests__/tools-loop-budget.test.ts b/src/lib/ai/coach/__tests__/tools-loop-budget.test.ts index 678e79f30..f7ae2181d 100644 --- a/src/lib/ai/coach/__tests__/tools-loop-budget.test.ts +++ b/src/lib/ai/coach/__tests__/tools-loop-budget.test.ts @@ -61,6 +61,7 @@ function round( }, workingProvider: { providerType: "codex" }, fallbackHops: [], + tokensSpent: 0, }; } diff --git a/src/lib/ai/coach/__tests__/tools-loop-callbacks.test.ts b/src/lib/ai/coach/__tests__/tools-loop-callbacks.test.ts index 0c09a5057..941e2889a 100644 --- a/src/lib/ai/coach/__tests__/tools-loop-callbacks.test.ts +++ b/src/lib/ai/coach/__tests__/tools-loop-callbacks.test.ts @@ -37,6 +37,7 @@ function round(content: string, toolCalls?: Call[]) { }, workingProvider: { providerType: "anthropic" }, fallbackHops: [], + tokensSpent: 0, }; } diff --git a/src/lib/ai/coach/__tests__/tools-loop-final-round-wire.test.ts b/src/lib/ai/coach/__tests__/tools-loop-final-round-wire.test.ts index c7f1e2abf..a97c0f021 100644 --- a/src/lib/ai/coach/__tests__/tools-loop-final-round-wire.test.ts +++ b/src/lib/ai/coach/__tests__/tools-loop-final-round-wire.test.ts @@ -98,6 +98,7 @@ async function capTurn( result: await client.generateCompletion({ ...params, ...extra }), workingProvider: { providerType }, fallbackHops: [], + tokensSpent: 0, }), ); return runCoachToolLoop({ diff --git a/src/lib/ai/coach/__tests__/tools-loop-miss-payloads.test.ts b/src/lib/ai/coach/__tests__/tools-loop-miss-payloads.test.ts index 8cc8aa3ee..381e65d51 100644 --- a/src/lib/ai/coach/__tests__/tools-loop-miss-payloads.test.ts +++ b/src/lib/ai/coach/__tests__/tools-loop-miss-payloads.test.ts @@ -65,6 +65,7 @@ describe("coach tool loop — which results reach the grounding guard", () => { }, workingProvider: { providerType: "anthropic" }, fallbackHops: [], + tokensSpent: 0, }) .mockResolvedValueOnce({ result: { @@ -76,6 +77,7 @@ describe("coach tool loop — which results reach the grounding guard", () => { }, workingProvider: { providerType: "anthropic" }, fallbackHops: [], + tokensSpent: 0, }); const loop = await runCoachToolLoop(baseArgs); @@ -110,6 +112,7 @@ describe("coach tool loop — which results reach the grounding guard", () => { }, workingProvider: { providerType: "anthropic" }, fallbackHops: [], + tokensSpent: 0, }) .mockResolvedValueOnce({ result: { @@ -121,6 +124,7 @@ describe("coach tool loop — which results reach the grounding guard", () => { }, workingProvider: { providerType: "anthropic" }, fallbackHops: [], + tokensSpent: 0, }); const loop = await runCoachToolLoop(baseArgs); diff --git a/src/lib/ai/coach/__tests__/tools-loop.test.ts b/src/lib/ai/coach/__tests__/tools-loop.test.ts index 36509a431..a71f44af7 100644 --- a/src/lib/ai/coach/__tests__/tools-loop.test.ts +++ b/src/lib/ai/coach/__tests__/tools-loop.test.ts @@ -42,6 +42,7 @@ function completion(opts: { }, workingProvider: { providerType: "anthropic" }, fallbackHops: [], + tokensSpent: 0, }; } diff --git a/src/lib/ai/coach/results/__tests__/loop-result-tables.test.ts b/src/lib/ai/coach/results/__tests__/loop-result-tables.test.ts index 02ad76e56..674a9577b 100644 --- a/src/lib/ai/coach/results/__tests__/loop-result-tables.test.ts +++ b/src/lib/ai/coach/results/__tests__/loop-result-tables.test.ts @@ -35,6 +35,7 @@ function round(content: string, toolCalls?: unknown[]) { }, workingProvider: { providerType: "anthropic" }, fallbackHops: [], + tokensSpent: 0, }; } diff --git a/src/lib/ai/coach/tools/loop.ts b/src/lib/ai/coach/tools/loop.ts index 5bfb5cc6a..1e5441adb 100644 --- a/src/lib/ai/coach/tools/loop.ts +++ b/src/lib/ai/coach/tools/loop.ts @@ -401,7 +401,7 @@ export async function runCoachToolLoop(args: { }); if (spend) { await spend.settleRound({ - tokens: result.tokensUsed ?? 0, + tokens: (result.tokensUsed ?? 0) + fallback.tokensSpent, cachedTokens: result.cachedInputTokens ?? 0, servedBy: fallback.workingProvider.providerType as ProviderChainType, final: isFinal, diff --git a/src/lib/ai/coach/turn/__tests__/ledger-reasoning.test.ts b/src/lib/ai/coach/turn/__tests__/ledger-reasoning.test.ts index 986ad92b5..a51a337d3 100644 --- a/src/lib/ai/coach/turn/__tests__/ledger-reasoning.test.ts +++ b/src/lib/ai/coach/turn/__tests__/ledger-reasoning.test.ts @@ -106,6 +106,14 @@ describe("reserveTurnBudget and the turn ledger", () => { ]); }); + it("keeps what a failed call reported spending when the turn closes (#1279)", async () => { + const l = await ledger(); + await l.close(2_700); + expect(b.reconcileSpend.mock.calls.map((c) => [c[1], c[2]])).toEqual([ + [9_000, 2_700], + ]); + }); + it("settles a forced final round against the final reserve", async () => { const l = await ledger(); await l.settleRound({ diff --git a/src/lib/ai/coach/turn/budget.ts b/src/lib/ai/coach/turn/budget.ts index 603a08019..1fe05c0e1 100644 --- a/src/lib/ai/coach/turn/budget.ts +++ b/src/lib/ai/coach/turn/budget.ts @@ -41,8 +41,10 @@ export interface TurnLedger extends RoundSpend { /** * Give back everything still reserved and not settled. Called once the * turn is over, whatever way it ended; a second call does nothing. + * #1279 — `spent`, what a failed provider call reported spending, stays + * charged: a model that reasons without answering is billed for it. */ - close(): Promise; + close(spent?: number): Promise; } export async function reserveTurnBudget(args: { @@ -193,13 +195,13 @@ export function createTurnLedger(args: { } await settle(portion, tokens, cachedTokens, servedBy); }, - async close() { + async close(spent = 0) { if (closed) return; closed = true; const unused = open.reduce((sum, n) => sum + n, 0) + finalHeld; open.length = 0; finalHeld = 0; - if (unused > 0) await settle(unused, 0, 0, null); + if (unused > 0 || spent > 0) await settle(unused, spent, 0, null); }, }; } diff --git a/src/lib/ai/coach/turn/model.ts b/src/lib/ai/coach/turn/model.ts index 6b989b55d..4f5ce6707 100644 --- a/src/lib/ai/coach/turn/model.ts +++ b/src/lib/ai/coach/turn/model.ts @@ -12,6 +12,7 @@ import { AllProvidersFailedError, runStreamingRawCompletionWithFallback, } from "@/lib/ai/provider-runner"; +import { spentOnFailure } from "@/lib/ai/reasoning/answerless"; import type { ProviderChainType } from "@/lib/ai/provider-chain"; import { singleUserTurn, type CompletionResult } from "@/lib/ai/types"; import { PROMPT_VERSION } from "@/lib/ai/prompts/insight-generator"; @@ -689,7 +690,8 @@ export async function runTurnModel(args: { }); const result = fallback.result; await args.ledger.settleRound({ - tokens: result.tokensUsed ?? 0, + // #1279 — with what the hops that failed before it spent. + tokens: (result.tokensUsed ?? 0) + fallback.tokensSpent, cachedTokens: result.cachedInputTokens ?? 0, servedBy: fallback.workingProvider.providerType, final: false, @@ -729,20 +731,17 @@ export async function runTurnModel(args: { }; } catch (err) { // The provider chain failed: every round that returned is already - // settled; give back what is still reserved before the error frame. - await args.ledger.close(); - // #781 — the client walked away mid-generation. The request's abort - // signal is threaded into every provider call, so the teardown surfaces - // here as an abort-shaped failure with `request.signal` already flipped. - // Close the turn honestly instead of letting the user message dangle - // unanswered: persist an EMPTY assistant marker tagged - // `providerType: "cancelled"` (the same channel "refusal" and "nudge" - // already use for non-provider rows) so the thread shows the turn as - // interrupted and offers a retry on reload. The content is empty by - // construction — nothing guarded was produced, and anything the client - // ever SAW was persisted before the first token frame left, so no - // unguarded partial text is ever written. The reservation was refunded - // above; a retry pays exactly what a fresh turn pays. + // settled; give back what is still reserved before the error frame, less + // what the failed hops reported spending (#1279). + await args.ledger.close(spentOnFailure(err)); + // #781 — the client walked away mid-generation (the abort signal is + // threaded into every provider call). Close the turn honestly instead of + // leaving the user message dangling: persist an EMPTY assistant marker + // tagged `providerType: "cancelled"` (the channel "refusal" and "nudge" + // use for non-provider rows), so the thread shows the turn as interrupted + // and offers a retry. Empty by construction: nothing guarded was + // produced, and anything the client SAW was persisted before the first + // token frame left. A retry pays exactly what a fresh turn pays. if (signal.aborted) { await appendMessage({ conversationId, diff --git a/src/lib/ai/local-client.ts b/src/lib/ai/local-client.ts index 1371aaac0..ee5e5b107 100644 --- a/src/lib/ai/local-client.ts +++ b/src/lib/ai/local-client.ts @@ -1,13 +1,7 @@ -import { safeFetch } from "@/lib/safe-fetch"; import { callTimeoutMs } from "./effective-timeout"; import { annotate } from "@/lib/logging/context"; import { aiEgressPolicyFor } from "./local-host-allowlist"; -import type { - AIProvider, - AiMessage, - CompletionParams, - CompletionResult, -} from "./types"; +import type { AIProvider, CompletionParams, CompletionResult } from "./types"; import { buildOpenAIMessages, mapFinishReason, @@ -24,24 +18,29 @@ import { resetJsonModeDialectCache, type JsonModeDialect, } from "./json-dialect"; +import { emptyReplyError, withStrictJsonPrefix } from "./local-wire"; +import { RequestHops } from "./request-hops"; +import { learnFromRefusal } from "./wire-refusal"; import type { ReasoningEffort } from "./reasoning-effort"; import { ANSWERS_ONLY_UNREASONED, - annotateReasoningDowngrade, - isReasoningParameterRejection, learnedReasoningDialect, - rememberReasoningDialect, + noteReasoningSeen, + refusesReasoningField, } from "./reasoning/dialect-cache"; import { uncontrolledThinkingRoom } from "./reasoning/levels"; import { askAgainUnreasoned, + knownReasoner, reasonsUnasked, - withEarlierSpend, + repeatUnreasoned, } from "./reasoning/answerless"; import { ReasoningCollector, + StreamedThinkSplitter, separateMessage, takeReasoning, + type LocalReasoner, } from "./local-reasoning"; interface LocalClientConfig { @@ -57,16 +56,10 @@ interface LocalClientConfig { operatorTrusted?: boolean; } -const STRICT_JSON_PREFIX = - "Return strict JSON only, no markdown, no commentary.\n\n"; - /** - * v1.28.28 (#470) — per-endpoint JSON-mode dialect, learned at runtime. - * v1.33.1 — the machinery moved to `json-dialect.ts` when the - * OpenAI-compatible gateway provider needed the same learning; the two - * clients share one cache keyed by base URL. The names below stay exported - * from here because this client's tests and call sites were the first - * consumers. + * #470 — the per-endpoint JSON-mode dialect, learned at runtime in + * `json-dialect.ts` (one cache keyed by base URL, shared with the gateway + * client). The names stay exported here for this client's first consumers. */ type LocalJsonDialect = JsonModeDialect; @@ -76,58 +69,11 @@ export const resetLocalJsonDialectCache = resetJsonModeDialectCache; export { isResponseFormatRejection }; /** - * v1.22 (#89) — absolute backstop for the streaming path. The real ceiling is - * the per-idle-gap timer (`CompletionParams.timeoutMs`); this only stops a - * server that streams forever from pinning a worker. Ten minutes is far beyond - * any legitimate single Coach turn. + * #89 — the streaming path's absolute backstop. The real ceiling is the + * per-idle-gap timer; this only stops a server that streams forever. */ const STREAM_ABSOLUTE_CEILING_MS = 10 * 60_000; -/** - * Prepend the strict-JSON instruction to the FIRST user turn's text. Local - * model templates respect an in-message instruction reliably, so it is sent - * in BOTH dialects — it is the only JSON steering left when an endpoint - * rejects `response_format` (dialect `"none"`). Mirrors the pre-refactor - * behaviour, which prepended to the single user prompt. - */ -function withStrictJsonPrefix(messages: AiMessage[]): AiMessage[] { - const idx = messages.findIndex((m) => m.role === "user"); - if (idx === -1) return messages; - const out = [...messages]; - const target = out[idx]; - if (typeof target.content === "string") { - out[idx] = { ...target, content: `${STRICT_JSON_PREFIX}${target.content}` }; - } else { - // Vision parts: prepend a leading text part with the instruction. - out[idx] = { - ...target, - content: [{ type: "text", text: STRICT_JSON_PREFIX }, ...target.content], - }; - } - return out; -} - -/** - * The empty-reply error. `httpStatus: 0` is the sentinel the chain classifier - * treats as a hard failure; `kind` says whether the reply held reasoning and - * no answer, or nothing at all, so the log names it instead of "network". - */ -function emptyReplyError(hadReasoning: boolean, spent: number | null): Error { - const err = new Error( - hadReasoning - ? "Local AI returned reasoning but no answer" - : "Local AI returned empty content", - ); - Object.assign(err, { - httpStatus: 0, - kind: hadReasoning ? "reasoning_only" : "empty_response", - upstream: "local", - // #1279 — what the reply cost, for the day's ledger to charge. - ...(spent !== null ? { tokensUsed: spent } : {}), - }); - return err; -} - /** * Talks to OpenAI-compatible servers: local runtimes (Ollama, LocalAI, * LM Studio, vLLM, …) AND hosted gateways (LiteLLM, OpenRouter, any @@ -138,12 +84,10 @@ function emptyReplyError(hadReasoning: boolean, spent: number | null): Error { * `LocalJsonDialect` above). The strict-JSON instruction prepended to the * user message stays in both dialects. * - * v1.20.0 — tool-calling DEGRADES silently here: most local servers reject an - * unknown `tools` field, so the client never forwards it (capability flag - * below). F1's tool loop must therefore tolerate a provider that never returns - * `toolCalls` and answer from base context. Prompt-caching, where the local - * server supports it (vLLM / Ollama prefix cache), is transparent — no flag, - * no harm. + * Tool calling degrades silently: most local servers reject an unknown + * `tools` field, so the client never forwards it (`supportsTools`), and the + * Coach's tool loop answers from base context. Prompt caching, where the + * server has it (vLLM / Ollama prefix cache), is transparent. */ export class LocalOpenAICompatibleClient implements AIProvider { readonly type = "local" as const; @@ -228,7 +172,11 @@ export class LocalOpenAICompatibleClient implements AIProvider { // v1.41 — a call that carries its own resolved level wins over it. // #1267 — once a model answered nothing while reasoning, the entry's // setting gives way to the `none` that `callReasoningEffort` sends. - ...(!params.reasoning && this.reasoningEffort && callEffort === null + // #1279 — not once the endpoint refused the field. + ...(!params.reasoning && + this.reasoningEffort && + callEffort === null && + this.sentReasoningEffort(params) !== null ? { reasoning_effort: this.reasoningEffort } : {}), ...(callEffort ? { reasoning_effort: callEffort } : {}), @@ -279,19 +227,36 @@ export class LocalOpenAICompatibleClient implements AIProvider { // expires, a call without a level of its own included. if (learned === ANSWERS_ONLY_UNREASONED) return "none"; if (!params.reasoning) return null; - if (learned === "none") return null; + if (refusesReasoningField(learned)) return null; return params.reasoning.effort === "off" ? "none" : params.reasoning.effort; } + /** + * #1276 — what this endpoint and model have shown of their reasoning, for + * one request (`LocalReasoner`). Seen on a request that asked to reason + * counts only for requests that ask too. + */ + private reasoner(params: CompletionParams, room: boolean): LocalReasoner { + const key = this.reasoningDialectKey(); + const sent = this.sentReasoningEffort(params); + const asked = sent !== null && sent !== "none"; + return { + room, + known: knownReasoner(key, asked), + seen: () => noteReasoningSeen(key, asked), + }; + } + /** * The `reasoning_effort` this request carries, whichever level decided it: - * the call's own (`callReasoningEffort`), else the entry's #1126 setting. + * the call's own (`callReasoningEffort`), else the entry's #1126 setting, + * unless the endpoint refused the field (#1279). */ private sentReasoningEffort(params: CompletionParams): string | null { - return ( - this.callReasoningEffort(params) ?? - (params.reasoning ? null : this.reasoningEffort) - ); + const call = this.callReasoningEffort(params); + if (call !== null || params.reasoning) return call; + const learned = learnedReasoningDialect(this.reasoningDialectKey()); + return refusesReasoningField(learned) ? null : this.reasoningEffort; } /** @@ -308,31 +273,93 @@ export class LocalOpenAICompatibleClient implements AIProvider { */ private learnAnswerlessReasoning( params: CompletionParams, + hops: RequestHops, hadReasoning: boolean, finishReason: string | undefined, ): boolean { - return askAgainUnreasoned({ - key: this.reasoningDialectKey(), - provider: "local", - signals: "ai.local", - sent: this.sentReasoningEffort(params), - hadReasoning, - finishReason, + return ( + // The lesson holds even when the budget leaves no repeat for it. + askAgainUnreasoned({ + key: this.reasoningDialectKey(), + provider: "local", + signals: "ai.local", + sent: this.sentReasoningEffort(params), + hadReasoning, + finishReason, + }) && hops.canSend + ); + } + + /** + * #1279 — the call once more with `none` (streamed when `onDelta` is + * given), carrying this reply's spend; see `repeatUnreasoned`. + */ + private askUnreasoned( + params: CompletionParams, + hops: RequestHops, + reply: { hadReasoning: boolean; spent: number | null }, + onDelta?: (delta: string) => void, + ): Promise { + return repeatUnreasoned( + onDelta + ? this.generateCompletionStream(params, onDelta, hops) + : this.generateCompletion(params, hops), + reply.spent, + () => emptyReplyError(reply.hadReasoning, reply.spent), + ); + } + + /** + * A non-2xx reply. A refused reasoning field or JSON flag is learned and + * the call asked once more without it (`wire-refusal.ts`), and a refused + * stream (`streamed`; some servers 400 on `stream` / `stream_options`) + * once more buffered, while the call's request budget allows. Otherwise + * the structured error; local endpoints and their proxies commonly leak + * bearer tokens in error envelopes, so the excerpt is stripped first. + */ + private async failed( + res: Response, + params: CompletionParams, + hops: RequestHops, + sent: { effort: string | null; json: boolean; streamed: boolean }, + ): Promise { + const bodyExcerpt = sanitiseBodyExcerpt(await res.text().catch(() => "")); + const lesson = learnFromRefusal({ + status: res.status, + body: bodyExcerpt, + model: this.config.model, + reasoning: sent.effort + ? { + sent: sent.effort, + key: this.reasoningDialectKey(), + provider: "local", + names: /reasoning|think/i, + } + : null, + json: sent.json + ? { endpoint: this.config.baseUrl, action: "ai.local.jsonDialect" } + : null, + }); + if ((lesson || sent.streamed) && hops.canSend) { + return this.generateCompletion(params, hops); + } + throw Object.assign(new Error(`Local AI request failed (${res.status})`), { + httpStatus: res.status, + upstream: "local", + model: this.config.model, + baseUrl: this.config.baseUrl, + bodyExcerpt, + ...(lesson === "reasoning" ? { reasoningRefused: true } : {}), }); } - /** #1279 — the streamed call once more, carrying this reply's spend. */ - private streamAgain( - params: CompletionParams, - onDelta: (delta: string) => void, - spent: number | null, - ): Promise { - const again = this.generateCompletionStream(params, onDelta); - return withEarlierSpend(again, spent); - } - + /** + * `hops` is the call's request budget (`request-hops.ts`): a caller never + * passes it, every repeat inside the call hands it on. + */ async generateCompletion( params: CompletionParams, + hops: RequestHops = new RequestHops(), ): Promise { const { url, headers, body, jsonDialect, room } = this.buildRequest( params, @@ -357,7 +384,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { const egress = aiEgressPolicyFor(url, { operatorTrusted: this.config.operatorTrusted, }); - const res = await safeFetch( + const res = await hops.fetch( url, { method: "POST", @@ -375,55 +402,12 @@ export class LocalOpenAICompatibleClient implements AIProvider { ); if (!res.ok) { - // Mirror the openai-client structured-error pattern. Local endpoints - // also commonly leak bearer tokens in error envelopes (proxies), so - // strip anything that looks like a key before logging. - const rawBody = await res.text().catch(() => ""); - const bodyExcerpt = sanitiseBodyExcerpt(rawBody); - // v1.41 — a server that refuses the reasoning field is asked once more - // without it and remembers. Before the JSON dialect, so a complaint - // about reasoning never switches JSON mode off. - const sentEffort = this.callReasoningEffort(params); - if ( - sentEffort && - isReasoningParameterRejection( - res.status, - bodyExcerpt, - /reasoning|think/i, - this.config.model, - ) - ) { - rememberReasoningDialect(this.reasoningDialectKey(), "none"); - annotateReasoningDowngrade("local", sentEffort, "none"); - return this.generateCompletion(params); - } - // v1.28.28 (#470) — dialect self-heal. A 4xx whose body names - // `response_format` (or an unknown parameter) means this endpoint - // rejects the standard JSON flag: learn the no-flag dialect for the - // baseUrl and retry ONCE without it (the recursion terminates because - // the cached dialect is now "none"). Unrelated 4xx/5xx errors are not - // retried. - if ( - params.responseFormat === "json" && - jsonDialect === "response_format" && - isResponseFormatRejection(res.status, bodyExcerpt) - ) { - rememberJsonModeDialect(this.config.baseUrl, "none"); - annotate({ - action: { name: "ai.local.jsonDialect" }, - meta: { dialect: "none", httpStatus: res.status }, - }); - return this.generateCompletion(params); - } - const err = new Error(`Local AI request failed (${res.status})`); - Object.assign(err, { - httpStatus: res.status, - upstream: "local", - model: this.config.model, - baseUrl: this.config.baseUrl, - bodyExcerpt, + return this.failed(res, params, hops, { + effort: this.sentReasoningEffort(params), + json: + params.responseFormat === "json" && jsonDialect === "response_format", + streamed: false, }); - throw err; } // v1.28.28 (#470) — the standard `response_format` request succeeded: @@ -470,12 +454,12 @@ export class LocalOpenAICompatibleClient implements AIProvider { params, json.choices?.[0]?.message, finish, - room, + this.reasoner(params, room), ); if (!content) { const spent = json.usage?.total_tokens ?? null; - if (this.learnAnswerlessReasoning(params, hadReasoning, finish)) { - return withEarlierSpend(this.generateCompletion(params), spent); + if (this.learnAnswerlessReasoning(params, hops, hadReasoning, finish)) { + return this.askUnreasoned(params, hops, { hadReasoning, spent }); } // The reasoning that did arrive still reaches `onReasoning`, as before. collector.result( @@ -507,6 +491,68 @@ export class LocalOpenAICompatibleClient implements AIProvider { }; } + /** + * A server that ignored the stream flag and answered buffered (or sent no + * readable body): parsed like the non-streaming path rather than paying a + * second round-trip, and the answer handed to `onDelta` whole. + */ + private async bufferedInsteadOfStream( + res: Response, + params: CompletionParams, + onDelta: (delta: string) => void, + hops: RequestHops, + room: boolean, + ): Promise { + const json = (await res.json().catch(() => null)) as { + choices?: Array<{ + message?: { content?: string }; + finish_reason?: string; + }>; + usage?: { + total_tokens?: number; + prompt_tokens_details?: { cached_tokens?: number }; + }; + } | null; + // The server ignored `stream` and answered buffered — the embedded + // failure can ride this body exactly as it rides the non-streaming one. + raiseEmbeddedError(json, { + upstream: "local", + label: "local AI endpoint", + model: this.config.model, + }); + const { content, hadReasoning, collector } = separateMessage( + params, + json?.choices?.[0]?.message, + json?.choices?.[0]?.finish_reason, + this.reasoner(params, room), + ); + if (!content) { + const spent = json?.usage?.total_tokens ?? null; + const finish = json?.choices?.[0]?.finish_reason; + if (this.learnAnswerlessReasoning(params, hops, hadReasoning, finish)) { + return this.askUnreasoned( + params, + hops, + { hadReasoning, spent }, + onDelta, + ); + } + collector.result(null); + throw emptyReplyError(hadReasoning, spent); + } + const reasoning = collector.result(null); + onDelta(content); + return { + content, + tokensUsed: json?.usage?.total_tokens ?? null, + cachedInputTokens: json ? parseCachedTokens(json) : null, + model: this.config.model, + providerType: "local", + finishReason: mapFinishReason(json?.choices?.[0]?.finish_reason), + ...(reasoning ? { reasoning } : {}), + }; + } + /** * v1.22 (#89) — true token streaming over the OpenAI-compatible * `stream: true` wire. POSTs with `stream: true`, reads the SSE response body @@ -530,8 +576,12 @@ export class LocalOpenAICompatibleClient implements AIProvider { async generateCompletionStream( params: CompletionParams, onDelta: (delta: string) => void, + hops: RequestHops = new RequestHops(), ): Promise { - const { url, headers, body, room } = this.buildRequest(params, true); + const { url, headers, body, jsonDialect, room } = this.buildRequest( + params, + true, + ); const egress = aiEgressPolicyFor(url, { operatorTrusted: this.config.operatorTrusted, }); @@ -547,7 +597,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { let res: Response; try { - res = await safeFetch( + res = await hops.fetch( url, { method: "POST", headers, body }, { @@ -558,68 +608,26 @@ export class LocalOpenAICompatibleClient implements AIProvider { signal, }, ); - } catch { + } catch (err) { // Network / private-host / abort before the stream opened → fall back to // the buffered path, which surfaces the structured error the chain runner // understands (or succeeds on a server that simply doesn't stream). - return this.generateCompletion(params); + if (!hops.canSend) throw err; + return this.generateCompletion(params, hops); } if (!res.ok) { - // The server rejected the streaming request (some servers 400 on - // `stream`/`stream_options`). Retry once on the buffered path so a - // non-streaming server still works and surfaces its real status. - return this.generateCompletion(params); + return this.failed(res, params, hops, { + effort: this.sentReasoningEffort(params), + json: + params.responseFormat === "json" && jsonDialect === "response_format", + streamed: true, + }); } const contentType = res.headers.get("content-type") ?? ""; if (!contentType.includes("text/event-stream") || !res.body) { - // The server ignored the stream flag and returned a buffered completion - // (or there is no readable body). Parse it like the non-streaming path - // rather than paying a second round-trip. - const json = (await res.json().catch(() => null)) as { - choices?: Array<{ - message?: { content?: string }; - finish_reason?: string; - }>; - usage?: { - total_tokens?: number; - prompt_tokens_details?: { cached_tokens?: number }; - }; - } | null; - // The server ignored `stream` and answered buffered — the embedded - // failure can ride this body exactly as it rides the non-streaming one. - raiseEmbeddedError(json, { - upstream: "local", - label: "local AI endpoint", - model: this.config.model, - }); - const { content, hadReasoning, collector } = separateMessage( - params, - json?.choices?.[0]?.message, - json?.choices?.[0]?.finish_reason, - room, - ); - if (!content) { - const spent = json?.usage?.total_tokens ?? null; - const finish = json?.choices?.[0]?.finish_reason; - if (this.learnAnswerlessReasoning(params, hadReasoning, finish)) { - return this.streamAgain(params, onDelta, spent); - } - collector.result(null); - throw emptyReplyError(hadReasoning, spent); - } - const reasoning = collector.result(null); - onDelta(content); - return { - content, - tokensUsed: json?.usage?.total_tokens ?? null, - cachedInputTokens: json ? parseCachedTokens(json) : null, - model: this.config.model, - providerType: "local", - finishReason: mapFinishReason(json?.choices?.[0]?.finish_reason), - ...(reasoning ? { reasoning } : {}), - }; + return this.bufferedInsteadOfStream(res, params, onDelta, hops, room); } const reader = res.body.getReader(); @@ -636,44 +644,12 @@ export class LocalOpenAICompatibleClient implements AIProvider { // reasoning never reaches `onDelta`; it would otherwise show up as the // answer. #1276 — for every caller (only one that asked is told). const collector = new ReasoningCollector(params); - let thinkState: "detect" | "inside" | "passthrough" = "detect"; - let pending = ""; - const passContent = (text: string) => { + const splitter = new StreamedThinkSplitter(collector, (text) => { if (!text) return; collector.finish(); content += text; onDelta(text); - }; - const onContent = (delta: string) => { - if (thinkState === "passthrough") { - passContent(delta); - return; - } - pending += delta; - if (thinkState === "detect") { - const lead = pending.replace(/^\s+/, ""); - if (lead.length < "".length && "".startsWith(lead)) { - return; - } - if (!lead.startsWith("")) { - thinkState = "passthrough"; - const out = pending; - pending = ""; - passContent(out); - return; - } - thinkState = "inside"; - pending = lead.slice("".length); - } - const close = pending.indexOf(""); - if (close === -1) return; - collector.add(pending.slice(0, close)); - collector.finish(); - const rest = pending.slice(close + "".length).replace(/^\s+/, ""); - pending = ""; - thinkState = "passthrough"; - passContent(rest); - }; + }); // An SSE frame can carry the same embedded failure the buffered body // carries. Remember the first one; it only matters if the stream ends @@ -741,7 +717,7 @@ export class LocalOpenAICompatibleClient implements AIProvider { collector.add(reasoningDelta); } const delta = choiceDelta?.content; - if (delta) onContent(delta); + if (delta) splitter.push(delta); if (typeof chunk.usage?.total_tokens === "number") { tokensUsed = chunk.usage.total_tokens; } @@ -773,24 +749,16 @@ export class LocalOpenAICompatibleClient implements AIProvider { reader.cancel().catch(() => {}); } - // A reply short enough to still sit in the `` detector is prose; - // an unclosed block is reasoning that ran out of budget. - // (The closures above move `thinkState`; read it afresh.) - const endState = thinkState as "detect" | "inside" | "passthrough"; - if (endState === "detect") { - thinkState = "passthrough"; - passContent(pending); - } else if (endState === "inside") { - collector.add(pending); - } - pending = ""; + const { unclosed } = splitter.end(); // #1276 — a lone ``, or unlabelled text that filled the ceiling. // Sent deltas are a liveness signal only; the guards read `content`. + const reasoner = this.reasoner(params, room); const tail = takeReasoning(collector, content, { fieldedReasoning: streamedReasoning, finishReason, - reasoningRoom: room, + reasoner, }); + if (streamedReasoning || splitter.opened || tail.took) reasoner.seen(); content = tail.content; if (!content) { // A stream that ended with a failure frame and no token reports the @@ -801,10 +769,16 @@ export class LocalOpenAICompatibleClient implements AIProvider { model: this.config.model, }); // An unclosed inline `` block is reasoning too. - const hadReasoning = - streamedReasoning || endState === "inside" || tail.took; - if (this.learnAnswerlessReasoning(params, hadReasoning, finishReason)) { - return this.streamAgain(params, onDelta, tokensUsed); + const hadReasoning = streamedReasoning || unclosed || tail.took; + if ( + this.learnAnswerlessReasoning(params, hops, hadReasoning, finishReason) + ) { + return this.askUnreasoned( + params, + hops, + { hadReasoning, spent: tokensUsed }, + onDelta, + ); } collector.result(reasoningTokens); throw emptyReplyError(hadReasoning, tokensUsed); diff --git a/src/lib/ai/local-reasoning.ts b/src/lib/ai/local-reasoning.ts index 7584bfb38..124686018 100644 --- a/src/lib/ai/local-reasoning.ts +++ b/src/lib/ai/local-reasoning.ts @@ -74,12 +74,23 @@ export function messageReasoning(message: unknown): string | null { return text ?? null; } +/** + * What one request's endpoint and model have shown of their reasoning + * (#1276): `room` the ceiling held room for it, `known` the model was seen + * reasoning before (`knownReasoner`), `seen` records a reply that reasoned. + */ +export interface LocalReasoner { + room: boolean; + known: boolean; + seen(): void; +} + /** * #1276 — take the reasoning out of `content` (`answerOf`: an inline - * `` block or only its closing tag, or unlabelled text that filled a - * ceiling sized for thinking) and hand it to `collector`. Returns the answer - * and whether anything was taken. For every caller: reasoning is never an - * answer, asked for or not. + * `` block or only its closing tag, or, from a model known to reason, + * unlabelled text that filled a ceiling sized for thinking) and hand it to + * `collector`. Returns the answer and whether anything was taken. For every + * caller: reasoning is never an answer, asked for or not. */ export function takeReasoning( collector: ReasoningCollector, @@ -87,10 +98,16 @@ export function takeReasoning( reply: { fieldedReasoning: boolean; finishReason: string | undefined; - reasoningRoom: boolean; + reasoner: LocalReasoner; }, ): { content: string; took: boolean } { - const separated = answerOf(content, { ...reply, hasToolCalls: false }); + const separated = answerOf(content, { + fieldedReasoning: reply.fieldedReasoning, + finishReason: reply.finishReason, + hasToolCalls: false, + reasoningRoom: reply.reasoner.room, + knownReasoner: reply.reasoner.known, + }); if (separated.reasoning) collector.add(separated.reasoning); if (separated.unseparated) { emitSignal({ @@ -110,7 +127,7 @@ export function separateMessage( params: CompletionParams, message: { content?: string | null } | undefined, finishReason: string | undefined, - reasoningRoom: boolean, + reasoner: LocalReasoner, ): { content: string; hadReasoning: boolean; collector: ReasoningCollector } { const collector = new ReasoningCollector(params); const fielded = messageReasoning(message); @@ -118,7 +135,77 @@ export function separateMessage( const { content, took } = takeReasoning(collector, message?.content ?? "", { fieldedReasoning: fielded !== null, finishReason, - reasoningRoom, + reasoner, }); - return { content, hadReasoning: fielded !== null || took, collector }; + const hadReasoning = fielded !== null || took; + if (hadReasoning) reasoner.seen(); + return { content, hadReasoning, collector }; +} + +/** + * A leading inline `` block in streamed `content`, split off delta by + * delta: the reasoning goes to the collector, never to `pass`, which gets the + * answer as it arrives. + */ +export class StreamedThinkSplitter { + private state: "detect" | "inside" | "passthrough" = "detect"; + private pending = ""; + /** A `` block opened. */ + opened = false; + + constructor( + private readonly collector: ReasoningCollector, + private readonly pass: (text: string) => void, + ) {} + + push(delta: string): void { + if (this.state === "passthrough") { + this.pass(delta); + return; + } + this.pending += delta; + if (this.state === "detect") { + const lead = this.pending.replace(/^\s+/, ""); + if (lead.length < "".length && "".startsWith(lead)) { + return; + } + if (!lead.startsWith("")) { + this.state = "passthrough"; + const out = this.pending; + this.pending = ""; + this.pass(out); + return; + } + this.state = "inside"; + this.opened = true; + this.pending = lead.slice("".length); + } + const close = this.pending.indexOf(""); + if (close === -1) return; + this.collector.add(this.pending.slice(0, close)); + this.collector.finish(); + const rest = this.pending + .slice(close + "".length) + .replace(/^\s+/, ""); + this.pending = ""; + this.state = "passthrough"; + this.pass(rest); + } + + /** + * The stream ended. A reply short enough to still sit in the detector is + * prose; an unclosed block is reasoning that ran out of budget, and the + * result says so. + */ + end(): { unclosed: boolean } { + const unclosed = this.state === "inside"; + if (this.state === "detect") { + this.state = "passthrough"; + this.pass(this.pending); + } else if (unclosed) { + this.collector.add(this.pending); + } + this.pending = ""; + return { unclosed }; + } } diff --git a/src/lib/ai/local-wire.ts b/src/lib/ai/local-wire.ts new file mode 100644 index 000000000..3a7b0fc08 --- /dev/null +++ b/src/lib/ai/local-wire.ts @@ -0,0 +1,58 @@ +/** + * Request and error shapes of the Local client that need nothing of its state. + * Split out of `local-client.ts`; that client is their only importer. + * + * Server-only. + */ +import type { AiMessage } from "./types"; + +const STRICT_JSON_PREFIX = + "Return strict JSON only, no markdown, no commentary.\n\n"; + +/** + * Prepend the strict-JSON instruction to the FIRST user turn's text. Local + * model templates respect an in-message instruction reliably, so it is sent + * in BOTH dialects — it is the only JSON steering left when an endpoint + * rejects `response_format` (dialect `"none"`). Mirrors the pre-refactor + * behaviour, which prepended to the single user prompt. + */ +export function withStrictJsonPrefix(messages: AiMessage[]): AiMessage[] { + const idx = messages.findIndex((m) => m.role === "user"); + if (idx === -1) return messages; + const out = [...messages]; + const target = out[idx]; + if (typeof target.content === "string") { + out[idx] = { ...target, content: `${STRICT_JSON_PREFIX}${target.content}` }; + } else { + // Vision parts: prepend a leading text part with the instruction. + out[idx] = { + ...target, + content: [{ type: "text", text: STRICT_JSON_PREFIX }, ...target.content], + }; + } + return out; +} + +/** + * The empty-reply error. `httpStatus: 0` is the sentinel the chain classifier + * treats as a hard failure; `kind` says whether the reply held reasoning and + * no answer, or nothing at all, so the log names it instead of "network". + */ +export function emptyReplyError( + hadReasoning: boolean, + spent: number | null, +): Error { + const err = new Error( + hadReasoning + ? "Local AI returned reasoning but no answer" + : "Local AI returned empty content", + ); + Object.assign(err, { + httpStatus: 0, + kind: hadReasoning ? "reasoning_only" : "empty_response", + upstream: "local", + // #1279 — what the reply cost, for the day's ledger to charge. + ...(spent !== null ? { tokensUsed: spent } : {}), + }); + return err; +} diff --git a/src/lib/ai/openai-client.ts b/src/lib/ai/openai-client.ts index 8380153a7..feb6efd5c 100644 --- a/src/lib/ai/openai-client.ts +++ b/src/lib/ai/openai-client.ts @@ -1,4 +1,3 @@ -import { safeFetch } from "@/lib/safe-fetch"; import { callTimeoutMs } from "./effective-timeout"; import { annotate } from "@/lib/logging/context"; import { emitSignal } from "@/lib/logging/signal"; @@ -15,16 +14,18 @@ import { } from "./openai-capabilities"; import { ANSWERS_ONLY_UNREASONED, - annotateReasoningDowngrade, - isReasoningParameterRejection, learnedReasoningDialect, - rememberReasoningDialect, + noteReasoningSeen, + refusesReasoningField, } from "./reasoning/dialect-cache"; import { askAgainUnreasoned, + knownReasoner, reasonsUnasked, - withEarlierSpend, + repeatUnreasoned, } from "./reasoning/answerless"; +import { RequestHops } from "./request-hops"; +import { learnFromRefusal } from "./wire-refusal"; import { REASONING_THINKING_BUDGET, uncontrolledThinkingRoom, @@ -39,7 +40,6 @@ import { } from "./reasoning/support"; import { hasLearnedJsonModeDialect, - isResponseFormatRejection, jsonModeDialectFor, rememberJsonModeDialect, } from "./json-dialect"; @@ -167,8 +167,13 @@ export class OpenAIClient implements AIProvider { return this.type === "openai-compatible"; } + /** + * `hops` is the call's request budget (`request-hops.ts`): a caller never + * passes it, every repeat inside the call hands it on. + */ async generateCompletion( params: CompletionParams, + hops: RequestHops = new RequestHops(), ): Promise { const url = `${this.config.baseUrl.replace(/\/$/, "")}/chat/completions`; @@ -209,10 +214,18 @@ export class OpenAIClient implements AIProvider { this.config.model, ); const reasoningWire = this.planReasoning(params, hasTools, capabilities); - // #1126 — the gateway entry's own setting, for a call without a resolved - // level (sent below unless the wire already carries `none`). + // #1126 — the gateway entry's own setting, for a call without a level, + // unless the wire carries one or the endpoint refused the field (#1279). const entryEffort = - !params.reasoning && this.isGateway ? this.reasoningEffort : null; + !params.reasoning && + this.isGateway && + this.reasoningEffort && + reasoningWire.kind === "none" && + !this.refusesField() + ? this.reasoningEffort + : null; + const sent = + reasoningWire.kind !== "none" ? reasoningWire.effort : entryEffort; const thinkingTokens = this.thinkingRoom(params, reasoningWire); const tokenBudget = (params.maxTokens ?? 1000) + thinkingTokens; const capabilityParams = capabilities.supportsSamplingControls @@ -234,7 +247,7 @@ export class OpenAIClient implements AIProvider { const trimmedKey = this.config.apiKey?.trim(); if (trimmedKey) headers.Authorization = `Bearer ${trimmedKey}`; - const res = await safeFetch( + const res = await hops.fetch( url, { method: "POST", @@ -256,16 +269,11 @@ export class OpenAIClient implements AIProvider { ? { prompt_cache_key: params.cacheKey } : {}), // #1126 — the gateway entry's reasoning setting; Default sends - // nothing. `api.openai.com` and Codex never get it. v1.41 — a call - // that carries its own resolved level (`params.reasoning`) wins. - // #1267 — once a model answered nothing while reasoning, the wire + // nothing. `api.openai.com` and Codex never get it. A call that + // carries its own resolved level (`params.reasoning`) wins, and + // (#1267) once a model answered nothing while reasoning, the wire // below carries `none` instead (`planReasoning`). - ...(!params.reasoning && - this.isGateway && - this.reasoningEffort && - reasoningWire.kind === "none" - ? { reasoning_effort: this.reasoningEffort } - : {}), + ...(entryEffort ? { reasoning_effort: this.reasoningEffort } : {}), ...(reasoningWire.kind === "reasoning_effort" ? { reasoning_effort: reasoningWire.effort } : reasoningWire.kind === "openrouter" @@ -273,30 +281,15 @@ export class OpenAIClient implements AIProvider { : {}), }), }, - // 60 s ceiling so a tar-pit upstream cannot pin a worker - // indefinitely. Real completions land well inside this budget. - // v1.11.2 — the base URL is user/admin-overridable (BYO gateway), so pin - // the connect-time DNS check: a base URL resolving to a private/metadata - // address is rejected, closing the SSRF/rebinding surface. - // v1.20.1 — compose the caller's cancel signal (Coach SSE disconnect) so - // a mid-generation abort tears the upstream call down early. - // The record owner's response-timeout setting, else the surface value, else 60 s. - // v1.33.1 (#470) — the gateway tag reuses the Local provider's host - // policy verbatim: public hosts always; a private host only when the - // operator granted its origin (`AI_PRIVATE_ORIGINS` or the legacy host - // list), and then only through the pinned operator-approved dispatcher. - // - // v1.37.30 — the `admin-key` tag consults the same policy. Its base URL - // is the one other operator-typed URL in this client (an operator can - // point the global provider at a private OpenAI-compatible proxy, e.g. - // an OAuth sidecar), and both ends of the grant are the operator's own: - // the URL comes from the admin settings, the allowlist from the - // operator's environment. Every non-operator construction of this tag - // pins the public `api.openai.com` constant, for which the policy - // resolves to `true` without ever consulting the allowlist — so the - // personal-key posture is unchanged. The `codex` tag keeps the hard - // pin: its base URL is a repository constant, there is nothing an - // operator legitimately redirects. + // The record owner's response-timeout setting (else the surface value, + // else 60 s), and the caller's cancel signal (a Coach disconnect tears + // the upstream call down). The gateway's base URL and the operator's + // `admin-key` one are typed by a person, so they take the Local + // provider's host policy: public hosts always, a private host only for + // an origin the operator granted (`AI_PRIVATE_ORIGINS`), dialled through + // the pinned operator-approved dispatcher; a personal key pins the + // public `api.openai.com`, for which the policy never consults the + // allowlist. `codex` keeps the hard pin: its URL is a repository constant. { timeoutMs: callTimeoutMs(params, this.responseTimeoutSeconds), ...(this.isGateway || this.type === "admin-key" @@ -309,68 +302,7 @@ export class OpenAIClient implements AIProvider { ); if (!res.ok) { - // Pull as much of the body as we can so upstream incidents (OpenAI 5xx, - // model-not-found, quota exceeded) are diagnosable from logs instead of - // surfacing as an opaque "OpenAI request failed (500)". Strip anything - // that looks like an API key from the excerpt before logging. - const rawBody = await res.text().catch(() => ""); - const bodyExcerpt = sanitiseBodyExcerpt(rawBody); - // v1.41 — an endpoint or model that refuses the reasoning field is - // asked once more without it and remembers, so the answer never fails - // on a reasoning parameter. Checked before the JSON dialect: a body - // naming the reasoning field must not switch JSON mode off. - if ( - reasoningWire.kind !== "none" && - isReasoningParameterRejection( - res.status, - bodyExcerpt, - /reasoning|thinking/i, - this.config.model, - ) - ) { - rememberReasoningDialect(this.reasoningDialectKey(), "none"); - annotateReasoningDowngrade(this.type, reasoningWire.effort, "none"); - return this.generateCompletion(params); - } - // v1.33.1 (#470) — dialect self-heal, gateway only. A 4xx whose body - // names `response_format` (or an unknown parameter) means this endpoint - // rejects the standard JSON flag: learn the no-flag dialect for the base - // URL and retry ONCE without it. The recursion terminates because the - // cached dialect is now "none". Unrelated 4xx/5xx errors are not retried - // and surface as the structured error they always were. - if ( - this.isGateway && - sendJsonFormat && - isResponseFormatRejection(res.status, bodyExcerpt) - ) { - rememberJsonModeDialect(this.config.baseUrl, "none"); - annotate({ - action: { name: "ai.compat.jsonDialect" }, - meta: { dialect: "none", httpStatus: res.status }, - }); - return this.generateCompletion(params); - } - // v1.42 — a proxy that stopped offering the configured model answers - // with a bare 500 that names nothing. Ask its model listing (free, no - // completion) and say so plainly when the name is gone, so the chain, - // the health card and the log name the actual fix. - const modelNotServed = await this.modelNoLongerServed(res.status); - const err = new Error( - modelNotServed - ? `${this.isGateway ? "OpenAI-compatible gateway" : "OpenAI"} request failed (${res.status}): model "${this.config.model}" is not offered by this endpoint` - : `${this.isGateway ? "OpenAI-compatible gateway" : "OpenAI"} request failed (${res.status})`, - ); - // Keep the body in a structured field rather than the message so - // Error.message stays short and the excerpt lands in dedicated log - // fields (bodyExcerpt) that can be filtered/truncated centrally. - Object.assign(err, { - httpStatus: res.status, - upstream: this.isGateway ? "openai-compatible" : "openai", - model: this.config.model, - bodyExcerpt, - ...(modelNotServed ? { modelNotServed: true } : {}), - }); - throw err; + return this.failed(res, params, hops, { sent, sendJsonFormat }); } // v1.33.1 (#470) — the standard `response_format` request succeeded: pin @@ -415,7 +347,10 @@ export class OpenAIClient implements AIProvider { const choice = json.choices?.[0]; const toolCalls = parseOpenAIToolCalls(choice); - const reply = this.separateReasoning(choice, toolCalls, thinkingTokens); + const reply = this.separateReasoning(choice, toolCalls, { + thinkingTokens, + sent, + }); let content: string = reply.content; // v1.28.28 (#470) — gateway JSON-mode shim. LiteLLM (and similar @@ -439,9 +374,8 @@ export class OpenAIClient implements AIProvider { // A reply with tool calls and no prose is valid (F1 tool loop); only an // empty reply with neither content NOR tool calls is an error. if (!content && !toolCalls) { - return this.answerless(params, { - sent: - reasoningWire.kind !== "none" ? reasoningWire.effort : entryEffort, + return this.answerless(params, hops, { + sent, hadReasoning: reply.hadReasoning, finishReason: choice?.finish_reason, spent: json.usage?.total_tokens ?? null, @@ -464,8 +398,9 @@ export class OpenAIClient implements AIProvider { ).map((field) => ({ [field]: message[field] })); const usage = json.usage as { completion_tokens_details?: { reasoning_tokens?: number } } | undefined; - const learnedNone = - learnedReasoningDialect(this.reasoningDialectKey()) === "none"; + const learnedNone = refusesReasoningField( + learnedReasoningDialect(this.reasoningDialectKey()), + ); return { content, @@ -519,6 +454,7 @@ export class OpenAIClient implements AIProvider { */ private answerless( params: CompletionParams, + hops: RequestHops, reply: { sent: string | null; hadReasoning: boolean; @@ -526,8 +462,21 @@ export class OpenAIClient implements AIProvider { spent: number | null; }, ): Promise { + const failure = () => + Object.assign( + new Error( + `${this.isGateway ? "OpenAI-compatible gateway" : "OpenAI"} returned empty content`, + ), + { + httpStatus: 0, + kind: reply.hadReasoning ? "reasoning_only" : "empty_response", + upstream: this.isGateway ? "openai-compatible" : "openai", + ...(reply.spent !== null ? { tokensUsed: reply.spent } : {}), + }, + ); if ( this.isGateway && + // The lesson holds even when the budget leaves no repeat for it. askAgainUnreasoned({ key: this.reasoningDialectKey(), provider: this.type, @@ -535,20 +484,80 @@ export class OpenAIClient implements AIProvider { sent: reply.sent, hadReasoning: reply.hadReasoning, finishReason: reply.finishReason, - }) + }) && + hops.canSend ) { - return withEarlierSpend(this.generateCompletion(params), reply.spent); + return repeatUnreasoned( + this.generateCompletion(params, hops), + reply.spent, + failure, + ); } - const err = new Error( - `${this.isGateway ? "OpenAI-compatible gateway" : "OpenAI"} returned empty content`, - ); - Object.assign(err, { - httpStatus: 0, - kind: reply.hadReasoning ? "reasoning_only" : "empty_response", - upstream: this.isGateway ? "openai-compatible" : "openai", - ...(reply.spent !== null ? { tokensUsed: reply.spent } : {}), + throw failure(); + } + + /** + * A non-2xx reply. As much of the body as can be read goes into a + * structured field (with anything key-shaped stripped), so an upstream + * incident is diagnosable from the log instead of an opaque status. + * + * An endpoint or model that refuses the reasoning field, or (#470, gateway + * only) the JSON flag, is asked once more without it and remembers + * (`wire-refusal.ts`), inside the call's request budget. A refusal the + * budget leaves no repeat for is marked `reasoningRefused`. + * + * A proxy that stopped offering the configured model answers with a bare + * 500 that names nothing; its model listing (free, no completion) says so + * plainly, so the chain, the health card and the log name the fix. + */ + private async failed( + res: Response, + params: CompletionParams, + hops: RequestHops, + sent: { sent: string | null; sendJsonFormat: boolean }, + ): Promise { + const bodyExcerpt = sanitiseBodyExcerpt(await res.text().catch(() => "")); + // The entry's own setting (#1126) included: a refused field is learned + // whichever level put it on the wire. + const effort = sent.sent; + const lesson = learnFromRefusal({ + status: res.status, + body: bodyExcerpt, + model: this.config.model, + reasoning: + effort === null + ? null + : { + sent: effort, + key: this.reasoningDialectKey(), + provider: this.type, + names: /reasoning|thinking/i, + }, + json: + this.isGateway && sent.sendJsonFormat + ? { endpoint: this.config.baseUrl, action: "ai.compat.jsonDialect" } + : null, }); - throw err; + if (lesson && hops.canSend) return this.generateCompletion(params, hops); + const modelNotServed = await this.modelNoLongerServed(res.status); + const label = this.isGateway ? "OpenAI-compatible gateway" : "OpenAI"; + // The excerpt rides a structured field, not the message, so it lands in + // the log fields that are filtered and truncated centrally. + throw Object.assign( + new Error( + modelNotServed + ? `${label} request failed (${res.status}): model "${this.config.model}" is not offered by this endpoint` + : `${label} request failed (${res.status})`, + ), + { + httpStatus: res.status, + upstream: this.isGateway ? "openai-compatible" : "openai", + model: this.config.model, + bodyExcerpt, + ...(modelNotServed ? { modelNotServed: true } : {}), + ...(lesson === "reasoning" ? { reasoningRefused: true } : {}), + }, + ); } // ── v1.41 reasoning ─────────────────────────────────────────────────────── @@ -598,19 +607,24 @@ export class OpenAIClient implements AIProvider { private separateReasoning( choice: OpenAIChoice | undefined, toolCalls: AiToolCall[] | undefined, - thinkingTokens: number, + request: { thinkingTokens: number; sent: string | null }, ): { content: string; reasoning: string | null; hadReasoning: boolean } { const message = (choice?.message ?? {}) as Record; const fielded = [message.reasoning, message.reasoning_content].find( (v): v is string => typeof v === "string" && v.trim().length > 0, ); const raw = choice?.message?.content ?? ""; + const key = this.reasoningDialectKey(); + const asked = request.sent !== null && request.sent !== "none"; const separated = answerOf(raw, { fieldedReasoning: fielded !== undefined, finishReason: choice?.finish_reason, hasToolCalls: toolCalls !== undefined, - reasoningRoom: this.isGateway && thinkingTokens > 0, + reasoningRoom: this.isGateway && request.thinkingTokens > 0, + knownReasoner: knownReasoner(key, asked), }); + const hadReasoning = fielded !== undefined || separated.content !== raw; + if (hadReasoning) noteReasoningSeen(key, asked); if (separated.unseparated) { emitSignal({ action: "ai.gateway.reasoning_unseparated", @@ -621,10 +635,17 @@ export class OpenAIClient implements AIProvider { return { content: separated.content, reasoning: fielded ?? separated.reasoning, - hadReasoning: fielded !== undefined || separated.content !== raw, + hadReasoning, }; } + /** #1279 — this endpoint and model refused the reasoning field. */ + private refusesField(): boolean { + return refusesReasoningField( + learnedReasoningDialect(this.reasoningDialectKey()), + ); + } + private reasoningDialectKey() { return { provider: "openai", @@ -663,7 +684,7 @@ export class OpenAIClient implements AIProvider { : { kind: "reasoning_effort", effort: "none" }; } if (!reasoning) return { kind: "none" }; - if (learned === "none") return { kind: "none" }; + if (refusesReasoningField(learned)) return { kind: "none" }; const level = reasoning.effort; if (this.isGateway) { const effort = level === "off" ? "none" : level; diff --git a/src/lib/ai/provider-health-ledger.ts b/src/lib/ai/provider-health-ledger.ts index 8368aadab..4ddafe397 100644 --- a/src/lib/ai/provider-health-ledger.ts +++ b/src/lib/ai/provider-health-ledger.ts @@ -64,12 +64,14 @@ export const HARD_FAILURE_SKIP_THRESHOLD = 3; * From `PROVIDER_PAUSE_THRESHOLD` consecutive hard failures on, each failure * re-arms the skip window at `PROVIDER_PAUSE_BASE_MS` doubled per further * failure, capped at `PROVIDER_PAUSE_MAX_MS`. Paused is the same skip as - * backoff (the provider moves to the tail of the chain, behind everything - * healthy, and is still walked as a last resort, so a chain never loses its - * only provider). When the window lifts, the next call tries it again: one - * success clears the count and resumes it in place, one more failure doubles - * the pause. Nothing needs to be reset by hand. The admin provider-health card - * shows a paused provider and until when. + * backoff: an interactive call moves the provider to the tail of the chain, + * behind everything healthy, and still walks it as a last resort, so a person + * never loses their only provider; background work leaves it out until the + * window lifts (#1279, `background-bench.ts`). When the window lifts, the next + * call tries it again: one success clears the count and resumes it in place, + * one more failure doubles the pause. Saving a fixed provider forgets its + * record (`forgetProviderHealth`). The admin provider-health card shows a + * paused provider and until when. * * Conservative on purpose: five failures in a row is far past a brown-out, * the first pause is half an hour, and the ceiling is a day, so a provider @@ -283,12 +285,12 @@ export const postgresProviderHealthLedger: ProviderHealthLedger = { * Drop the recorded outcomes of a credential that has just been replaced. * * The ledger describes a credential, not a provider slot: once the person - * (or, for `admin-codex`, the operator) signs in again, the `auth_failed` - * row of the refused credential would otherwise keep the fresh one at the - * tail of the chain and reported as expired for the rest of its cooldown. - * Without `userId` the rows of every account are dropped — the operator's - * central connection is one credential shared by all of them. Best effort, - * like every ledger write. + * (or, for the `admin-*` providers, the operator) signs in again or saves a + * fixed setup, the row of the refused one would otherwise keep the fresh one + * at the tail of every interactive chain, out of background work entirely + * (#1279), and reported as expired for the rest of its window. Without + * `userId` the rows of every account are dropped: an operator provider is one + * credential shared by all of them. Best effort, like every ledger write. */ export async function forgetProviderHealth(target: { providerType: ProviderChainType; @@ -302,7 +304,8 @@ export async function forgetProviderHealth(target: { }, }); } catch { - // A stale row only deprioritises; it never blocks. + // A stale row lapses with its window; until then it keeps background + // work off the replaced credential (#1279). } } diff --git a/src/lib/ai/provider-runner.ts b/src/lib/ai/provider-runner.ts index c4bd364c8..684853b5c 100644 --- a/src/lib/ai/provider-runner.ts +++ b/src/lib/ai/provider-runner.ts @@ -38,50 +38,36 @@ import { const NEVER_SKIP: ReadonlySet = new Set(["local"]); /** - * v1.4.16 — multi-provider redundancy runner. + * The multi-provider redundancy runner: a provider completion with a + * hard-failure walk across an ordered list of providers, so one provider's + * outage (a 401 from an expired Codex sign-in, a 503 brown-out, a network + * reset) cascades to the next configured provider instead of failing the + * request. * - * The chain runner wraps a provider completion with a hard-failure - * walkthrough across an ordered list of providers. The motivation is - * the maintainer's "ein Provider-Ausfall darf die Insights nicht zerschiessen" - * mandate: a 401 from Codex (OAuth expired), a 503 from OpenAI - * (upstream brown-out), or a network reset MUST cascade to the next - * configured provider rather than 422-ing the whole request. + * Hard failures (`isHardProviderFailure`) cascade. A malformed-JSON reply is + * NOT a hard failure — it is a prompt-following issue, so it bubbles to the + * caller's own validator rather than masking itself as provider + * unavailability and walking the chain. Every provider receives the same + * enriched system prompt; `outcome.workingProvider.providerType` attributes + * feedback per provider. * - * Hard failures (`isHardProviderFailure`) cascade. A malformed-JSON - * reply is NOT a hard failure — it is a prompt-following issue, so it - * bubbles to the caller's own validator rather than masking itself as - * provider unavailability and walking the chain. - * - * Cross-feature coupling: - * - B5a's `MEDICAL_REFERENCES` is shared via the system prompt; each - * provider in the chain receives the same enriched context. - * - B5e (planned v1.4.17) reads `outcome.workingProvider.providerType` - * to attribute feedback per provider; that's why we re-export it - * out of the runner instead of the wrapper. - * - * The cache: a provider that worked on the last call gets first crack - * on the next, so a user whose Codex OAuth lapsed doesn't re-burn the - * 401 round-trip on every request for the next hour. Cache is - * in-process (per worker), keyed by `userId`, TTL 1h. No coordination - * across workers — when scaling out we'd move this to Redis, but for - * v1.4.16 the per-worker cache is enough to absorb the typical - * "polling reload" pattern. - * - * v1.18.7 — DECISION: the volatile per-worker hint is left - * in-process and NOT folded into the durable Postgres health ledger. The - * ledger already shares the load-bearing signal across workers (the - * skip-hint reorder in `resolveChainOrder` pushes a dead-credential / - * backoff provider to the chain tail for every worker), so the only thing + * The cache: a provider that worked on the last call gets first crack on the + * next, so a user whose Codex sign-in lapsed doesn't re-burn the 401 + * round-trip on every request for the next hour. In-process (per worker), + * keyed by `userId`, TTL 1h. It stays out of the durable Postgres health + * ledger on purpose: the ledger already shares the load-bearing signal across + * workers (the skip-hint reorder in `resolveChainOrder`), so the only thing * uncoordinated is a soft "tried this one last, put it first" reorder whose - * worst case is a single redundant first-hop per worker per hour — exactly - * the bound the v1.18.7 AI audit deemed acceptable at single-maintainer - * scale. Promoting the hint to a per-request ledger write would add write - * amplification on the hot path for a negligible saving; revisit only if a - * multi-worker deploy shows measurable redundant-hop cost. + * worst case is one redundant first hop per worker per hour. Promoting it to + * a per-request ledger write would add write amplification on the hot path; + * revisit only if a multi-worker deploy shows measurable redundant-hop cost. */ const LAST_WORKING_TTL_MS = 60 * 60 * 1000; // 1h +/** The `failureReason` of a hop the operator cost cap refused unsent. */ +const OPERATOR_CAP_REFUSAL = "operator-cost-cap-exhausted"; + interface CacheEntry { providerType: ProviderChainType; expiresAt: number; @@ -223,6 +209,13 @@ export class AllProvidersFailedError extends Error { * a model that reasons without answering is paid for every time. */ readonly tokensSpent: number; + /** + * #1279 — true when no hop sent a request: every provider was benched for + * background work or refused by the operator cost cap. Such a failure says + * nothing about what was asked, so a caller that backs off the record it + * failed on does not count it. + */ + readonly sentNothing: boolean; constructor(attempts: FallbackHop[]) { super( @@ -236,6 +229,13 @@ export class AllProvidersFailedError extends Error { (sum, a) => sum + (a.tokensSpent ?? 0), 0, ); + this.sentNothing = + attempts.length > 0 && + attempts.every( + (a) => + a.failureKind === "paused" || + a.failureReason === OPERATOR_CAP_REFUSAL, + ); const first = attempts[0]; this.primaryCredentialExpired = first !== undefined && @@ -327,7 +327,7 @@ async function resolveChainOrder( userId: string, providers: ProviderChainResolved[], ledger: ProviderHealthLedger, - surface: BudgetSurface, + background: boolean, ): Promise<{ ordered: ProviderChainResolved[]; benched: Array>; @@ -344,10 +344,9 @@ async function resolveChainOrder( }); } // #1279 — background work leaves a benched provider out entirely. - const { walkable, benched } = - surface === "job" - ? partitionForBackgroundWork(providers, skips, (t) => NEVER_SKIP.has(t)) - : { walkable: providers, benched: [] }; + const { walkable, benched } = background + ? partitionForBackgroundWork(providers, skips, (t) => NEVER_SKIP.has(t)) + : { walkable: providers, benched: [] }; const afterSkips = applyHealthLedgerSkips(walkable, skips); return { ordered: applyLastWorkingCache(userId, afterSkips), benched }; } @@ -511,6 +510,29 @@ function signalFailedHops( } } +/** What both raw runners take. */ +interface ChainWalkArgs { + userId: string; + providers: ProviderChainResolved[]; + params: CompletionParams; + /** + * Which ceiling an operator-funded FALLBACK hop is rationed against. + * `"job"` gets the background share, exactly as the reservation did; + * `"coach"` gets the whole ceiling. + */ + surface: BudgetSurface; + /** + * #1279 — a person is waiting on this call although it is rationed as + * background work (the on-demand briefing). A `"job"` walk otherwise leaves + * every provider the health ledger benched out (`background-bench.ts`); + * this one walks them at the tail of the chain like any interactive call, + * so the person gets the provider's own answer or refusal. + */ + interactive?: boolean; + /** The health ledger; the durable Postgres one unless a test passes its own. */ + ledger?: ProviderHealthLedger; +} + /** * Shared chain walker for the two RAW runners (streaming + buffered). Owns the * one set of chain / fallback / health-ledger semantics — empty-chain guard, @@ -522,13 +544,7 @@ function signalFailedHops( * Schema-class failures still bubble (they are not hard) exactly as before. */ async function runRawChain( - args: { - userId: string; - providers: ProviderChainResolved[]; - params: CompletionParams; - surface: BudgetSurface; - ledger?: ProviderHealthLedger; - }, + args: ChainWalkArgs, // v1.41 — `params` reaches every candidate unchanged, `reasoning` and // `onReasoning` included. A fallback hop may see `providerState` another // provider made on an earlier round; each client re-inserts only its own @@ -549,7 +565,7 @@ async function runRawChain( userId, providers, ledger, - surface, + surface === "job" && !args.interactive, ); const hops: FallbackHop[] = []; const failedHops: HopSignal[] = []; @@ -615,7 +631,7 @@ async function runRawChain( const hop: FallbackHop = { providerType: candidate.providerType, attempt: i + 1, - failureReason: "operator-cost-cap-exhausted", + failureReason: OPERATOR_CAP_REFUSAL, httpStatus: null, }; hops.push(hop); @@ -658,6 +674,7 @@ async function runRawChain( result, workingProvider: candidate, fallbackHops: hops, + tokensSpent: hops.reduce((sum, h) => sum + (h.tokensSpent ?? 0), 0), }; } catch (error) { if (!isHardProviderFailure(error)) { @@ -720,21 +737,12 @@ async function runRawChain( * full reply — to confirm real tokens flowed. The returned shape matches the * buffered runner so the route is path-agnostic. */ -export async function runStreamingRawCompletionWithFallback(args: { - userId: string; - providers: ProviderChainResolved[]; - params: CompletionParams; - /** - * v1.38.19 — which ceiling an operator-funded FALLBACK - * hop is rationed against. `"job"` gets the background share, exactly as the - * reservation did; `"coach"` gets the whole ceiling. - */ - surface: BudgetSurface; - /** Called once per streamed token/chunk for providers that stream. */ - onDelta: (delta: string) => void; - /** See `RunWithFallbackParams.ledger`. */ - ledger?: ProviderHealthLedger; -}): Promise { +export async function runStreamingRawCompletionWithFallback( + args: ChainWalkArgs & { + /** Called once per streamed token/chunk for providers that stream. */ + onDelta: (delta: string) => void; + }, +): Promise { return runRawChain( args, // Prefer the provider's streaming wire; fall back to the buffered call for @@ -759,6 +767,12 @@ export interface RunRawWithFallbackResult { result: CompletionResult; workingProvider: ProviderChainResolved; fallbackHops: FallbackHop[]; + /** + * #1279 — tokens the hops that failed before `workingProvider` answered + * reported spending, summed. Not in `result.tokensUsed`, which is the + * answering call's alone; a caller that reconciles budget charges both. + */ + tokensSpent: number; } /** @@ -772,15 +786,9 @@ export interface RunRawWithFallbackResult { * last-working cache + structured logging are shared with the streaming * sibling via `runRawChain`. */ -export async function runRawCompletionWithFallback(args: { - userId: string; - providers: ProviderChainResolved[]; - params: CompletionParams; - /** See {@link runStreamingRawCompletionWithFallback}'s `surface`. */ - surface: BudgetSurface; - /** See `RunWithFallbackParams.ledger`. */ - ledger?: ProviderHealthLedger; -}): Promise { +export async function runRawCompletionWithFallback( + args: ChainWalkArgs, +): Promise { return runRawChain(args, (candidate) => candidate.instance.generateCompletion(args.params), ); diff --git a/src/lib/ai/reasoning/__tests__/support.test.ts b/src/lib/ai/reasoning/__tests__/support.test.ts index 34cfa9ef7..73999c462 100644 --- a/src/lib/ai/reasoning/__tests__/support.test.ts +++ b/src/lib/ai/reasoning/__tests__/support.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + answerOf, anthropicModelVersion, anthropicRefusesSampling, anthropicThinkingDialect, @@ -195,6 +196,56 @@ describe("parsing helpers", () => { }); }); + it("leaves a closing tag mentioned in a sentence alone, unless the model is known to reason", () => { + const prose = + "Some models end their thinking with before they answer."; + expect(splitThinkTags(prose)).toEqual({ content: prose, reasoning: null }); + // The same text from a model seen reasoning: the template opened the + // block, so the tag closes it wherever it sits. + expect(splitThinkTags("Weighing the weekSteady.", true)).toEqual({ + content: "Steady.", + reasoning: "Weighing the week", + }); + // A tag that opens a line splits whoever sent it. + expect(splitThinkTags("Weighing the week\n \nSteady.")).toEqual({ + content: "Steady.", + reasoning: "Weighing the week", + }); + }); + + describe("answerOf, a reply cut off at the ceiling", () => { + const cut = '{"summary": "Your weight has been stable and'; + const reply = (knownReasoner: boolean) => ({ + fieldedReasoning: false, + finishReason: "length", + hasToolCalls: false, + reasoningRoom: true, + knownReasoner, + }); + + it("is the answer from a model not known to reason", () => { + expect(answerOf(cut, reply(false))).toEqual({ + content: cut, + reasoning: null, + unseparated: false, + }); + }); + + it("is reasoning the server did not separate from a model known to reason", () => { + expect(answerOf(cut, reply(true))).toEqual({ + content: "", + reasoning: cut, + unseparated: true, + }); + }); + + it("is the answer when the ceiling held no room for reasoning", () => { + expect( + answerOf(cut, { ...reply(true), reasoningRoom: false }).content, + ).toBe(cut); + }); + }); + it("maps wire efforts back onto levels", () => { expect(levelOfWireEffort("none")).toBe("off"); expect(levelOfWireEffort("minimal")).toBe("off"); diff --git a/src/lib/ai/reasoning/answerless.ts b/src/lib/ai/reasoning/answerless.ts index 0c16a9081..bcf2a2e18 100644 --- a/src/lib/ai/reasoning/answerless.ts +++ b/src/lib/ai/reasoning/answerless.ts @@ -11,8 +11,14 @@ * and a GLM chat template reads an effort it does not know as its maximum * (vllm-project/vllm#54744). That is learned too (`REASONS_REGARDLESS`): later * calls go out as the call asks, with the thinking room #1276 keeps, and an - * answerless reply is final. So one call costs two requests at most, and - * after the first such call, one. + * answerless reply is final. After the first such call, one request. + * + * An endpoint that refuses the reasoning field altogether can be asked + * neither way: it learns `"none"` (no field), and once its model is seen + * reasoning unasked, `FIELD_REFUSED_REASONS_REGARDLESS`. Its `none` repeat is + * the refused field itself, so a refusal of that repeat is learned as both. + * Every request of a call, these repeats included, counts against its budget + * of two (`request-hops.ts`). * * What bounds the calls themselves is not here: the chain keeps background * work off a provider that keeps failing (`provider-runner.ts`), and each job @@ -21,9 +27,12 @@ import { emitSignal } from "@/lib/logging/signal"; import { ANSWERS_ONLY_UNREASONED, + FIELD_REFUSED_REASONS_REGARDLESS, REASONS_REGARDLESS, annotateReasoningDowngrade, learnedReasoningDialect, + reasoningSeen, + refusesReasoningField, rememberReasoningDialect, type ReasoningDialectKey, } from "./dialect-cache"; @@ -47,9 +56,19 @@ export function nextAnswerlessStep( sent: string | null, hadReasoning: boolean, ): AnswerlessStep { - if (learnedReasoningDialect(key) === REASONS_REGARDLESS) { + const learned = learnedReasoningDialect(key); + if ( + learned === REASONS_REGARDLESS || + learned === FIELD_REFUSED_REASONS_REGARDLESS + ) { return { kind: "final", learnedRegardless: false }; } + // The endpoint refuses the field: there is no `none` to ask with. + if (learned === "none") { + if (!hadReasoning) return { kind: "final", learnedRegardless: false }; + rememberReasoningDialect(key, FIELD_REFUSED_REASONS_REGARDLESS); + return { kind: "final", learnedRegardless: true }; + } if (sent === "none") { if (!hadReasoning) return { kind: "final", learnedRegardless: false }; rememberReasoningDialect(key, REASONS_REGARDLESS); @@ -111,7 +130,51 @@ export function askAgainUnreasoned(args: { */ export function reasonsUnasked(key: ReasoningDialectKey): boolean { const learned = learnedReasoningDialect(key); - return learned === ANSWERS_ONLY_UNREASONED || learned === REASONS_REGARDLESS; + return ( + learned === ANSWERS_ONLY_UNREASONED || + learned === REASONS_REGARDLESS || + learned === FIELD_REFUSED_REASONS_REGARDLESS + ); +} + +/** + * #1276 — true when the model has shown it reasons, for a request that did + * (`asked`) or did not ask it to: a lesson that says it reasons unasked, or a + * reply seen reasoning (`reasoningSeen`). Only then is unlabelled text that + * ran into the ceiling read as reasoning (`answerOf`). + */ +export function knownReasoner( + key: ReasoningDialectKey, + asked: boolean, +): boolean { + return reasonsUnasked(key) || reasoningSeen(key, asked); +} + +/** + * The endpoint refused the reasoning field `sent`: send none from now on. A + * model already known to reason unasked keeps that lesson too, so it is never + * asked with `none` again. + */ +export function learnReasoningFieldRefused( + key: ReasoningDialectKey, + provider: string, + sent: string, +): void { + const before = learnedReasoningDialect(key); + rememberReasoningDialect( + key, + reasonsUnasked(key) ? FIELD_REFUSED_REASONS_REGARDLESS : "none", + ); + annotateReasoningDowngrade(provider, sent, "none"); + // One line per lesson, not per call: the operator learns that the + // endpoint takes no reasoning field (an entry setting it refuses included). + if (!refusesReasoningField(before)) { + emitSignal({ + action: "ai.provider.reasoning_field_refused", + level: "warn", + meta: { provider, effort: sent }, + }); + } } /** @@ -136,6 +199,27 @@ export async function withEarlierSpend( } } +/** + * The `none` repeat after an answerless reply, carrying that reply's spend. + * When the endpoint refused the `none` itself on the call's last request + * (`reasoningRefused` on the error), the call ends with the first reply's + * failure (`answerless`): the model reasoned without answering, and that is + * what the call ran into, not the field the client then tried. + */ +export async function repeatUnreasoned( + repeat: Promise, + earlier: number | null, + answerless: () => Error, +): Promise { + try { + return await withEarlierSpend(repeat, earlier); + } catch (err) { + const refused = (err as { reasoningRefused?: unknown } | null) + ?.reasoningRefused; + throw refused === true ? answerless() : err; + } +} + /** * Tokens a failed call spent upstream, as its error carries them * (`tokensUsed`), or `null` when it carries none. diff --git a/src/lib/ai/reasoning/dialect-cache.ts b/src/lib/ai/reasoning/dialect-cache.ts index 60151707a..b1d7f841c 100644 --- a/src/lib/ai/reasoning/dialect-cache.ts +++ b/src/lib/ai/reasoning/dialect-cache.ts @@ -58,6 +58,21 @@ export const ANSWERS_ONLY_UNREASONED = "off"; */ export const REASONS_REGARDLESS = "unswitchable"; +/** + * #1279 — the learned value for an endpoint that refuses the reasoning field + * outright (LiteLLM in front of Ollama: "does not support parameters: + * ['reasoning_effort']") in front of a model that reasons unasked. Both facts + * at once: no field is sent, as with `"none"`, and the thinking room stays + * and an answerless reply is final, as with `REASONS_REGARDLESS`. Held as one + * value each, the two lessons overwrote each other on every request. + */ +export const FIELD_REFUSED_REASONS_REGARDLESS = "none-unswitchable"; + +/** True when `learned` says the endpoint refuses the reasoning field. */ +export function refusesReasoningField(learned: string | undefined): boolean { + return learned === "none" || learned === FIELD_REFUSED_REASONS_REGARDLESS; +} + /** How long a learned downgrade holds before the full request is tried again. */ export const DIALECT_TTL_MS = 60 * 60 * 1000; @@ -135,9 +150,58 @@ export function isReasoningParameterRejection( return params.test(text) && PARAMETER_REFUSAL.test(text); } +/** + * #1276 — endpoints and models seen reasoning: a reasoning field or a think + * tag in a reply. `unasked` when the request carried no effort (or `none`), + * and it outlasts a later sighting on a request that asked. Held as long as a + * lesson (`DIALECT_TTL_MS`) and on `globalThis` for the same reason. + */ +const SEEN_SLOT = Symbol.for("healthlog.ai.reasoningSeen"); +const seen: Map = (( + globalThis as Record +)[SEEN_SLOT] ??= new Map()) as Map< + string, + { unasked: boolean; expiresAt: number } +>; + +function seenEntry(k: string) { + const entry = seen.get(k); + if (entry && Date.now() >= entry.expiresAt) { + seen.delete(k); + return undefined; + } + return entry; +} + +/** Record a reply of `key` that reasoned; `asked`: its request asked to. */ +export function noteReasoningSeen( + key: ReasoningDialectKey, + asked: boolean, +): void { + const k = cacheKey(key); + seen.set(k, { + unasked: !asked || (seenEntry(k)?.unasked ?? false), + expiresAt: Date.now() + DIALECT_TTL_MS, + }); +} + +/** + * True when `key` was seen reasoning in a way that speaks for a request that + * did (`asked`) or did not ask to reason: a model that reasoned unasked + * speaks for both, one seen reasoning only when asked for the first. + */ +export function reasoningSeen( + key: ReasoningDialectKey, + asked: boolean, +): boolean { + const entry = seenEntry(cacheKey(key)); + return entry !== undefined && (entry.unasked || asked); +} + /** Test hook: forget everything learned. */ export function resetReasoningDialectCache(): void { learned.clear(); + seen.clear(); } /** diff --git a/src/lib/ai/reasoning/support.ts b/src/lib/ai/reasoning/support.ts index 38f41989c..b5555ab00 100644 --- a/src/lib/ai/reasoning/support.ts +++ b/src/lib/ai/reasoning/support.ts @@ -275,16 +275,25 @@ export function reasoningTitleOf(text: string): string | null { * * #1276 — a reply can also carry only the closing tag: GLM, Qwen3 and * DeepSeek-R1 chat templates end the generation prompt with ``, so the - * model's output starts inside the block. Everything before the first - * `` of such a reply is reasoning. + * model's output starts inside the block. Everything before that `` + * is reasoning when the tag opens a line (the templates close the block on a + * line of its own, or at the very start), or anywhere once the model is known + * to reason (`knownReasoner`). An ordinary answer that mentions the tag in a + * sentence is left whole. */ -export function splitThinkTags(content: string): { +export function splitThinkTags( + content: string, + knownReasoner = false, +): { content: string; reasoning: string | null; } { const m = /^\s*([\s\S]*?)(<\/think>|$)/.exec(content) ?? - /^([\s\S]*?)(<\/think>)/.exec(content); + (knownReasoner + ? /^([\s\S]*?)(<\/think>)/ + : /^((?:[\s\S]*?\n)?)[ \t]*(<\/think>)/ + ).exec(content); if (!m) return { content, reasoning: null }; const reasoning = m[1].trim(); const rest = m[2] ? content.slice(m[0].length).replace(/^\s+/, "") : ""; @@ -299,10 +308,12 @@ export function splitThinkTags(content: string): { * off by {@link splitThinkTags}. * - A reply that ran into a ceiling which held room for reasoning * (`finish_reason: "length"`, `reasoningRoom`), with no reasoning field - * beside it, no tag and no tool call, is reasoning the server did not - * separate: the budget was sized for thinking plus an answer, so text - * that fills all of it unlabelled is the thought, cut off. It is never - * the answer. + * beside it, no tag and no tool call, from a model known to reason + * (`knownReasoner`), is reasoning the server did not separate: the budget + * was sized for thinking plus an answer, so text that fills all of it + * unlabelled is the thought, cut off. From any other model it is the + * answer, cut off, and the caller reads `finish_reason` (a model that + * does not reason, asked with any level, still answers). * * `reasoning` is the text taken out of `content`, for the reasoning channel. * `unseparated` marks the second case so the caller can say so in its log. @@ -316,11 +327,13 @@ export function answerOf( hasToolCalls: boolean; /** The request's ceiling held room for reasoning on top of the answer. */ reasoningRoom: boolean; + /** This endpoint and model have shown they reason (`knownReasoner`). */ + knownReasoner: boolean; }, ): { content: string; reasoning: string | null; unseparated: boolean } { const text = content ?? ""; if (!text) return { content: "", reasoning: null, unseparated: false }; - const split = splitThinkTags(text); + const split = splitThinkTags(text, reply.knownReasoner); if (split.content !== text || split.reasoning !== null) { return { content: split.content, @@ -331,6 +344,7 @@ export function answerOf( if ( reply.finishReason === "length" && reply.reasoningRoom && + reply.knownReasoner && !reply.fieldedReasoning && !reply.hasToolCalls ) { diff --git a/src/lib/ai/request-hops.ts b/src/lib/ai/request-hops.ts new file mode 100644 index 000000000..4932be6c7 --- /dev/null +++ b/src/lib/ai/request-hops.ts @@ -0,0 +1,49 @@ +/** + * #1279 — the HTTP requests one completion call may send. + * + * The Local and the OpenAI-compatible gateway clients send a request again on + * their own: without a reasoning field the endpoint refused, without the JSON + * flag it refused, with `reasoning_effort: "none"` after a reply that + * reasoned without answering, and the Local stream falls back to a buffered + * request. Each repeat was bounded by what it had just learned, and two + * lessons that undo each other were not bounded at all: a gateway that + * refuses `reasoning_effort` in front of a model that reasons unasked made + * one call send requests until something outside stopped it. + * + * So each call carries one budget through every repeat, and the budget bounds + * it, not the lessons: at most `MAX_REQUESTS_PER_CALL` requests, after which + * the call ends with the failure it has. What was learned on the way still + * holds for the next call. `ai-request-hops-guard.test.ts` holds both clients + * to sending only through here and to handing the budget to every repeat. + * + * Server-only. + */ +import { safeFetch } from "@/lib/safe-fetch"; + +/** The requests one call may send: the first, and one repeat. */ +export const MAX_REQUESTS_PER_CALL = 2; + +export class RequestHops { + private sent = 0; + + /** True while the call may send one more request. */ + get canSend(): boolean { + return this.sent < MAX_REQUESTS_PER_CALL; + } + + /** + * `safeFetch`, counted. A request past the budget is not sent: every repeat + * asks `canSend` first, so this throw marks a repeat that did not. + */ + fetch(...args: Parameters): Promise { + if (!this.canSend) { + return Promise.reject( + Object.assign(new Error("AI call has no request left to send"), { + httpStatus: 0, + }), + ); + } + this.sent += 1; + return safeFetch(...args); + } +} diff --git a/src/lib/ai/wire-refusal.ts b/src/lib/ai/wire-refusal.ts new file mode 100644 index 000000000..64136db6b --- /dev/null +++ b/src/lib/ai/wire-refusal.ts @@ -0,0 +1,69 @@ +/** + * What a refused request teaches the Local and the OpenAI-compatible gateway + * clients, in one place for both. + * + * Only an HTTP 400 that names a field the request carried teaches anything: + * the reasoning field (`isReasoningParameterRejection`, learned per endpoint + * and model) or the JSON flag (`isResponseFormatRejection`, learned per + * endpoint). The reasoning field is checked first, so a body that names it + * never switches JSON mode off. Whether the call then sends the lesser request + * is the client's, inside its request budget (`request-hops.ts`). + * + * Server-only. + */ +import { annotate } from "@/lib/logging/context"; + +import { + isResponseFormatRejection, + rememberJsonModeDialect, +} from "./json-dialect"; +import { learnReasoningFieldRefused } from "./reasoning/answerless"; +import { + isReasoningParameterRejection, + type ReasoningDialectKey, +} from "./reasoning/dialect-cache"; + +export type RefusalLesson = "reasoning" | "json" | null; + +export function learnFromRefusal(args: { + status: number; + body: string; + model: string; + /** The reasoning field the request carried, or `null` for none. */ + reasoning: { + sent: string; + key: ReasoningDialectKey; + provider: string; + /** The parameter names this endpoint's refusals use. */ + names: RegExp; + } | null; + /** The endpoint, when the request carried the JSON flag. */ + json: { endpoint: string; action: string } | null; +}): RefusalLesson { + const { reasoning, json } = args; + if ( + reasoning && + isReasoningParameterRejection( + args.status, + args.body, + reasoning.names, + args.model, + ) + ) { + learnReasoningFieldRefused( + reasoning.key, + reasoning.provider, + reasoning.sent, + ); + return "reasoning"; + } + if (json && isResponseFormatRejection(args.status, args.body)) { + rememberJsonModeDialect(json.endpoint, "none"); + annotate({ + action: { name: json.action }, + meta: { dialect: "none", httpStatus: args.status }, + }); + return "json"; + } + return null; +} diff --git a/src/lib/analytics/__tests__/mood-series.test.ts b/src/lib/analytics/__tests__/mood-series.test.ts index e29667876..a3ba1fcd2 100644 --- a/src/lib/analytics/__tests__/mood-series.test.ts +++ b/src/lib/analytics/__tests__/mood-series.test.ts @@ -31,7 +31,14 @@ vi.mock("@/lib/rollups/mood-rollups", async () => { import { buildMoodDailySeries } from "../mood-series"; import { prisma } from "@/lib/db"; -import { readMoodDayRollups } from "@/lib/rollups/mood-rollups"; +import { + ensureUserMoodRollupsFresh, + readMoodDayRollups, +} from "@/lib/rollups/mood-rollups"; +import { + pendingBackgroundTaskCount, + settleBackgroundTasks, +} from "@/lib/logging/background-tasks"; beforeEach(() => { vi.mocked(prisma.moodEntry.findMany).mockResolvedValue([] as never); @@ -104,4 +111,26 @@ describe("buildMoodDailySeries", () => { expect(series.source).toBe("rollup"); expect(series.summary.count).toBe(0); }); + + it("files the rollup warm-up with the background registry", async () => { + // The warm-up is not awaited by the read, so a caller that has to know + // when the request's work is over (the statement counter, the suite's + // settle hook) can only see it through the registry. Left unregistered, + // a stale account's full recompute bled into whatever ran next. + let release: () => void = () => {}; + vi.mocked(ensureUserMoodRollupsFresh).mockImplementationOnce( + () => + new Promise((resolve) => { + release = () => resolve({ recomputed: true }); + }), + ); + + await buildMoodDailySeries("user-1"); + + expect(ensureUserMoodRollupsFresh).toHaveBeenCalledWith("user-1"); + expect(pendingBackgroundTaskCount()).toBe(1); + release(); + await settleBackgroundTasks(); + expect(pendingBackgroundTaskCount()).toBe(0); + }); }); diff --git a/src/lib/analytics/mood-series.ts b/src/lib/analytics/mood-series.ts index 8e57d64e3..e921062c5 100644 --- a/src/lib/analytics/mood-series.ts +++ b/src/lib/analytics/mood-series.ts @@ -12,6 +12,8 @@ */ import type { PrismaClient } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; +import { runDetached } from "@/lib/logging/detached"; +import { trackBackgroundTask } from "@/lib/logging/background-tasks"; import { summarize, type DataPoint } from "@/lib/analytics/trends"; import { ensureUserMoodRollupsFresh, @@ -94,7 +96,9 @@ export async function buildMoodDailySeries( ): Promise { // Fire-and-forget warm-up so the next cold mount for this user lands on // the rollup tier even when the boot-time backfill hasn't reached them. - void ensureUserMoodRollupsFresh(userId); + // Registered so a caller that must know when the request's work is over + // (the suite's settle hook, the statement counter) can await it. + trackBackgroundTask(runDetached(() => ensureUserMoodRollupsFresh(userId))); const since = new Date(Date.now() - MOOD_SERIES_WINDOW_MS); const rollups = await readMoodDayRollups(userId, since); diff --git a/src/lib/analytics/response/default-body.ts b/src/lib/analytics/response/default-body.ts index 4ba36afe1..e8c5bb87d 100644 --- a/src/lib/analytics/response/default-body.ts +++ b/src/lib/analytics/response/default-body.ts @@ -5,6 +5,8 @@ * the sleep stages. The stages run in the order the route always ran them. */ import type { SessionUser } from "@/lib/auth/session-user"; +import { runDetached } from "@/lib/logging/detached"; +import { trackBackgroundTask } from "@/lib/logging/background-tasks"; import { annotate } from "@/lib/logging/context"; import type { Locale } from "@/lib/i18n/config"; import { DEFAULT_TIMEZONE } from "@/lib/tz/resolver"; @@ -42,7 +44,7 @@ export async function buildAnalyticsResponse( // measurement lands, and the next request returns the up-to-date // value once the background refresh completes. `ensureUserRollupsFresh` // wraps its own try/catch, so the void call cannot reject. - void ensureUserRollupsFresh(user.id); + trackBackgroundTask(runDetached(() => ensureUserRollupsFresh(user.id))); // Single per-type coverage probe shared by the // bp_in_target / healthScore / correlations branches below. The diff --git a/src/lib/analytics/summaries-slice.ts b/src/lib/analytics/summaries-slice.ts index 031022a05..3d506ac52 100644 --- a/src/lib/analytics/summaries-slice.ts +++ b/src/lib/analytics/summaries-slice.ts @@ -59,6 +59,8 @@ import pLimit from "p-limit"; import type { MeasurementType } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; +import { runDetached } from "@/lib/logging/detached"; +import { trackBackgroundTask } from "@/lib/logging/background-tasks"; import { HOURLY_MEAN_DAY_TYPES, usesHourlyMeanDay, @@ -277,14 +279,12 @@ export async function computeSummariesSlice( // One clock for every window of the slice, JS and SQL alike (see // `request-clock-sql.ts`). const now = new Date(); - // v1.4.37.1 — fire-and-forget. See `src/app/api/analytics/route.ts` - // for the full rationale: awaiting this on the read path can stall - // the event loop for tens of seconds on tenants with large iOS step - // sample sets that keep the 90-day window slightly stale. The downstream - // coverage probe falls back to live SQL for uncovered types, so - // correctness is preserved; the read just doesn't block waiting - // for the background refresh. - void ensureUserRollupsFresh(userId); + // v1.4.37.1 — fire-and-forget (see `src/app/api/analytics/route.ts`): + // awaiting it can stall the event loop for tens of seconds on accounts + // whose iOS steps keep the 90-day window stale, and the coverage probe + // falls back to live SQL meanwhile. Registered so the suite's settle hook + // and the statement counter can await it. + trackBackgroundTask(runDetached(() => ensureUserRollupsFresh(userId))); // v1.4.36 — per-type coverage probe replaces the legacy global // COUNT. The previous gate returned true as soon as ANY type had at diff --git a/src/lib/auth/guards/record.ts b/src/lib/auth/guards/record.ts index 076450841..44010e554 100644 --- a/src/lib/auth/guards/record.ts +++ b/src/lib/auth/guards/record.ts @@ -1,5 +1,7 @@ import type { AccountGrant } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; +import { runDetached } from "@/lib/logging/detached"; +import { trackBackgroundTask } from "@/lib/logging/background-tasks"; import { SESSION_USER_SELECT, type SessionUser } from "@/lib/auth/session-user"; import { annotate, getEvent } from "@/lib/logging/context"; import { recordDelegatedAccess } from "@/lib/auth/audit"; @@ -372,13 +374,15 @@ async function resolveSwitchedRecord( }); // Fire-and-forget, the `ApiToken.lastUsedAt` posture: the read must not wait // on the bookkeeping, and the bookkeeping failing must not fail the read. - void touchGrantUsage(grant.id); + trackBackgroundTask(runDetached(() => touchGrantUsage(grant.id))); // The owner's copy of the same fact, and the one they can read: a day- // coalesced row on their own record saying who opened it. It lands here // rather than in the handlers because a route cannot forget what it never // had to remember — every delegated request in the product passes through // this line, and only requests that got past the grant check reach it. - void recordDelegatedAccess(owner.id, auth.user.id); + trackBackgroundTask( + runDetached(() => recordDelegatedAccess(owner.id, auth.user.id)), + ); return { session: auth.session, diff --git a/src/lib/dashboard/snapshot-read.ts b/src/lib/dashboard/snapshot-read.ts index cd98809e0..4038d91cd 100644 --- a/src/lib/dashboard/snapshot-read.ts +++ b/src/lib/dashboard/snapshot-read.ts @@ -24,6 +24,8 @@ import type { SessionUser } from "@/lib/auth/session-user"; import { prisma } from "@/lib/db"; +import { runDetached } from "@/lib/logging/detached"; +import { trackBackgroundTask } from "@/lib/logging/background-tasks"; import { annotate } from "@/lib/logging/context"; import { cachedSwr, caches, type ServerCache } from "@/lib/cache/server-cache"; import { dashboardSnapshotCacheKey } from "@/lib/cache/invalidate"; @@ -201,14 +203,18 @@ export async function readDashboardSnapshotCached( // its claim, never thrown. if (briefingAi.available) { const lastSeen = cached.tiles?.lastSeenByType ?? {}; - void requestTodayBriefingWarm({ - userId: user.id, - locale, - timezone: user.timezone, - generatedAt, - lastSeenAt: (type) => lastSeen[type]?.lastSeenAt ?? null, - now, - }).catch(() => undefined); + trackBackgroundTask( + runDetached(() => + requestTodayBriefingWarm({ + userId: user.id, + locale, + timezone: user.timezone, + generatedAt, + lastSeenAt: (type) => lastSeen[type]?.lastSeenAt ?? null, + now, + }), + ).catch(() => undefined), + ); } return { body, locale }; diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 316c278d5..92cead915 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -516,8 +516,6 @@ export const USER_KEPT_FIELDS: Readonly> = { oidcIssuer: "a sign-in credential — the external identity binding", oidcSub: "a sign-in credential — the external identity binding", createdAt: "when the account was created; the account is preserved", - previewFeaturesJson: - "which preview features the person opted into — how the interface is shown, not record content", updatedAt: "row bookkeeping maintained by Prisma", managedProfileAt: "whether this record is one somebody else runs — account shape, not record content. Clearing it would strip the guardian's reach from a record that has no credentials of its own, leaving an account nobody can open, administer or delete. Emptying a record is not emancipating the person in it.", diff --git a/src/lib/documents/__tests__/fenced-chat-pipeline.test.ts b/src/lib/documents/__tests__/fenced-chat-pipeline.test.ts index facf860c6..4eb1f4845 100644 --- a/src/lib/documents/__tests__/fenced-chat-pipeline.test.ts +++ b/src/lib/documents/__tests__/fenced-chat-pipeline.test.ts @@ -106,6 +106,7 @@ beforeEach(() => { result: { content: "ok", tokensUsed: 10, model: "m" }, workingProvider: { providerType: "anthropic" }, fallbackHops: [], + tokensSpent: 0, } as never); }); @@ -194,6 +195,7 @@ describe("streamFencedReply — numeric grounding over the UNION", () => { }, workingProvider: { providerType: "anthropic" }, fallbackHops: [], + tokensSpent: 0, } as never); const res = await streamFencedReply({ ...baseArgs, diff --git a/src/lib/documents/fenced-chat.ts b/src/lib/documents/fenced-chat.ts index 90a834650..2321af54f 100644 --- a/src/lib/documents/fenced-chat.ts +++ b/src/lib/documents/fenced-chat.ts @@ -42,6 +42,7 @@ import { AllProvidersFailedError, runStreamingRawCompletionWithFallback, } from "@/lib/ai/provider-runner"; +import { spentOnFailure } from "@/lib/ai/reasoning/answerless"; import { singleUserTurn } from "@/lib/ai/types"; import { buildDateKey, @@ -285,9 +286,8 @@ export async function streamFencedReply( args; // ── Provider resolution: document order (local-first, codex last), SINGLE - // pick, NO cascade — so the exact egress equals the checked target. The pick - // re-checks `documentAi` for that provider, consent receipt included; a - // refusal is thrown (403) before any egress. ── + // pick, NO cascade, so the egress is the checked target. The pick re-checks + // `documentAi` for it, receipt included, and throws 403 before egress. ── const { pick, withheld } = await resolveDocumentTextProvider(userId); if (withheld) throw withheld; if (!pick) { @@ -364,8 +364,7 @@ Reply now as the assistant, grounded ONLY in the documents above, in ${ contractLocale === "de" ? "German" : "English" }.`; - // The authoritative grounding set: the LIVE attachments' texts only. Never - // history, never a snapshot (none exists) — recomputed from `docs` this turn. + // The grounding set: the LIVE attachments' texts only, recomputed this turn. const groundingSources = docs.map((d) => d.text); type Outcome = @@ -397,7 +396,9 @@ Reply now as the assistant, grounded ONLY in the documents above, in ${ }); result = fallback.result; } catch (err) { - await reconcileSpend(userId, reservation.reserved, 0, dateKey, 0, { + // #1279 — what the failed call reported spending stays charged. + const spent = spentOnFailure(err); + await reconcileSpend(userId, reservation.reserved, spent, dateKey, 0, { servedBy: null, reservedOwner: reservation.owner, }).catch(() => {}); diff --git a/src/lib/export/__tests__/account-settings-backup.test.ts b/src/lib/export/__tests__/account-settings-backup.test.ts index 4784869d2..27bf42b17 100644 --- a/src/lib/export/__tests__/account-settings-backup.test.ts +++ b/src/lib/export/__tests__/account-settings-backup.test.ts @@ -111,6 +111,21 @@ describe("admitAccountSettings", () => { expect(admitAccountSettings({}, ctx)).toEqual({ data: {}, refused: [] }); }); + it("accepts and ignores the retired preview-feature opt-ins of an older file", async () => { + // Written by v1.44 and v1.45, whose column v1.46 dropped. The file must + // still restore, so the field is neither refused nor written. + const { accountSettingsBackupSchema } = + await import("@/lib/export/account-settings-backup-schema"); + const entry = accountSettingsBackupSchema.parse({ + timezone: "Europe/Berlin", + previewFeaturesJson: { topics: true }, + }); + expect(admitAccountSettings(entry, ctx)).toEqual({ + data: { timezone: "Europe/Berlin" }, + refused: [], + }); + }); + it("refuses a language, unit or time zone this release does not know", () => { const { data, refused } = admitAccountSettings( { diff --git a/src/lib/export/account-settings-backup-schema.ts b/src/lib/export/account-settings-backup-schema.ts index ea418f938..c8d2269a0 100644 --- a/src/lib/export/account-settings-backup-schema.ts +++ b/src/lib/export/account-settings-backup-schema.ts @@ -66,6 +66,9 @@ export const accountSettingsBackupSchema = z moodTagLayoutJson: z.unknown().optional(), documentsLayoutJson: z.unknown().optional(), moduleListLayoutJson: z.unknown().optional(), + // Retired in v1.46.0 with its column (the preview-feature opt-ins). A file + // written by v1.44 or v1.45 still carries it; it is accepted and ignored, + // because no codec reads it, so such a file restores everything else. previewFeaturesJson: z.unknown().optional(), activityGoalsJson: z.unknown().optional(), reportSelectionJson: z.unknown().optional(), diff --git a/src/lib/export/account-settings-backup.ts b/src/lib/export/account-settings-backup.ts index 235827f93..3a2490a99 100644 --- a/src/lib/export/account-settings-backup.ts +++ b/src/lib/export/account-settings-backup.ts @@ -539,7 +539,6 @@ const ACCOUNT_SETTING_CODECS = { moodTagLayoutJson: json("moodTagLayoutJson"), documentsLayoutJson: json("documentsLayoutJson"), moduleListLayoutJson: json("moduleListLayoutJson"), - previewFeaturesJson: json("previewFeaturesJson"), activityGoalsJson: activityGoals, reportSelectionJson: json("reportSelectionJson"), globalExcludedInjectionSites: excludedSites, diff --git a/src/lib/http/retired-routes.ts b/src/lib/http/retired-routes.ts index 8ed40ff6d..22580e55c 100644 --- a/src/lib/http/retired-routes.ts +++ b/src/lib/http/retired-routes.ts @@ -160,6 +160,14 @@ export const RETIRED_ROUTES: readonly RetiredRoute[] = [ "Research Mode was an opt-in for the estimated drug-level curve. The chart had stopped consulting the flag several releases earlier and painted for every account regardless, so the switch governed nothing. The curve is simply part of the medication page now and needs no preference read.", methods: ["GET", "POST", "DELETE"], }, + { + path: "/api/settings/preview-features", + removedIn: "1.46.0", + replacedBy: null, + reason: + "Stored per-account opt-ins to preview features, of which there was one, the topic overview pages. Those pages were dropped before they shipped, so the switch governed nothing; the route and its column are gone, and no preview is gated per account.", + methods: ["GET", "PATCH"], + }, ]; const BY_PATH: ReadonlyMap = new Map( diff --git a/src/lib/insights/__tests__/briefing-provider-budget.test.ts b/src/lib/insights/__tests__/briefing-provider-budget.test.ts index 508fba298..cf19b4441 100644 --- a/src/lib/insights/__tests__/briefing-provider-budget.test.ts +++ b/src/lib/insights/__tests__/briefing-provider-budget.test.ts @@ -110,6 +110,7 @@ function mockProviderReply( }, workingProvider: { providerType: "admin-openai" }, fallbackHops: [], + tokensSpent: 0, }); } @@ -133,6 +134,43 @@ describe("runBriefingCompletion — ledger accounting", () => { expect(ledgerTotal).toBe(1234); }); + it("charges what the hops that failed before the answer spent (#1279)", async () => { + runRawCompletionWithFallback.mockResolvedValue({ + result: { content: "{}", model: "m", tokensUsed: 1234 }, + workingProvider: { providerType: "admin-openai" }, + fallbackHops: [], + tokensSpent: 2700, + }); + + await runBriefingCompletion(completionArgs()); + + expect(ledgerTotal).toBe(1234 + 2700); + }); + + it("reserves and sends nothing once the caller's bound fired (#1279)", async () => { + mockProviderReply(100); + const controller = new AbortController(); + controller.abort(); + + await expect( + runBriefingCompletion(completionArgs({ signal: controller.signal })), + ).rejects.toThrow(); + + expect(runRawCompletionWithFallback).not.toHaveBeenCalled(); + expect(ledgerTotal).toBe(0); + }); + + it("hands the bound's signal to the provider call", async () => { + mockProviderReply(100); + const controller = new AbortController(); + + await runBriefingCompletion(completionArgs({ signal: controller.signal })); + + expect(runRawCompletionWithFallback.mock.calls[0][0].params.signal).toBe( + controller.signal, + ); + }); + it("charges the reservation when the provider reports no token count", async () => { mockProviderReply(null); diff --git a/src/lib/insights/__tests__/comprehensive-briefing-grounding-disposal.test.ts b/src/lib/insights/__tests__/comprehensive-briefing-grounding-disposal.test.ts index 17c106731..88bcdecd4 100644 --- a/src/lib/insights/__tests__/comprehensive-briefing-grounding-disposal.test.ts +++ b/src/lib/insights/__tests__/comprehensive-briefing-grounding-disposal.test.ts @@ -165,6 +165,7 @@ function completion(content: string) { result: { content, tokensUsed: 10, model: "m" }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }; } diff --git a/src/lib/insights/__tests__/comprehensive-briefing-reroll.test.ts b/src/lib/insights/__tests__/comprehensive-briefing-reroll.test.ts index 00ce1b374..64bb13794 100644 --- a/src/lib/insights/__tests__/comprehensive-briefing-reroll.test.ts +++ b/src/lib/insights/__tests__/comprehensive-briefing-reroll.test.ts @@ -162,6 +162,7 @@ describe("daily-briefing re-roll on unchanged data", () => { }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); const outcome = await generateComprehensiveInsight("u1", { locale: "de" }); @@ -226,6 +227,7 @@ describe("daily-briefing re-roll on unchanged data", () => { result: { content: "not json at all", tokensUsed: 5, model: "m" }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); const outcome = await generateComprehensiveInsight("u1", { locale: "de" }); diff --git a/src/lib/insights/__tests__/comprehensive-budget-refusal.test.ts b/src/lib/insights/__tests__/comprehensive-budget-refusal.test.ts index e2b135eb3..9f61a3efe 100644 --- a/src/lib/insights/__tests__/comprehensive-budget-refusal.test.ts +++ b/src/lib/insights/__tests__/comprehensive-budget-refusal.test.ts @@ -166,6 +166,7 @@ beforeEach(() => { result: { content: VALID, tokensUsed: 900, model: "m" }, workingProvider: { providerType: "admin-openai" }, fallbackHops: [], + tokensSpent: 0, }); }); @@ -269,11 +270,13 @@ describe("generateComprehensiveInsight — daily token ceiling", () => { result: { content: "not json", tokensUsed: 400, model: "m" }, workingProvider: { providerType: "admin-openai" }, fallbackHops: [], + tokensSpent: 0, }) .mockResolvedValueOnce({ result: { content: VALID, tokensUsed: 900, model: "m" }, workingProvider: { providerType: "admin-openai" }, fallbackHops: [], + tokensSpent: 0, }); const outcome = await generateComprehensiveInsight("u1", { locale: "de" }); diff --git a/src/lib/insights/__tests__/comprehensive-generate-force.test.ts b/src/lib/insights/__tests__/comprehensive-generate-force.test.ts index 3878245df..c1ccda632 100644 --- a/src/lib/insights/__tests__/comprehensive-generate-force.test.ts +++ b/src/lib/insights/__tests__/comprehensive-generate-force.test.ts @@ -301,6 +301,7 @@ describe("generateComprehensiveInsight — content-hash gate (v1.16.8)", () => { }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); const outcome = await generateComprehensiveInsight("u1", { @@ -353,6 +354,7 @@ describe("generateComprehensiveInsight — content-hash gate (v1.16.8)", () => { }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); // The kilogram fingerprint no longer matches: a unit switch regenerates. @@ -429,6 +431,7 @@ describe("generateComprehensiveInsight — content-hash gate (v1.16.8)", () => { }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); const outcome = await generateComprehensiveInsight("u1", { @@ -467,6 +470,7 @@ describe("generateComprehensiveInsight — content-hash gate (v1.16.8)", () => { }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); const outcome = await generateComprehensiveInsight("u1", { @@ -503,6 +507,7 @@ describe("generateComprehensiveInsight — content-hash gate (v1.16.8)", () => { }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); const outcome = await generateComprehensiveInsight("u1", { @@ -553,6 +558,7 @@ describe("generateComprehensiveInsight — usage level", () => { }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); } diff --git a/src/lib/insights/__tests__/comprehensive-json-retry.test.ts b/src/lib/insights/__tests__/comprehensive-json-retry.test.ts index 3a6f0e78f..3795d2d85 100644 --- a/src/lib/insights/__tests__/comprehensive-json-retry.test.ts +++ b/src/lib/insights/__tests__/comprehensive-json-retry.test.ts @@ -130,11 +130,13 @@ describe("comprehensive JSON-retry", () => { }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }) .mockResolvedValueOnce({ result: { content: VALID, tokensUsed: 10, model: "m" }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); const outcome = await generateComprehensiveInsight("u1", { locale: "de" }); @@ -161,6 +163,7 @@ describe("comprehensive JSON-retry", () => { result: { content: "still not json", tokensUsed: 5, model: "m" }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); const outcome = await generateComprehensiveInsight("u1", { locale: "de" }); @@ -174,6 +177,7 @@ describe("comprehensive JSON-retry", () => { result: { content: VALID, tokensUsed: 10, model: "m" }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); const outcome = await generateComprehensiveInsight("u1", { locale: "de" }); diff --git a/src/lib/insights/__tests__/comprehensive-outbound-repair.test.ts b/src/lib/insights/__tests__/comprehensive-outbound-repair.test.ts index 97adb76f7..2bbcc3ea6 100644 --- a/src/lib/insights/__tests__/comprehensive-outbound-repair.test.ts +++ b/src/lib/insights/__tests__/comprehensive-outbound-repair.test.ts @@ -137,6 +137,7 @@ function reply(content: string) { result: { content, tokensUsed: 10, model: "m" }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }; } diff --git a/src/lib/insights/__tests__/comprehensive-plan-progress.test.ts b/src/lib/insights/__tests__/comprehensive-plan-progress.test.ts index eaf83631b..062085827 100644 --- a/src/lib/insights/__tests__/comprehensive-plan-progress.test.ts +++ b/src/lib/insights/__tests__/comprehensive-plan-progress.test.ts @@ -128,6 +128,7 @@ beforeEach(() => { }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); }); diff --git a/src/lib/insights/__tests__/comprehensive-privacy-flip.test.ts b/src/lib/insights/__tests__/comprehensive-privacy-flip.test.ts index 87fbb55b1..76408f423 100644 --- a/src/lib/insights/__tests__/comprehensive-privacy-flip.test.ts +++ b/src/lib/insights/__tests__/comprehensive-privacy-flip.test.ts @@ -167,6 +167,7 @@ describe("generateComprehensiveInsight — consent withdrawn in flight", () => { }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); userUpdateMany.mockResolvedValue({ count: 1 }); const { AiUnavailableError } = @@ -209,6 +210,7 @@ describe("generateComprehensiveInsight — scope/version commit guard", () => { }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); // Some User update landed: the guarded commit matches zero rows. userUpdateMany.mockResolvedValue({ count: 0 }); @@ -258,6 +260,7 @@ describe("generateComprehensiveInsight — scope/version commit guard", () => { }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); // The real DB returns count: 1 here too, since the guarded predicate // never references `updatedAt` — the credential-refresh write cannot @@ -304,6 +307,7 @@ describe("generateComprehensiveInsight — scope/version commit guard", () => { }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); // Simulates a genuine concurrent settings/profile change (exclude list, // gender, or display name edited) between the read above and the @@ -349,6 +353,7 @@ describe("generateComprehensiveInsight — scope/version commit guard", () => { }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); userUpdateMany.mockResolvedValue({ count: 1 }); @@ -388,6 +393,7 @@ describe("generateComprehensiveInsight — scope/version commit guard", () => { }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); userUpdateMany.mockResolvedValue({ count: 0 }); diff --git a/src/lib/insights/__tests__/comprehensive-timeout-honored.test.ts b/src/lib/insights/__tests__/comprehensive-timeout-honored.test.ts index 39307a17c..19f543c8a 100644 --- a/src/lib/insights/__tests__/comprehensive-timeout-honored.test.ts +++ b/src/lib/insights/__tests__/comprehensive-timeout-honored.test.ts @@ -135,6 +135,7 @@ describe("comprehensive timeout honouring", () => { result: { content: VALID, tokensUsed: 10, model: "m" }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); await generateComprehensiveInsight("u1", { locale: "de" }); @@ -150,6 +151,7 @@ describe("comprehensive timeout honouring", () => { result: { content: VALID, tokensUsed: 10, model: "m" }, workingProvider: { providerType: "openai" }, fallbackHops: [], + tokensSpent: 0, }); await generateComprehensiveInsight("u1", { locale: "de" }); diff --git a/src/lib/insights/__tests__/comprehensive-warm-timeout.test.ts b/src/lib/insights/__tests__/comprehensive-warm-timeout.test.ts index ef7cd05ad..5468a5de7 100644 --- a/src/lib/insights/__tests__/comprehensive-warm-timeout.test.ts +++ b/src/lib/insights/__tests__/comprehensive-warm-timeout.test.ts @@ -145,6 +145,7 @@ describe("comprehensive warm-path timeout + failure observability", () => { result: { content: VALID, tokensUsed: 10, model: "m" }, workingProvider: { providerType: "codex" }, fallbackHops: [], + tokensSpent: 0, }); const outcome = await generateComprehensiveInsight("u1", { locale: "de" }); @@ -154,6 +155,34 @@ describe("comprehensive warm-path timeout + failure observability", () => { expect(params.timeoutMs).toBe(AI_BUDGETS.comprehensive.timeoutMs); }); + it("sends nothing more once the caller's bound has given up (#1279)", async () => { + // A slow model: its reply (not valid JSON, so the generator would ask + // again) lands only after the warm's bound fired and aborted the run. + const controller = new AbortController(); + runRawCompletionWithFallback.mockImplementation(async () => { + controller.abort(); + return { + result: { content: "not json", tokensUsed: 10, model: "m" }, + workingProvider: { providerType: "codex" }, + fallbackHops: [], + tokensSpent: 0, + }; + }); + + await generateComprehensiveInsight("u1", { + locale: "de", + signal: controller.signal, + }); + + // The call in flight carried the bound's signal, so the client could tear + // its request down, and no retry left after the bound. + expect(runRawCompletionWithFallback).toHaveBeenCalledTimes(1); + expect(runRawCompletionWithFallback.mock.calls[0][0].params.signal).toBe( + controller.signal, + ); + expect(userUpdate).not.toHaveBeenCalled(); + }); + it("does not silently persist an empty block when every provider fails", async () => { // The mocked class ignores the argument; `[]` satisfies the real type. runRawCompletionWithFallback.mockRejectedValue( diff --git a/src/lib/insights/__tests__/status-negative-window.test.ts b/src/lib/insights/__tests__/status-negative-window.test.ts index 63cf60d1e..fbc8c9263 100644 --- a/src/lib/insights/__tests__/status-negative-window.test.ts +++ b/src/lib/insights/__tests__/status-negative-window.test.ts @@ -22,6 +22,7 @@ vi.mock("@/lib/db", () => ({ import { STATUS_NEGATIVE_WINDOW_BASE_MS, clearStatusNegativeWindows, + holdStatusNoteWhileBenched, statusNegativeWindowMs, statusNoteCoolingDown, writeStatusNegativeWindow, @@ -116,6 +117,57 @@ describe("writeStatusNegativeWindow", () => { }); }); +describe("holdStatusNoteWhileBenched", () => { + // A run that sent nothing because every provider was benched says nothing + // about the note: counting it left the note waiting up to two hours after + // the provider recovered. + it("opens the first window and leaves the streak as it was", async () => { + findUnique.mockResolvedValue({ retryAt: new Date(Date.now() - MINUTE) }); + const before = Date.now(); + await holdStatusNoteWhileBenched({ + userId: "u1", + cacheAction: "insights.weight-status.en", + todayKey: TODAY, + }); + const args = written(); + expect(args.update).not.toHaveProperty("failureStreak"); + expect(args.create).toMatchObject({ failureStreak: 0, dateKey: TODAY }); + const window = args.update.retryAt.getTime() - before; + expect(window).toBeGreaterThanOrEqual(STATUS_NEGATIVE_WINDOW_BASE_MS); + expect(window).toBeLessThan(STATUS_NEGATIVE_WINDOW_BASE_MS + MINUTE); + }); + + it("leaves an open window alone, and the failure after it uncounted", async () => { + findUnique.mockResolvedValue({ + retryAt: new Date(Date.now() + MINUTE), + failureStreak: 3, + }); + await holdStatusNoteWhileBenched({ + userId: "u1", + cacheAction: "insights.weight-status.en", + todayKey: TODAY, + }); + await writeStatusNegativeWindow({ + userId: "u1", + cacheAction: "insights.weight-status.en", + todayKey: TODAY, + reason: "error", + }); + expect(upsert).not.toHaveBeenCalled(); + }); + + it("never throws: a missed hold only lets the next failure count", async () => { + findUnique.mockRejectedValue(new Error("pool exhausted")); + await expect( + holdStatusNoteWhileBenched({ + userId: "u1", + cacheAction: "insights.weight-status.en", + todayKey: TODAY, + }), + ).resolves.toBeUndefined(); + }); +}); + describe("statusNegativeWindowMs", () => { it("doubles from five minutes to a two-hour ceiling", () => { expect([1, 2, 3, 4, 5, 6, 7, 30].map(statusNegativeWindowMs)).toEqual( @@ -157,4 +209,14 @@ describe("clearStatusNegativeWindows", () => { updateMany.mockRejectedValue(new Error("pool exhausted")); await expect(clearStatusNegativeWindows("u1")).resolves.toBeUndefined(); }); + + it("closes them for every record when the operator's provider changed", async () => { + await clearStatusNegativeWindows(); + expect(updateMany).toHaveBeenCalledWith({ + where: { + OR: [{ retryAt: { not: null } }, { failureStreak: { gt: 0 } }], + }, + data: { retryAt: null, negativeReason: null, failureStreak: 0 }, + }); + }); }); diff --git a/src/lib/insights/__tests__/status-provider-budget.test.ts b/src/lib/insights/__tests__/status-provider-budget.test.ts index a6aaca6be..046eb60bd 100644 --- a/src/lib/insights/__tests__/status-provider-budget.test.ts +++ b/src/lib/insights/__tests__/status-provider-budget.test.ts @@ -60,6 +60,7 @@ vi.mock("@/lib/db", () => ({ // #1279 — the note's negative window, read before any provider work. insightStatusCache: { findUnique: vi.fn(async () => noteWindow), + upsert: vi.fn(async () => ({})), }, }, })); @@ -125,6 +126,8 @@ function mockProviderReply(tokensUsed: number | null) { runRawCompletionWithFallback.mockResolvedValue({ result: { content: '{"summary":"ok"}', model: "m", tokensUsed }, workingProvider: { providerType: "admin-openai" }, + fallbackHops: [], + tokensSpent: 0, }); } @@ -269,6 +272,19 @@ describe("runStatusCompletion — ledger accounting", () => { expect(chat.allowed).toBe(true); }); + it("charges what the hops that failed before the answer spent (#1279)", async () => { + runRawCompletionWithFallback.mockResolvedValue({ + result: { content: '{"summary":"ok"}', model: "m", tokensUsed: 100 }, + workingProvider: { providerType: "admin-openai" }, + fallbackHops: [], + tokensSpent: 2700, + }); + + await runStatusCompletion(completionArgs()); + + expect(ledgerTotal).toBe(100 + 2700); + }); + it("charges the reservation when the provider reports no token count", async () => { mockProviderReply(null); @@ -320,6 +336,27 @@ describe("runStatusCompletion — ledger accounting", () => { expect(ledgerTotal).toBe(0); }); + it("aborts the call its bound cut off, so nothing runs on detached (#1279)", async () => { + vi.useFakeTimers(); + try { + let signal: AbortSignal | undefined; + runRawCompletionWithFallback.mockImplementation( + (args: { params: { signal?: AbortSignal } }) => { + signal = args.params.signal; + return new Promise(() => {}); + }, + ); + const pending = runStatusCompletion(completionArgs()); + await vi.waitFor(() => expect(signal).toBeDefined()); + expect(signal?.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(10 * 60_000); + expect((await pending).kind).toBe("timeout"); + expect(signal?.aborted).toBe(true); + } finally { + vi.useRealTimers(); + } + }); + it("asks no provider about a note inside its negative window (#1279)", async () => { noteWindow = { retryAt: new Date(Date.now() + 60_000) }; const result = await runStatusCompletion( @@ -338,6 +375,36 @@ describe("runStatusCompletion — ledger accounting", () => { expect(again.kind).toBe("ok"); }); + it("holds the note without counting a run the bench sent nothing for (#1279)", async () => { + runRawCompletionWithFallback.mockRejectedValue( + Object.assign(new Error("All 1 configured providers failed"), { + sentNothing: true, + }), + ); + const { prisma } = await import("@/lib/db"); + const result = await runStatusCompletion( + completionArgs({ cacheAction: "insights.weight-status.en" }), + ); + expect(result.kind).toBe("error"); + expect(prisma.insightStatusCache.upsert).toHaveBeenCalledWith( + expect.objectContaining({ + create: expect.objectContaining({ failureStreak: 0 }), + update: expect.not.objectContaining({ + failureStreak: expect.anything(), + }), + }), + ); + }); + + it("leaves a run that reached a provider to the caller's counted window", async () => { + runRawCompletionWithFallback.mockRejectedValue(new Error("upstream down")); + const { prisma } = await import("@/lib/db"); + await runStatusCompletion( + completionArgs({ cacheAction: "insights.weight-status.en" }), + ); + expect(prisma.insightStatusCache.upsert).not.toHaveBeenCalled(); + }); + it("charges what a failed chain reported spending (#1279)", async () => { // A model that reasons and never answers is billed for every call; a // full refund let background work spend without the ceiling seeing it. @@ -377,6 +444,7 @@ describe("runStatusCompletion — cost owner decides the ceiling", () => { runRawCompletionWithFallback.mockResolvedValue({ result: { content: '{"summary":"ok"}', model: "m", tokensUsed: 100 }, workingProvider: { providerType: "anthropic" }, + tokensSpent: 0, }); const result = await runStatusCompletion(completionArgs()); diff --git a/src/lib/insights/__tests__/status-provider-slow-model.test.ts b/src/lib/insights/__tests__/status-provider-slow-model.test.ts index e5bce7962..08db9e390 100644 --- a/src/lib/insights/__tests__/status-provider-slow-model.test.ts +++ b/src/lib/insights/__tests__/status-provider-slow-model.test.ts @@ -82,6 +82,7 @@ function answersAfter(afterMs: number) { tokensUsed: 80, }, workingProvider: { providerType: "local" }, + tokensSpent: 0, }), afterMs, ), diff --git a/src/lib/insights/briefing-provider.ts b/src/lib/insights/briefing-provider.ts index d8ad8aa2d..d4758b9a2 100644 --- a/src/lib/insights/briefing-provider.ts +++ b/src/lib/insights/briefing-provider.ts @@ -104,6 +104,23 @@ export interface RunBriefingCompletionArgs { * budget; the client adds the thinking budget itself. */ reasoningJob?: BackgroundReasoningJob; + /** + * #1279 — a person is waiting on this call: the on-demand "Re-run + * analysis". It keeps the background share of the day's ceiling, and walks + * a provider the health ledger benched at the tail of the chain instead of + * leaving it out, so the person gets that provider's own answer or refusal. + * Absent for the scheduled generator. + */ + interactive?: boolean; + /** + * #1279 — the caller's outer bound (the nightly and forced warms, the + * morning refresh). It reaches every provider call, so a call the bound + * gave up on is torn down, and once it fired no further call is reserved + * or sent: a briefing that outlives its bound does not keep running + * detached. The on-demand route passes none on purpose: it finishes and + * writes its briefing after the browser stopped waiting (#786). + */ + signal?: AbortSignal; } /** @@ -123,6 +140,7 @@ export interface RunBriefingCompletionArgs { export async function runBriefingCompletion( args: RunBriefingCompletionArgs, ): Promise { + args.signal?.throwIfAborted(); const dateKey = args.dateKey ?? (await buildDateKey(args.userId)); // v1.41 — one resolution per call: the operator's switch and cap, the payer @@ -175,9 +193,9 @@ export async function runBriefingCompletion( try { outcome = await runRawCompletionWithFallback({ userId: args.userId, - // A background generator: an operator-funded fallback hop is rationed at - // the job share, exactly as the reservation above was. + // Rationed at the job share, exactly as the reservation above was. surface: "job", + ...(args.interactive ? { interactive: true } : {}), providers: args.chain, params: { ...singleUserTurn({ @@ -187,6 +205,7 @@ export async function runBriefingCompletion( maxTokens: args.maxTokens, seed: args.seed, timeoutMs: args.timeoutMs, + signal: args.signal, // Every briefing call parses its reply with `JSON.parse`, so opt the // chains with a native JSON mode into it. responseFormat: "json", @@ -219,7 +238,9 @@ export async function runBriefingCompletion( throw err; } - const actualTokens = outcome.result.tokensUsed ?? reservation.reserved; + // #1279 — plus what the hops that failed before it spent. + const actualTokens = + (outcome.result.tokensUsed ?? reservation.reserved) + outcome.tokensSpent; await reconcileSpend( args.userId, reservation.reserved, diff --git a/src/lib/insights/briefing-request/completion.ts b/src/lib/insights/briefing-request/completion.ts index 923c29720..de7bb137a 100644 --- a/src/lib/insights/briefing-request/completion.ts +++ b/src/lib/insights/briefing-request/completion.ts @@ -62,6 +62,8 @@ export function runBriefingCall( // the client's 60 s default on large accounts. timeoutMs: ctx.effectiveTimeoutMs, stage: options.stage, + // The person who pressed "Re-run analysis" is waiting on this call. + interactive: true, // A briefing asked for by hand reasons exactly like the scheduled one; // the retries repair a reply and do not. ...(options.reasoning ? { reasoningJob: "daily_briefing" as const } : {}), diff --git a/src/lib/insights/comprehensive-aggregator.ts b/src/lib/insights/comprehensive-aggregator.ts index 0c9e035a5..6dcef4f28 100644 --- a/src/lib/insights/comprehensive-aggregator.ts +++ b/src/lib/insights/comprehensive-aggregator.ts @@ -98,6 +98,8 @@ * shape `summarize()` would return. */ import { prisma } from "@/lib/db"; +import { runDetached } from "@/lib/logging/detached"; +import { trackBackgroundTask } from "@/lib/logging/background-tasks"; import type { DataSummary } from "@/lib/analytics/trends"; import { liveTrend, @@ -254,14 +256,12 @@ export async function buildComprehensiveAggregate( // bp-raw, first-at, coverage-probe) without re-instrumenting. const timings: Record = {}; - // v1.4.37.1 — fire-and-forget. See `src/app/api/analytics/route.ts` - // for the full rationale: awaiting this on the read path can stall - // the event loop for 30–60 s on power-user accounts whose iOS step - // samples keep the 90-day window slightly stale. The downstream - // coverage probe falls back to live SQL for uncovered types, so - // correctness is preserved; the read just doesn't block waiting - // for the background refresh. - void ensureUserRollupsFresh(userId); + // v1.4.37.1 — fire-and-forget (see `src/app/api/analytics/route.ts`): + // awaiting it can stall the event loop for 30–60 s on accounts whose iOS + // steps keep the 90-day window stale, and the coverage probe falls back to + // live SQL meanwhile. Registered so the suite's settle hook and the + // statement counter can await it. + trackBackgroundTask(runDetached(() => ensureUserRollupsFresh(userId))); // v1.4.36 — per-type coverage probe. The legacy global COUNT // returned true as soon as any one type had a DAY bucket, which made diff --git a/src/lib/insights/comprehensive-generate.ts b/src/lib/insights/comprehensive-generate.ts index 8ae3feb59..a5fb5d5a9 100644 --- a/src/lib/insights/comprehensive-generate.ts +++ b/src/lib/insights/comprehensive-generate.ts @@ -244,6 +244,8 @@ async function rerollBriefingParagraph(args: { * slow-backend allowance the main generation does. */ effectiveTimeoutMs: number; + /** The caller's bound: a re-roll it gave up on sends nothing more. */ + signal?: AbortSignal; }): Promise<{ text: string; providerType: string } | null> { let cached: Record; try { @@ -277,6 +279,7 @@ async function rerollBriefingParagraph(args: { // generation is not clipped at the client's 60 s default mid-stream. // v1.25 — honours the per-user response-timeout setting (see caller). timeoutMs: args.effectiveTimeoutMs, + signal: args.signal, stage: "reroll", // v1.41 — the re-roll is a fresh pass at the paragraph, so it may think // like the generation it replaces; the operator's switch still wins. @@ -716,13 +719,10 @@ interface GenerateOptions { /** Skip the 24 h cache short-circuit and force a fresh generation. */ force?: boolean; /** - * v1.9.0 — abort the generation when the caller has given up on it. - * - * The on-demand `forceWarmUser` path bounds the comprehensive step with a - * timeout and then proceeds to warm the per-status caches itself. Without - * this signal a comprehensive that resolves AFTER the timeout would still - * write a cache row + timestamp the caller no longer expects. The signal - * lets the caller cut the generation off before the cache write. + * The caller's bound gave up on the generation (the warms bound it, then + * move on). Every provider call carries it, so the call in flight is torn + * down and no later one is sent (#1279), and a generation that still + * resolves writes no cache row the caller no longer expects. */ signal?: AbortSignal; } @@ -1141,6 +1141,7 @@ export async function generateComprehensiveInsight( features, comparison: groundingExtra, effectiveTimeoutMs, + signal, }); if (rerolled) { // The reroll rewrites content derived from the scope captured before @@ -1232,6 +1233,7 @@ export async function generateComprehensiveInsight( // generation is not clipped at the client's 60 s default mid-stream. // v1.25 — honours the per-user response-timeout setting. timeoutMs: effectiveTimeoutMs, + signal, stage: "generate", // v1.41 — the briefing is one of the few background jobs that may // reason (medium on the person's own provider, low on the operator's, @@ -1328,6 +1330,7 @@ export async function generateComprehensiveInsight( // v1.21.5 — wider upstream budget; see the first generation call. // v1.25 — honours the per-user response-timeout setting. timeoutMs: effectiveTimeoutMs, + signal, stage: "json-retry", }); result = retry.result; @@ -1421,6 +1424,7 @@ export async function generateComprehensiveInsight( // v1.21.5 — wider upstream budget; see the first generation call. // v1.25 — honours the per-user response-timeout setting. timeoutMs: effectiveTimeoutMs, + signal, stage: "grounding-retry", }); const retryInsights = parseComprehensiveResult(retry.result.content); @@ -1553,6 +1557,7 @@ export async function generateComprehensiveInsight( temperature: AI_BUDGETS.comprehensive.temperature, maxTokens: resolveInsightsMaxTokens(), timeoutMs: effectiveTimeoutMs, + signal, stage: "screen-retry", }); const retryInsights = parseComprehensiveResult(retry.result.content); @@ -1594,15 +1599,10 @@ export async function generateComprehensiveInsight( } } - // v1.9.0 — the caller abandoned this generation (its bounded timeout - // fired and it has moved on to warm the per-status caches itself). Bail - // out before the cache write so a late-resolving comprehensive cannot - // stamp a timestamp/text the caller no longer expects. The provider call - // already cost what it cost; what we must NOT do now is touch the cache. + // The caller abandoned this generation (its bound fired). Bail out before + // the cache write, annotated and marked like every other failure: an + // abandoned generation is a real "no fresh briefing tonight" outcome. if (signal?.aborted) { - // v1.28.30 — annotated + marked like every other failure: an abandoned - // generation is a real "no fresh briefing tonight" outcome, and the read - // path needs the marker to render an honest state instead of silence. return failBeforeProvider(userId, locale, "aborted"); } diff --git a/src/lib/insights/comprehensive-overview/mood-series.ts b/src/lib/insights/comprehensive-overview/mood-series.ts index 40efa5afd..6deda77c9 100644 --- a/src/lib/insights/comprehensive-overview/mood-series.ts +++ b/src/lib/insights/comprehensive-overview/mood-series.ts @@ -22,6 +22,8 @@ * left on it deliberately (see their call sites in `build.ts`). */ import { prisma } from "@/lib/db"; +import { runDetached } from "@/lib/logging/detached"; +import { trackBackgroundTask } from "@/lib/logging/background-tasks"; import type { DataPoint } from "@/lib/analytics/trends"; import { ensureUserMoodRollupsFresh, @@ -41,7 +43,7 @@ export async function readOverviewMoodSeries( userId: string, since: Date, ): Promise { - void ensureUserMoodRollupsFresh(userId); + trackBackgroundTask(runDetached(() => ensureUserMoodRollupsFresh(userId))); const moodRollupDayRows = await readMoodDayRollups(userId, since); if (moodRollupDayRows.length > 0) { return { diff --git a/src/lib/insights/mood-daily-points.ts b/src/lib/insights/mood-daily-points.ts index 00b91b87a..9309c1bb0 100644 --- a/src/lib/insights/mood-daily-points.ts +++ b/src/lib/insights/mood-daily-points.ts @@ -4,6 +4,8 @@ * Split out of `features.ts`, which builds the block from them. */ import { prisma } from "@/lib/db"; +import { runDetached } from "@/lib/logging/detached"; +import { trackBackgroundTask } from "@/lib/logging/background-tasks"; import { ensureUserMoodRollupsFresh, readMoodDayRollups, @@ -39,7 +41,7 @@ export async function readMoodDailyPoints( totalEntries: number; latestScore: number | null; }> { - void ensureUserMoodRollupsFresh(userId); + trackBackgroundTask(runDetached(() => ensureUserMoodRollupsFresh(userId))); // Five-year window mirrors the mood-rollup writer default; covers // every realistic user history span without an unbounded scan. const fiveYearsAgo = new Date(Date.now() - 5 * 365 * 24 * 60 * 60 * 1000); diff --git a/src/lib/insights/status-negative-window.ts b/src/lib/insights/status-negative-window.ts index bafb032a2..e748f89cd 100644 --- a/src/lib/insights/status-negative-window.ts +++ b/src/lib/insights/status-negative-window.ts @@ -72,6 +72,48 @@ export async function writeStatusNegativeWindow(args: { }); } +/** + * #1279 — hold a note while its chain sent nothing: every provider was + * benched for background work (or refused by the operator cost cap). Such a + * run says nothing about the note, and counting it left the note waiting up + * to two hours after the provider recovered. Opens the first window unless + * one is open and leaves the streak as it was, so the caller's own failure + * write lands inside it and neither moves nor counts. Best-effort: a missed + * hold only lets that write count. + */ +export async function holdStatusNoteWhileBenched(args: { + userId: string; + cacheAction: string; + todayKey: string; +}): Promise { + const key = statusCacheKey(args.cacheAction); + if (!key) return; + const where = { userId_metric_locale: { userId: args.userId, ...key } }; + try { + const row = await prisma.insightStatusCache.findUnique({ + where, + select: { retryAt: true }, + }); + const now = Date.now(); + if (row?.retryAt && row.retryAt.getTime() > now) return; + const retryAt = new Date(now + STATUS_NEGATIVE_WINDOW_BASE_MS); + await prisma.insightStatusCache.upsert({ + where, + create: { + userId: args.userId, + ...key, + dateKey: args.todayKey, + retryAt, + negativeReason: "error", + failureStreak: 0, + }, + update: { retryAt, negativeReason: "error" }, + }); + } catch { + // The caller's own write still opens a window. + } +} + /** * True while the note's negative-cache window is open. #1279 — checked right * before a status generation reaches a provider (`runStatusCompletion`), so @@ -94,15 +136,17 @@ export async function statusNoteCoolingDown( /** * Close every open window on one record's notes, streaks included: the person * asked for fresh notes (the regenerate action) or changed what answers them. - * Best-effort: a failed write leaves windows that lapse on their own. + * Without `userId`, on every record's notes: the operator changed the + * provider every account may fall back on. Best-effort: a failed write leaves + * windows that lapse on their own. */ export async function clearStatusNegativeWindows( - userId: string, + userId?: string, ): Promise { try { await prisma.insightStatusCache.updateMany({ where: { - userId, + ...(userId ? { userId } : {}), OR: [{ retryAt: { not: null } }, { failureStreak: { gt: 0 } }], }, data: { retryAt: null, negativeReason: null, failureStreak: 0 }, diff --git a/src/lib/insights/status-provider.ts b/src/lib/insights/status-provider.ts index 0f6945bcf..37750e7c5 100644 --- a/src/lib/insights/status-provider.ts +++ b/src/lib/insights/status-provider.ts @@ -21,6 +21,7 @@ import { reserveBudget, resolveCostOwner, resolveDailyCapFor, + type BudgetCostOwner, } from "@/lib/ai/coach/budget"; import { AI_BUDGETS, @@ -36,7 +37,10 @@ import { type BackgroundReasoningJob, } from "@/lib/ai/reasoning/levels"; import { resolveJobReasoning } from "@/lib/ai/reasoning/controls"; -import { statusNoteCoolingDown } from "./status-negative-window"; +import { + holdStatusNoteWhileBenched, + statusNoteCoolingDown, +} from "./status-negative-window"; /** * v1.41 — the fallback ceiling for a status-path call that reasons. Thinking @@ -228,15 +232,11 @@ export async function runStatusCompletion( return { kind: "none", reason: refusal.reason }; } - // Honour the per-user response-timeout setting the operator dials in for a - // slow self-hosted / local backend (Settings → AI). This is the single - // chokepoint every status / reference surface funnels through — per-metric - // cards, the batched assessment, the derived assessments, and the period - // narratives — so resolving it here threads the setting onto all of them at - // once. A positive stored value wins (seconds → ms); unset falls back to the - // status-path budget. Applied to the upstream call's own `timeoutMs` and, - // times the calls a run may make, to the outer `withTimeout` cap, so a - // raised value is not silently clipped by a wall-clock sized for less. + // The per-user response-timeout setting (Settings → AI), threaded here onto + // every status / reference surface at once. A positive stored value wins + // (seconds → ms); unset falls back to the status-path budget. Applied to the + // call's own `timeoutMs` and, times the calls a run may make, to the outer + // `withTimeout` cap, so a raised value is not clipped by a shorter bound. const settingsRow = await prisma.user.findUnique({ where: { id: userId }, select: { aiResponseTimeoutSeconds: true }, @@ -260,12 +260,9 @@ export async function runStatusCompletion( // The answer budget. Each client adds the thinking budget on top itself. const maxTokens = args.maxTokens ?? AI_BUDGETS.status.maxTokens; - // The day's token ledger. Until now this chokepoint — the provider entry for - // EVERY status/reference family (the specialised cards, the generic metric - // cards, biomarker cards, the batched assessment, the derived scores, the - // period narrative, and the off-request Coach memory workers: rolling - // summary, fact extraction, plan proposals) — ran with no accounting at all, - // so none of that spend appeared in `coach_usage` and no ceiling applied. + // The day's token ledger, for EVERY status/reference family (the cards, the + // batched assessment, the derived scores, the period narrative, and the + // off-request Coach memory workers). // // The reservation is atomic (single upsert-increment), matching the Coach and // document paths: a read-then-write check would let concurrent generations @@ -302,6 +299,10 @@ export async function runStatusCompletion( return { kind: "error" }; } + // #1279 — the run's bound aborts the call it cuts off, so a provider call + // the bound gave up on (and any repeat inside it) stops instead of running + // on detached, with its spend reported nowhere. + const abort = new AbortController(); const raced = await withTimeout( () => runRawCompletionWithFallback({ @@ -324,34 +325,26 @@ export async function runStatusCompletion( responseFormat: args.responseFormat === "text" ? undefined : "json", timeoutMs: effectiveTimeoutMs, }), + signal: abort.signal, // v1.41 — only a named job ever carries this; see `reasoningJob`. ...(reasoning ? { reasoning } : {}), }, }), statusRunBoundMs(effectiveTimeoutMs), null, + () => abort.abort(), ); if (raced.timedOut || raced.errored || raced.value === null) { - // A timeout or failure: charge only what a reply reported spending, never - // an invented figure. #1279 — a failed chain's report counts: a model - // that reasons without answering bills every call, and a full refund let - // background work spend without the day's ceiling ever seeing it. - const spent = spentOnFailure(raced.error); - await reconcileSpend(userId, reservation.reserved, spent, dateKey, 0, { - servedBy: null, - reservedOwner: reservation.owner, - }); - return { kind: raced.timedOut ? "timeout" : "error" }; + return failedRun(raced, { userId, cacheAction, dateKey, reservation }); } - const { result, workingProvider } = raced.value; - // Reconcile against what the provider actually reported. This runs for the - // empty-content branch too: those tokens were burned upstream even though the - // reply was unusable, so they stay on the ledger rather than being refunded - // into a free retry loop. Falls back to the reservation when the provider - // reports no count, so an unreported generation is never billed as zero. - const actualTokens = result.tokensUsed ?? reservation.reserved; + const { result, workingProvider, tokensSpent } = raced.value; + // What the provider reported, plus (#1279) what the hops that failed before + // it spent. The empty-content branch is charged too: those tokens were + // burned upstream. No count falls back to the reservation, never to zero. + const actualTokens = + (result.tokensUsed ?? reservation.reserved) + tokensSpent; await reconcileSpend(userId, reservation.reserved, actualTokens, dateKey, 0, { servedBy: workingProvider.providerType, reservedOwner: reservation.owner, @@ -397,6 +390,44 @@ export async function runStatusCompletion( }; } +/** + * A run that timed out or failed. It charges only what a reply reported + * spending, never an invented figure; #1279 — a failed chain's report counts, + * since a model that reasons without answering bills every call, and a full + * refund let background work spend without the day's ceiling seeing it. A + * chain that sent nothing (the bench) holds the note without counting it. + */ +async function failedRun( + raced: { timedOut: boolean; error?: unknown }, + run: { + userId: string; + cacheAction: string; + dateKey: string; + reservation: { reserved: number; owner: BudgetCostOwner }; + }, +): Promise { + const spent = spentOnFailure(raced.error); + await reconcileSpend( + run.userId, + run.reservation.reserved, + spent, + run.dateKey, + 0, + { + servedBy: null, + reservedOwner: run.reservation.owner, + }, + ); + if ((raced.error as { sentNothing?: unknown } | null)?.sentNothing === true) { + await holdStatusNoteWhileBenched({ + userId: run.userId, + cacheAction: run.cacheAction, + todayKey: run.dateKey, + }); + } + return { kind: raced.timedOut ? "timeout" : "error" }; +} + /** * Over the day's ceiling. Reported as `error` — a TRANSIENT miss the caller * serves the fallback for without persisting it — deliberately NOT `none`, diff --git a/src/lib/integrations/sync-kit/__tests__/after-backfill.test.ts b/src/lib/integrations/sync-kit/__tests__/after-backfill.test.ts new file mode 100644 index 000000000..b5e09f480 --- /dev/null +++ b/src/lib/integrations/sync-kit/__tests__/after-backfill.test.ts @@ -0,0 +1,88 @@ +/** + * The end of a provider's full-history backfill marks the read caches stale. + * + * The per-write tail always did (through `afterMeasurementMutation`), but the + * backfill tail refolded the rollups and re-warmed the status cards without + * touching the analytics caches, so the dashboard and the analytics reads went + * on serving the pre-backfill body until its TTL ran out. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/rollups/measurement-rollups", () => ({ + collapseToTypeDayKeys: (keys: Array<{ type: string; measuredAt: Date }>) => + keys, + recomputeUserRollups: vi.fn(async () => undefined), +})); +vi.mock("@/lib/cache/invalidate", () => ({ + invalidateUserMeasurements: vi.fn(), + invalidateUserDashboardSnapshot: vi.fn(), +})); +vi.mock("@/lib/insights/status-invalidation", () => ({ + invalidateStatusInsightsForTypes: vi.fn(async () => undefined), +})); +vi.mock("@/lib/rollups/after-measurement-mutation", () => ({ + afterMeasurementMutation: vi.fn(), +})); +vi.mock("@/lib/arrivals/measurement-emit", () => ({ + emitInsertedMeasurementArrivals: vi.fn(async () => undefined), +})); + +import { invalidateUserMeasurements } from "@/lib/cache/invalidate"; +import { invalidateStatusInsightsForTypes } from "@/lib/insights/status-invalidation"; +import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; + +import { afterProviderBackfill } from "../after-write"; + +const keys = [ + { type: "WEIGHT" as const, measuredAt: new Date("2026-03-05T00:00:00Z") }, + { type: "PULSE" as const, measuredAt: new Date("2026-01-10T00:00:00Z") }, + { type: "WEIGHT" as const, measuredAt: new Date("2026-02-01T00:00:00Z") }, +]; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("afterProviderBackfill", () => { + it("marks the user's read caches stale back to the first touched day", async () => { + await afterProviderBackfill({ userId: "u1", provider: "fitbit", keys }); + + expect(recomputeUserRollups).toHaveBeenCalledWith("u1", { + types: ["WEIGHT", "PULSE"], + from: new Date("2026-01-10T00:00:00Z"), + to: new Date("2026-03-06T00:00:00Z"), + }); + expect(invalidateUserMeasurements).toHaveBeenCalledTimes(1); + expect(invalidateUserMeasurements).toHaveBeenCalledWith("u1", { + touchedFrom: new Date("2026-01-10T00:00:00Z"), + }); + // After the fold, so a reader rebuilds against the new rollups. + expect( + vi.mocked(invalidateUserMeasurements).mock.invocationCallOrder[0], + ).toBeGreaterThan( + vi.mocked(recomputeUserRollups).mock.invocationCallOrder[0]!, + ); + expect(invalidateStatusInsightsForTypes).toHaveBeenCalledWith( + "u1", + ["WEIGHT", "PULSE"], + { source: "sync" }, + ); + }); + + it("marks them stale even when the fold fails, and still throws for the caller", async () => { + vi.mocked(recomputeUserRollups).mockRejectedValueOnce(new Error("fold")); + + await expect( + afterProviderBackfill({ userId: "u1", provider: "fitbit", keys }), + ).rejects.toThrow("fold"); + expect(invalidateUserMeasurements).toHaveBeenCalledWith("u1", { + touchedFrom: new Date("2026-01-10T00:00:00Z"), + }); + }); + + it("does nothing for a backfill that touched nothing", async () => { + await afterProviderBackfill({ userId: "u1", provider: "fitbit", keys: [] }); + expect(recomputeUserRollups).not.toHaveBeenCalled(); + expect(invalidateUserMeasurements).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/integrations/sync-kit/after-write.ts b/src/lib/integrations/sync-kit/after-write.ts index 99c0b542c..61c0c394b 100644 --- a/src/lib/integrations/sync-kit/after-write.ts +++ b/src/lib/integrations/sync-kit/after-write.ts @@ -29,7 +29,10 @@ import { emitInsertedMeasurementArrivals, type InsertedMeasurementArrivalRow, } from "@/lib/arrivals/measurement-emit"; -import { invalidateUserDashboardSnapshot } from "@/lib/cache/invalidate"; +import { + invalidateUserDashboardSnapshot, + invalidateUserMeasurements, +} from "@/lib/cache/invalidate"; import { invalidateStatusInsightsForTypes } from "@/lib/insights/status-invalidation"; import { getEvent } from "@/lib/logging/context"; import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation"; @@ -116,8 +119,14 @@ export function invalidateProviderStatus( /** * The end of a full-history backfill: refold the whole touched range once - * (from the first touched day to the end of the last), then re-warm the - * touched status cards. Throws when the fold fails, for the caller to count. + * (from the first touched day to the end of the last), mark the read caches + * stale back to the first touched day, then re-warm the touched status cards. + * Throws when the fold fails, for the caller to count. + * + * The caches are marked even when the fold fails: the rows changed either + * way, and a cached analytics body built before the backfill would otherwise + * be served until its TTL ran out. That is the same order and the same + * posture as the per-write tail (`afterMeasurementMutation`). */ export async function afterProviderBackfill(input: { userId: string; @@ -135,6 +144,10 @@ export async function afterProviderBackfill(input: { // The keys are UTC day starts; the end is past the last touched day so the // aggregator's `< to` bound covers it. const to = new Date(sorted[sorted.length - 1]! + 24 * 60 * 60 * 1000); - await recomputeUserRollups(input.userId, { types, from, to }); + try { + await recomputeUserRollups(input.userId, { types, from, to }); + } finally { + invalidateUserMeasurements(input.userId, { touchedFrom: from }); + } invalidateProviderStatus(input.userId, input.provider, types); } diff --git a/src/lib/jobs/registry/__tests__/preload-handlers.test.ts b/src/lib/jobs/registry/__tests__/preload-handlers.test.ts new file mode 100644 index 000000000..cbb8a425a --- /dev/null +++ b/src/lib/jobs/registry/__tests__/preload-handlers.test.ts @@ -0,0 +1,167 @@ +/** + * The worker's boot preload of every job handler module. + * + * Handlers load on their queue's first job, so a module that cannot load + * stopped failing the boot and started failing jobs instead: the queue burned + * its retries, and a one-shot job (a restore, an import tail) ended failed + * with nobody told at deploy time. The worker now loads every handler once in + * the background after binding and reports each failure by queue name. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { Job, PgBoss } from "pg-boss"; + +const bound = vi.hoisted( + () => new Map Promise>(), +); + +vi.mock("@/lib/jobs/reminder/registrar-shared", () => ({ + createAndSchedule: vi.fn(async () => undefined), + retireQueues: vi.fn(async () => undefined), + createAndWork: vi.fn( + async ( + _boss: unknown, + queue: string, + _opts: unknown, + handler: (jobs: unknown[]) => Promise, + ) => { + bound.set(queue, handler); + }, + ), +})); + +vi.mock("@/lib/jobs/reminder/shared", () => ({ workerLog: vi.fn() })); + +vi.mock("@/lib/logging/signal", () => ({ emitSignal: vi.fn() })); +vi.mock("@/lib/jobs/report-worker-error", () => ({ + reportWorkerError: vi.fn(async () => undefined), +})); +vi.mock("@/lib/jobs/job-failures", () => ({ + readConsecutiveRunFailures: vi.fn(async () => 0), +})); + +import { workerLog } from "@/lib/jobs/reminder/shared"; +import { emitSignal } from "@/lib/logging/signal"; +import { reportWorkerError } from "@/lib/jobs/report-worker-error"; +import { reportHandlerLoadFailure } from "@/lib/jobs/run-job"; + +import { readSource } from "@/__tests__/helpers/source-files"; + +import type { JobDefinition, JobGroup } from "../define-job"; +import { + preloadHandlers, + registerJobGroup, + type BoundLoader, +} from "../register"; + +const boss = {} as PgBoss; + +function definition( + queue: string, + load: () => Promise, +): JobDefinition { + return { queue, work: { batchSize: 1 }, load } as unknown as JobDefinition; +} + +beforeEach(() => { + bound.clear(); + vi.clearAllMocks(); +}); + +describe("preloading the job handlers at worker boot", () => { + it("reports a module that fails to load, by queue, and loads the rest", async () => { + const ok = vi.fn(async () => async () => ({ ok: true as const })); + const broken = vi.fn(async () => { + throw new Error("Cannot find module './gone'"); + }); + const group = { + name: "maintenance", + jobs: [ + definition("first-ok", ok), + definition("broken-module", broken), + definition("second-ok", ok), + ], + } as unknown as JobGroup; + + const loaders: BoundLoader[] = []; + await registerJobGroup(boss, group, (b) => loaders.push(b)); + expect(loaders.map((b) => b.queue)).toEqual([ + "first-ok", + "broken-module", + "second-ok", + ]); + + const report = vi.fn(async () => undefined); + const failed = await preloadHandlers(loaders, report); + + expect(failed).toEqual(["broken-module"]); + expect(report).toHaveBeenCalledTimes(1); + expect(report).toHaveBeenCalledWith( + "broken-module", + expect.objectContaining({ message: "Cannot find module './gone'" }), + ); + expect(ok).toHaveBeenCalledTimes(2); + expect(workerLog).toHaveBeenCalledWith( + "error", + expect.stringContaining("failed to load: broken-module"), + ); + }); + + it("preloads through the loader the queue is bound to, so the first job does not import again", async () => { + const handler = vi.fn(async () => ({ ok: true as const })); + const load = vi.fn(async () => handler); + const group = { + name: "maintenance", + jobs: [definition("warm", load)], + } as unknown as JobGroup; + + const loaders: BoundLoader[] = []; + await registerJobGroup(boss, group, (b) => loaders.push(b)); + await preloadHandlers(loaders, vi.fn()); + await bound.get("warm")!([{ id: "j1", data: {} } as Job]); + + expect(load).toHaveBeenCalledTimes(1); + expect(handler).toHaveBeenCalledTimes(1); + }); + + it("does not stop the sweep when a report itself fails", async () => { + const loaders: BoundLoader[] = [ + { queue: "a", load: () => Promise.reject(new Error("a")) }, + { queue: "b", load: () => Promise.reject(new Error("b")) }, + ]; + const report = vi.fn(async () => { + throw new Error("sink down"); + }); + await expect(preloadHandlers(loaders, report)).resolves.toEqual(["a", "b"]); + expect(report).toHaveBeenCalledTimes(2); + }); + + it("reports a load failure on the job.run.failed line, at error, naming the queue", async () => { + const error = new Error("SyntaxError in handler"); + await reportHandlerLoadFailure("broken-module", error); + + expect(emitSignal).toHaveBeenCalledWith({ + action: "job.run.failed", + level: "error", + meta: { + queue: "broken-module", + reason: "handler_load_failed", + consecutiveFailures: 0, + }, + error, + }); + expect(reportWorkerError).toHaveBeenCalledWith("broken-module", error, { + phase: "handler_preload", + }); + }); + + it("is started by the worker after binding, without holding up the boot", () => { + const source = readSource("src/lib/jobs/reminder-worker.ts", [ + "registry.registerAllJobGroups(", + "void registry.preloadHandlers(bound)", + ]); + const register = source.indexOf("registry.registerAllJobGroups("); + const preload = source.indexOf("void registry.preloadHandlers(bound)"); + expect(register).toBeGreaterThan(-1); + expect(preload).toBeGreaterThan(register); + }); +}); diff --git a/src/lib/jobs/registry/index.ts b/src/lib/jobs/registry/index.ts index a6d02a764..6ab42fe0c 100644 --- a/src/lib/jobs/registry/index.ts +++ b/src/lib/jobs/registry/index.ts @@ -10,7 +10,13 @@ import type { JobDefinition, JobGroup } from "./define-job"; import { INTEGRATION_SYNC_GROUP } from "./integration-sync"; import { MAINTENANCE_GROUP } from "./maintenance"; import { REMINDERS_GROUP } from "./reminders"; -import { registerJobGroup, runBootDiscovery } from "./register"; +import { + registerJobGroup, + runBootDiscovery, + type BoundLoader, +} from "./register"; + +export { preloadHandlers, type BoundLoader } from "./register"; import { ROLLUP_GROUP } from "./rollup"; import { STATUS_GROUP } from "./status"; @@ -29,16 +35,18 @@ export function allJobDefinitions(): readonly JobDefinition[] { /** * Register every group in order. `onGroup` runs before each one, so the boot - * can time the groups as steps. Returns every queue name created. + * can time the groups as steps; `onBound` receives every queue's handler + * loader as it is bound. Returns every queue name created. */ export async function registerAllJobGroups( boss: PgBoss, onGroup: (group: JobGroup) => void = () => {}, + onBound: (bound: BoundLoader) => void = () => {}, ): Promise { const queues: string[] = []; for (const group of JOB_GROUPS) { onGroup(group); - queues.push(...(await registerJobGroup(boss, group))); + queues.push(...(await registerJobGroup(boss, group, onBound))); } return queues; } diff --git a/src/lib/jobs/registry/register.ts b/src/lib/jobs/registry/register.ts index 3a2a36bef..f53205381 100644 --- a/src/lib/jobs/registry/register.ts +++ b/src/lib/jobs/registry/register.ts @@ -13,6 +13,14 @@ * queue's protections do not depend on whether its module has loaded yet. A * module that fails to load fails the job that asked for it, with the queue's * name on the error, and the next job tries the import again. + * + * Loading lazily moved a broken module from a failed boot to a failed job: the + * worker comes up healthy, and the queue burns its retries the first time it + * has work, which for a one-shot job (a restore, an import tail) means the + * job ends failed with nobody told at deploy time. `preloadHandlers` closes + * that gap. The worker runs it once after binding, in the background, through + * the same memoised loader the queue uses, and reports every module that does + * not load before any job needs it. */ import type { Job, PgBoss } from "pg-boss"; @@ -26,6 +34,7 @@ import { type ScheduleEntry, } from "@/lib/jobs/reminder/registrar-shared"; import { workerLog } from "@/lib/jobs/reminder/shared"; +import { reportHandlerLoadFailure } from "@/lib/jobs/run-job"; import type { BatchHandler, @@ -35,6 +44,34 @@ import type { JobGroup, } from "./define-job"; +/** A queue's handler loader, memoised: one import, retried after a failure. */ +export type HandlerLoader = () => Promise>>; + +/** One bound queue and the loader its handler goes through. */ +export interface BoundLoader { + readonly queue: string; + readonly load: HandlerLoader; +} + +/** + * The loader a definition's binding shares: the first call imports, every + * later call answers from the same promise, and a failed import is forgotten + * so the next call tries again. + */ +export function handlerLoader( + definition: JobDefinition, + ctx: JobContext, +): HandlerLoader { + let loading: Promise>> | null = null; + return () => { + loading ??= definition.load(ctx).catch((err: unknown) => { + loading = null; + throw err; + }); + return loading; + }; +} + /** * The handler a definition binds: its loaded module behind the lock and the * lane it declares. @@ -42,15 +79,8 @@ import type { export function bindableHandler( definition: JobDefinition, ctx: JobContext, + load: HandlerLoader = handlerLoader(definition, ctx), ): BatchHandler> { - let loading: Promise>> | null = null; - const load = (): Promise>> => { - loading ??= definition.load(ctx).catch((err: unknown) => { - loading = null; - throw err; - }); - return loading; - }; let handler: BatchHandler> = async (jobs) => (await load())(jobs); if (definition.lane) { @@ -88,11 +118,13 @@ export function policiesOf(jobs: readonly JobDefinition[]): QueuePolicyTable { /** * Register one group: create, schedule, retire, bind. Returns the queue names - * it created. + * it created. `onBound` is handed each queue's loader as it is bound, so the + * boot can preload through the very loader the queue will use. */ export async function registerJobGroup( boss: PgBoss, group: JobGroup, + onBound: (bound: BoundLoader) => void = () => {}, ): Promise { const queues = group.jobs.map((job) => job.queue); await createAndSchedule( @@ -104,7 +136,14 @@ export async function registerJobGroup( if (group.retired) await retireQueues(boss, group.retired); const ctx: JobContext = { boss }; for (const job of group.jobs) { - await createAndWork(boss, job.queue, job.work, bindableHandler(job, ctx)); + const load = handlerLoader(job, ctx); + await createAndWork( + boss, + job.queue, + job.work, + bindableHandler(job, ctx, load), + ); + onBound({ queue: job.queue, load }); } if (group.afterBind) await group.afterBind(ctx); return queues; @@ -144,3 +183,45 @@ export async function runBootDiscovery( } } } + +/** + * Load every bound handler once, in declaration order, and report each one + * that fails: `job.run.failed` at error with the queue's name and the reason + * `handler_load_failed`, the line the repeated-failure alert already keys on, + * plus the worker error report (stderr and GlitchTip). Never throws, and the + * caller does not wait for it, so a broken module costs the boot nothing but + * the report. A module that loads stays loaded, and the queue's first job + * finds it in the loader's memo. Returns the queues that failed. + */ +export async function preloadHandlers( + bound: readonly BoundLoader[], + report: ( + queue: string, + error: unknown, + ) => Promise = reportHandlerLoadFailure, +): Promise { + const failed: string[] = []; + for (const { queue, load } of bound) { + try { + await load(); + } catch (error) { + failed.push(queue); + try { + await report(queue, error); + } catch (reportError) { + // The report failing must not stop the sweep; say so on stderr. + workerLog( + "error", + `[registry] reporting the load failure of ${queue} failed`, + reportError, + ); + } + } + } + workerLog( + failed.length === 0 ? "info" : "error", + `[registry] preloaded ${bound.length - failed.length}/${bound.length} job handlers` + + (failed.length === 0 ? "" : `; failed to load: ${failed.join(", ")}`), + ); + return failed; +} diff --git a/src/lib/jobs/reminder-worker.ts b/src/lib/jobs/reminder-worker.ts index c882ca002..715ce80b2 100644 --- a/src/lib/jobs/reminder-worker.ts +++ b/src/lib/jobs/reminder-worker.ts @@ -17,6 +17,7 @@ import { reportJobsCutOffAtBoot } from "@/lib/jobs/job-observer"; import { beginBootStep, reportBootSteps } from "@/lib/boot/boot-steps"; import { assertSubsystemEnabled } from "@/lib/process-type"; import { DATABASE_URL, workerLog } from "./reminder/shared"; +import type { BoundLoader } from "./registry/index"; /** * Give the event loop a turn around a module load, so requests and the @@ -76,8 +77,11 @@ export async function startReminderWorker() { "registry", () => import("./registry/index"), ); - const registeredQueues = await registry.registerAllJobGroups(boss, (group) => - beginBootStep(`register_${group.name}`), + const bound: BoundLoader[] = []; + const registeredQueues = await registry.registerAllJobGroups( + boss, + (group) => beginBootStep(`register_${group.name}`), + (loader) => bound.push(loader), ); workerLog( "info", @@ -100,6 +104,11 @@ export async function startReminderWorker() { await registry.runAllBootDiscovery(); await reportBootSteps({ boot_queue_count: registeredQueues.length }); + + // Every handler module is loaded once now, in the background, so one that + // cannot load is reported at deploy time rather than on its queue's first + // job. Not awaited: readiness does not wait on it, and it never throws. + void registry.preloadHandlers(bound); return boss; } diff --git a/src/lib/jobs/run-job.ts b/src/lib/jobs/run-job.ts index f4df20089..6a7b74716 100644 --- a/src/lib/jobs/run-job.ts +++ b/src/lib/jobs/run-job.ts @@ -90,6 +90,20 @@ async function signalRunFailed( }); } +/** + * A queue's handler module that failed to load before any job asked for it + * (the worker's boot preload, `preloadHandlers`). Reported the way a failed + * run is, so the operator hears about it at deploy time on the line the + * repeated-failure alert already reads, instead of on the queue's first job. + */ +export async function reportHandlerLoadFailure( + queue: string, + error: unknown, +): Promise { + await reportWorkerError(queue, error, { phase: "handler_preload" }); + await signalRunFailed(queue, "handler_load_failed", error); +} + /** * Wrap a handler for `boss.work`. Resolves with the bounded, redacted * persistence representation on `ok: true`; reports and rethrows on diff --git a/src/lib/logging/background-tasks.ts b/src/lib/logging/background-tasks.ts index 5516ef44e..7b1dee275 100644 --- a/src/lib/logging/background-tasks.ts +++ b/src/lib/logging/background-tasks.ts @@ -56,9 +56,14 @@ const inFlight = new Set>(); export function trackBackgroundTask(settled: Promise): void { if (!TRACK_IN_FLIGHT) return; - inFlight.add(settled); - void settled.finally(() => { - inFlight.delete(settled); + // A test double that stands in for the background work often returns + // nothing at all. Filed as is, `undefined` would never leave the set and + // `settleBackgroundTasks` would spin on it forever at full CPU. A real + // promise passes through `Promise.resolve` as the same object. + const handle = Promise.resolve(settled); + inFlight.add(handle); + void handle.finally(() => { + inFlight.delete(handle); }); } diff --git a/src/lib/openapi/routes/settings.ts b/src/lib/openapi/routes/settings.ts index 5b1ac7c2b..2a269fcb5 100644 --- a/src/lib/openapi/routes/settings.ts +++ b/src/lib/openapi/routes/settings.ts @@ -27,10 +27,7 @@ import { MODULE_LIST_VIEWS, MODULE_SORT_DIRS, } from "@/lib/settings/module-list-layout"; -import { - moduleListLayoutPutSchema, - previewFeaturesPatchSchema, -} from "@/lib/validations/settings-prefs"; +import { moduleListLayoutPutSchema } from "@/lib/validations/settings-prefs"; const moduleListPrefsEntry = z .object({ @@ -72,20 +69,6 @@ const moduleListLayoutSchema = z "Per-user presentation of the labs, illness and checkups lists. Display-only; a client that does not render a list ignores it.", }); -const previewFeaturesSchema = z - .object({ - topics: z - .boolean() - .describe( - "Topic overview pages above the single-metric Insights pages. Off by default.", - ), - }) - .meta({ - id: "PreviewFeatures", - description: - "Per-user opt-in to preview features, one boolean per flag, every flag off by default.", - }); - // v1.16.11 — the one threshold read every dose-status consumer makes // (cards, table, take-all-due derivation). `lateMinutes` / // `missedMinutes` are the operator-level singleton; the low-stock @@ -970,57 +953,6 @@ export const settingsPaths: NonNullable = { }, }, }, - "/api/settings/preview-features": { - get: { - tags: ["Meta"], - summary: "Get the preview-feature opt-ins", - description: "Every flag, off unless the caller switched it on.", - responses: { - "200": { - description: "The opt-ins.", - content: { - "application/json": { - schema: dataEnvelope( - previewFeaturesSchema, - "PreviewFeaturesEnvelope", - ), - }, - }, - }, - ...stdResponses, - }, - }, - patch: { - tags: ["Meta"], - summary: "Switch preview features on or off", - description: - "Sets the flags the body names and keeps the rest. An unknown flag or an empty body is refused with 422.", - requestBody: { - required: true, - content: { - "application/json": { - schema: previewFeaturesPatchSchema.meta({ - id: "PreviewFeaturesPatchBody", - }), - }, - }, - }, - responses: { - "200": { - description: "The opt-ins as stored.", - content: { - "application/json": { - schema: dataEnvelope( - previewFeaturesSchema, - "PreviewFeaturesSavedEnvelope", - ), - }, - }, - }, - ...stdResponses, - }, - }, - }, "/api/settings/privacy-summary": { get: { tags: ["Meta"], diff --git a/src/lib/settings/__tests__/preview-features.test.ts b/src/lib/settings/__tests__/preview-features.test.ts deleted file mode 100644 index df5251b6d..000000000 --- a/src/lib/settings/__tests__/preview-features.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { - DEFAULT_PREVIEW_FEATURES, - resolvePreviewFeatures, -} from "../preview-features"; - -describe("resolvePreviewFeatures", () => { - it("is all off by default", () => { - expect(DEFAULT_PREVIEW_FEATURES).toEqual({ topics: false }); - expect(resolvePreviewFeatures(null)).toEqual({ topics: false }); - }); - - it("reads a flag only on an explicit true", () => { - expect(resolvePreviewFeatures({ topics: true })).toEqual({ topics: true }); - expect(resolvePreviewFeatures({ topics: "true" })).toEqual({ - topics: false, - }); - }); - - it("drops unknown keys", () => { - expect(resolvePreviewFeatures({ topics: true, retired: true })).toEqual({ - topics: true, - }); - }); -}); diff --git a/src/lib/settings/preview-features.ts b/src/lib/settings/preview-features.ts deleted file mode 100644 index 514b55fd1..000000000 --- a/src/lib/settings/preview-features.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Per-user opt-in to preview features, persisted in `User.previewFeaturesJson`. - * - * One boolean per flag, every flag off by default. A preview feature is - * something finished enough to try and not yet the default; switching one on - * adds it beside what is there and never replaces an existing page. - * - * Flags: - * - `topics`: topic overview pages above the single-metric Insights pages. - * Stored, served by `/api/settings/preview-features` and carried by the - * backup. The Settings > System switch is not listed until the topic pages - * ship, because nothing reads the flag yet. - * - * Unknown keys in a stored blob are dropped on read, so a flag retired later - * disappears cleanly and a malformed blob reads as everything off. - */ - -export const PREVIEW_FEATURE_KEYS = ["topics"] as const; -export type PreviewFeatureKey = (typeof PREVIEW_FEATURE_KEYS)[number]; - -export type PreviewFeatures = Record; - -export const DEFAULT_PREVIEW_FEATURES: PreviewFeatures = { topics: false }; - -export function resolvePreviewFeatures(raw: unknown): PreviewFeatures { - const out: PreviewFeatures = { ...DEFAULT_PREVIEW_FEATURES }; - if (raw === null || raw === undefined || typeof raw !== "object") return out; - const blob = raw as Record; - for (const key of PREVIEW_FEATURE_KEYS) { - if (blob[key] === true) out[key] = true; - } - return out; -} diff --git a/src/lib/settings/section-slugs.ts b/src/lib/settings/section-slugs.ts index b6f69ed40..843203284 100644 --- a/src/lib/settings/section-slugs.ts +++ b/src/lib/settings/section-slugs.ts @@ -114,10 +114,6 @@ export const SETTINGS_SECTION_SLUGS = [ // pieces into one coherent privacy pane. Sits next to `advanced` (the // destructive-action home) and before `about`. "privacy", - // v1.44 — "Preview features" (per-user opt-ins, Settings > System) is not - // listed until the first preview it switches, the topic pages, ships: a - // switch that changes nothing is a promise the page cannot keep. The stored - // flag, `/api/settings/preview-features` and its backup round-trip stay. "about", ] as const; diff --git a/src/lib/sync/changes/read-rows.ts b/src/lib/sync/changes/read-rows.ts index b23595c79..544d36161 100644 --- a/src/lib/sync/changes/read-rows.ts +++ b/src/lib/sync/changes/read-rows.ts @@ -28,12 +28,6 @@ const MEASUREMENT_SELECT = { ingestToken: APPLE_HEALTH_MIRROR_SELECT.ingestToken, } satisfies Prisma.MeasurementSelect; -const MEASUREMENT_MIRROR_SELECT = { - id: true, - source: true, - ingestToken: { select: { mirrorToAppleHealth: true, mirrorChangedAt: true } }, -} satisfies Prisma.MeasurementSelect; - const MOOD_SELECT = { id: true, date: true, @@ -81,8 +75,15 @@ export type CycleDayRow = Prisma.CycleDayLogGetPayload<{ include: typeof dayLogSymptomInclude; }>; export type CycleRow = Prisma.MenstrualCycleGetPayload; +/** A reading the mirror walk re-delivers, with its token's current setting. */ export type MeasurementMirrorRow = Prisma.MeasurementGetPayload<{ - select: typeof MEASUREMENT_MIRROR_SELECT; + select: { + id: true; + source: true; + ingestToken: { + select: { mirrorToAppleHealth: true; mirrorChangedAt: true }; + }; + }; }>; /** @@ -109,41 +110,94 @@ export interface DomainRows { measurementMirror: MeasurementMirrorRow[]; } +const MEASUREMENT_MIRROR_ROW_SELECT = { + id: true, + source: true, +} satisfies Prisma.MeasurementSelect; + /** * The live readings of every token whose Apple Health setting changed past * the watermark, in `(token.mirrorChangedAt, measurement.id)` order. Switching * the setting writes the token row only, so the readings keep their * `updatedAt` (nothing keyed on it recomputes) and arrive through this walk * instead of the measurement one. + * + * The walk starts from the tokens, not the readings. Ordering readings by a + * column of the joined token row cannot use any index on `measurements`, so + * the single-query form sorted every row of every token that had ever been + * switched, on every poll, even when nothing had changed since the client's + * watermark. Reading the switched tokens first is a lookup on a handful of + * rows; nearly every poll finds none and ends there. Otherwise the readings + * are walked one change stamp at a time, by `(ingest_token_id, id)`, which is + * the order the index already holds. Tokens that share a stamp are walked + * together so their rows interleave by id exactly as the one-query form + * ordered them. + * + * The token lookup filters on the token's owner. That is exact because a + * reading only ever links to a token of its own account: the link is written + * for a narrow `measurements:write` Bearer, which never acts on another record + * (`requireRecordAuth` refuses the carrier), and a restore keeps the link only + * for a token the restoring account holds. */ -function readMeasurementMirrorRows( +async function readMeasurementMirrorRows( userId: string, wm: DomainWatermark | undefined, limit: number, ): Promise { const at = wm ? new Date(wm.updatedAtMs) : null; - return prisma.measurement.findMany({ + const tokens = await prisma.apiToken.findMany({ where: { userId, - deletedAt: null, - ingestToken: { mirrorChangedAt: { not: null } }, - ...(at && - wm && { - OR: [ - { ingestToken: { mirrorChangedAt: { gt: at } } }, - { - AND: [ - { ingestToken: { mirrorChangedAt: at } }, - { id: { gt: wm.id } }, - ], - }, - ], - }), + mirrorChangedAt: at ? { gte: at } : { not: null }, }, - orderBy: [{ ingestToken: { mirrorChangedAt: "asc" } }, { id: "asc" }], - take: limit + 1, - select: MEASUREMENT_MIRROR_SELECT, + orderBy: [{ mirrorChangedAt: "asc" }, { id: "asc" }], + select: { id: true, mirrorToAppleHealth: true, mirrorChangedAt: true }, }); + if (tokens.length === 0) return []; + + // One group per change stamp, in stamp order. + const groups: { stampMs: number; tokens: typeof tokens }[] = []; + for (const token of tokens) { + const stampMs = token.mirrorChangedAt!.getTime(); + const last = groups[groups.length - 1]; + if (last && last.stampMs === stampMs) last.tokens.push(token); + else groups.push({ stampMs, tokens: [token] }); + } + + const out: MeasurementMirrorRow[] = []; + for (const group of groups) { + const want = limit + 1 - out.length; + if (want <= 0) break; + // The watermark's own stamp resumes after the last row delivered; every + // later stamp starts from its first row. + const resume = wm && group.stampMs === wm.updatedAtMs ? wm.id : null; + const byId = new Map(group.tokens.map((t) => [t.id, t])); + const rows = await prisma.measurement.findMany({ + where: { + ingestTokenId: + group.tokens.length === 1 + ? group.tokens[0].id + : { in: group.tokens.map((t) => t.id) }, + userId, + deletedAt: null, + ...(resume !== null && { id: { gt: resume } }), + }, + orderBy: { id: "asc" }, + take: want, + select: { ...MEASUREMENT_MIRROR_ROW_SELECT, ingestTokenId: true }, + }); + for (const { ingestTokenId, ...row } of rows) { + const token = byId.get(ingestTokenId!)!; + out.push({ + ...row, + ingestToken: { + mirrorToAppleHealth: token.mirrorToAppleHealth, + mirrorChangedAt: token.mirrorChangedAt, + }, + }); + } + } + return out; } /** diff --git a/src/lib/targets/build-response.ts b/src/lib/targets/build-response.ts index 1c28f8775..b236b894a 100644 --- a/src/lib/targets/build-response.ts +++ b/src/lib/targets/build-response.ts @@ -11,6 +11,8 @@ * are read in the user's own calendar. */ import { prisma } from "@/lib/db"; +import { runDetached } from "@/lib/logging/detached"; +import { trackBackgroundTask } from "@/lib/logging/background-tasks"; import { annotate } from "@/lib/logging/context"; import { COURSES_COMPLIANCE_SELECT, @@ -102,7 +104,7 @@ export async function buildTargetsResponse(user: AuthedUser) { queryNow.getTime() - 5 * 365 * 24 * 60 * 60 * 1000, ); - void ensureUserMoodRollupsFresh(userId); + trackBackgroundTask(runDetached(() => ensureUserMoodRollupsFresh(userId))); // All independent reads stay in one bounded fan-out. The medication event // read is the sole dependent query and remains chained to its medication diff --git a/src/lib/validations/settings-prefs.ts b/src/lib/validations/settings-prefs.ts index c3cc82112..26ab7745f 100644 --- a/src/lib/validations/settings-prefs.ts +++ b/src/lib/validations/settings-prefs.ts @@ -1,8 +1,8 @@ /** * Request schemas for the per-user Settings preferences that are not part of * the account payload: the list presentation of the labs, illness and - * checkups lists, and the preview-feature opt-ins. Shared by the routes and - * the OpenAPI registry so the wire contract has one source. + * checkups lists. Shared by the route and the OpenAPI registry so the wire + * contract has one source. */ import { z } from "zod/v4"; @@ -26,12 +26,3 @@ export const moduleListLayoutPutSchema = z sortDir: z.enum(MODULE_SORT_DIRS).optional(), }) .strict(); - -export const previewFeaturesPatchSchema = z - .object({ - topics: z.boolean().optional(), - }) - .strict() - .refine((body) => Object.keys(body).length > 0, { - message: "Name at least one preview feature.", - }); diff --git a/tests/durations/unit.json b/tests/durations/unit.json index 58db3c2d0..f2898cc5f 100644 --- a/tests/durations/unit.json +++ b/tests/durations/unit.json @@ -2660,7 +2660,6 @@ "src/lib/search/__tests__/match.test.ts": 0, "src/lib/search/__tests__/search-records.test.ts": 0, "src/lib/settings/__tests__/module-list-layout.test.ts": 0, - "src/lib/settings/__tests__/preview-features.test.ts": 0, "src/lib/sharing/__tests__/account-access-parse.test.ts": 0, "src/lib/sharing/__tests__/account-access.test.ts": 0, "src/lib/sharing/__tests__/account-label.test.ts": 0, diff --git a/tests/integration/__golden__/backup-parity/export-disaster-recovery.json b/tests/integration/__golden__/backup-parity/export-disaster-recovery.json index a2a4bf6b5..5825e765a 100644 --- a/tests/integration/__golden__/backup-parity/export-disaster-recovery.json +++ b/tests/integration/__golden__/backup-parity/export-disaster-recovery.json @@ -124,9 +124,6 @@ "moduleListLayoutJson": { "roundTrip": "rt-User-moduleListLayoutJson" }, - "previewFeaturesJson": { - "roundTrip": "rt-User-previewFeaturesJson" - }, "activityGoalsJson": { "stepsPerMonth": 250000, "workoutsPerWeek": 4 @@ -2577,9 +2574,6 @@ "moduleListLayoutJson": { "roundTrip": "rt-User-moduleListLayoutJson" }, - "previewFeaturesJson": { - "roundTrip": "rt-User-previewFeaturesJson" - }, "activityGoalsJson": { "stepsPerMonth": 250000, "workoutsPerWeek": 4 diff --git a/tests/integration/__golden__/backup-parity/export-portable.json b/tests/integration/__golden__/backup-parity/export-portable.json index 473808185..532cfbb76 100644 --- a/tests/integration/__golden__/backup-parity/export-portable.json +++ b/tests/integration/__golden__/backup-parity/export-portable.json @@ -63,9 +63,6 @@ "moduleListLayoutJson": { "roundTrip": "rt-User-moduleListLayoutJson" }, - "previewFeaturesJson": { - "roundTrip": "rt-User-previewFeaturesJson" - }, "activityGoalsJson": { "stepsPerMonth": 250000, "workoutsPerWeek": 4 @@ -1735,9 +1732,6 @@ "moduleListLayoutJson": { "roundTrip": "rt-User-moduleListLayoutJson" }, - "previewFeaturesJson": { - "roundTrip": "rt-User-previewFeaturesJson" - }, "activityGoalsJson": { "stepsPerMonth": 250000, "workoutsPerWeek": 4 diff --git a/tests/integration/__golden__/backup-parity/restore-disaster-recovery-over-live.json b/tests/integration/__golden__/backup-parity/restore-disaster-recovery-over-live.json index 43417f7d6..4dca595b4 100644 --- a/tests/integration/__golden__/backup-parity/restore-disaster-recovery-over-live.json +++ b/tests/integration/__golden__/backup-parity/restore-disaster-recovery-over-live.json @@ -335,7 +335,7 @@ "symptomDefinitions": 2, "lifeEvents": 3 }, - "accountSettingsApplied": 62, + "accountSettingsApplied": 61, "skipped": { "catalogueKeys": [], "links": 0 diff --git a/tests/integration/__golden__/backup-parity/restore-disaster-recovery.json b/tests/integration/__golden__/backup-parity/restore-disaster-recovery.json index 66db5ca0f..ff5e45346 100644 --- a/tests/integration/__golden__/backup-parity/restore-disaster-recovery.json +++ b/tests/integration/__golden__/backup-parity/restore-disaster-recovery.json @@ -335,7 +335,7 @@ "symptomDefinitions": 0, "lifeEvents": 0 }, - "accountSettingsApplied": 62, + "accountSettingsApplied": 61, "skipped": { "catalogueKeys": [], "links": 0 @@ -2773,9 +2773,6 @@ "moduleListLayoutJson": { "roundTrip": "rt-User-moduleListLayoutJson" }, - "previewFeaturesJson": { - "roundTrip": "rt-User-previewFeaturesJson" - }, "activityGoalsJson": { "stepsPerMonth": 250000, "workoutsPerWeek": 4 @@ -3043,9 +3040,6 @@ "moduleListLayoutJson": { "roundTrip": "rt-User-moduleListLayoutJson" }, - "previewFeaturesJson": { - "roundTrip": "rt-User-previewFeaturesJson" - }, "activityGoalsJson": { "stepsPerMonth": 250000, "workoutsPerWeek": 4 diff --git a/tests/integration/__golden__/backup-parity/restore-portable.json b/tests/integration/__golden__/backup-parity/restore-portable.json index 8b313cb53..b73ea56a6 100644 --- a/tests/integration/__golden__/backup-parity/restore-portable.json +++ b/tests/integration/__golden__/backup-parity/restore-portable.json @@ -381,7 +381,7 @@ "symptomDefinitions": 0, "lifeEvents": 0 }, - "accountSettingsApplied": 62, + "accountSettingsApplied": 61, "skipped": { "catalogueKeys": [ { @@ -2267,9 +2267,6 @@ "moduleListLayoutJson": { "roundTrip": "rt-User-moduleListLayoutJson" }, - "previewFeaturesJson": { - "roundTrip": "rt-User-previewFeaturesJson" - }, "activityGoalsJson": { "stepsPerMonth": 250000, "workoutsPerWeek": 4 @@ -2476,9 +2473,6 @@ "moduleListLayoutJson": { "roundTrip": "rt-User-moduleListLayoutJson" }, - "previewFeaturesJson": { - "roundTrip": "rt-User-previewFeaturesJson" - }, "activityGoalsJson": { "stepsPerMonth": 250000, "workoutsPerWeek": 4 diff --git a/tests/integration/__golden__/backup-parity/restore-skips.json b/tests/integration/__golden__/backup-parity/restore-skips.json index 20f298931..901ab0a0f 100644 --- a/tests/integration/__golden__/backup-parity/restore-skips.json +++ b/tests/integration/__golden__/backup-parity/restore-skips.json @@ -898,7 +898,6 @@ "documentsLayoutJson": null, "moodTagLayoutJson": null, "moduleListLayoutJson": null, - "previewFeaturesJson": null, "activityGoalsJson": null, "glucoseUnit": null, "hasDiabetes": false, diff --git a/tests/integration/__golden__/backup-parity/restore-version-1.json b/tests/integration/__golden__/backup-parity/restore-version-1.json index cc4f8dc5a..bc26e8895 100644 --- a/tests/integration/__golden__/backup-parity/restore-version-1.json +++ b/tests/integration/__golden__/backup-parity/restore-version-1.json @@ -1031,7 +1031,6 @@ "documentsLayoutJson": null, "moodTagLayoutJson": null, "moduleListLayoutJson": null, - "previewFeaturesJson": null, "activityGoalsJson": null, "glucoseUnit": null, "hasDiabetes": false, @@ -1251,7 +1250,6 @@ "moodTagLayoutJson": null, "documentsLayoutJson": null, "moduleListLayoutJson": null, - "previewFeaturesJson": null, "activityGoalsJson": null, "reportSelectionJson": null, "globalExcludedInjectionSites": [], diff --git a/tests/integration/__golden__/backup-parity/seeded-account.json b/tests/integration/__golden__/backup-parity/seeded-account.json index 065d219de..1e9a3bc58 100644 --- a/tests/integration/__golden__/backup-parity/seeded-account.json +++ b/tests/integration/__golden__/backup-parity/seeded-account.json @@ -2386,9 +2386,6 @@ "moduleListLayoutJson": { "roundTrip": "rt-User-moduleListLayoutJson" }, - "previewFeaturesJson": { - "roundTrip": "rt-User-previewFeaturesJson" - }, "activityGoalsJson": { "stepsPerMonth": 250000, "workoutsPerWeek": 4 diff --git a/tests/integration/__golden__/status-parity/batch.json b/tests/integration/__golden__/status-parity/batch.json index 9f6cf622e..2083b80b5 100644 --- a/tests/integration/__golden__/status-parity/batch.json +++ b/tests/integration/__golden__/status-parity/batch.json @@ -151,6 +151,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -260,6 +261,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -611,6 +613,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -727,6 +730,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -844,6 +848,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -955,6 +960,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -1074,6 +1080,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -1189,6 +1196,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -1301,6 +1309,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -1412,6 +1421,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", diff --git a/tests/integration/__golden__/status-parity/biomarker.json b/tests/integration/__golden__/status-parity/biomarker.json index 9a7f5299a..21df561a7 100644 --- a/tests/integration/__golden__/status-parity/biomarker.json +++ b/tests/integration/__golden__/status-parity/biomarker.json @@ -147,6 +147,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -434,6 +435,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -847,6 +849,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -1043,6 +1046,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", diff --git a/tests/integration/__golden__/status-parity/blood-pressure.json b/tests/integration/__golden__/status-parity/blood-pressure.json index 537fe2b70..0ec8173b4 100644 --- a/tests/integration/__golden__/status-parity/blood-pressure.json +++ b/tests/integration/__golden__/status-parity/blood-pressure.json @@ -25,6 +25,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -231,6 +232,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -431,6 +433,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -670,6 +673,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -1091,6 +1095,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -1239,6 +1244,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -1452,6 +1458,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", diff --git a/tests/integration/__golden__/status-parity/bmi.json b/tests/integration/__golden__/status-parity/bmi.json index 84f41a1fd..47c214e9f 100644 --- a/tests/integration/__golden__/status-parity/bmi.json +++ b/tests/integration/__golden__/status-parity/bmi.json @@ -155,6 +155,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -385,6 +386,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -741,6 +743,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -945,6 +948,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", diff --git a/tests/integration/__golden__/status-parity/general.json b/tests/integration/__golden__/status-parity/general.json index e290135c3..9909db6ae 100644 --- a/tests/integration/__golden__/status-parity/general.json +++ b/tests/integration/__golden__/status-parity/general.json @@ -25,6 +25,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -228,6 +229,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -418,6 +420,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -647,6 +650,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -1058,6 +1062,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -1196,6 +1201,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -1399,6 +1405,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", diff --git a/tests/integration/__golden__/status-parity/medication-compliance.json b/tests/integration/__golden__/status-parity/medication-compliance.json index ee8a8edd3..cba27c192 100644 --- a/tests/integration/__golden__/status-parity/medication-compliance.json +++ b/tests/integration/__golden__/status-parity/medication-compliance.json @@ -191,6 +191,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -486,6 +487,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -895,6 +897,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -1052,6 +1055,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -1294,6 +1298,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", diff --git a/tests/integration/__golden__/status-parity/metric.json b/tests/integration/__golden__/status-parity/metric.json index fed29fd08..145d93a52 100644 --- a/tests/integration/__golden__/status-parity/metric.json +++ b/tests/integration/__golden__/status-parity/metric.json @@ -264,6 +264,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -654,6 +655,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -1200,6 +1202,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -1395,6 +1398,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -1760,6 +1764,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -2125,6 +2130,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -2371,6 +2377,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -2721,6 +2728,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -2876,6 +2884,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -3096,6 +3105,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -3617,6 +3627,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -4175,6 +4186,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -4941,6 +4953,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -5200,6 +5213,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -5733,6 +5747,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", diff --git a/tests/integration/__golden__/status-parity/mood.json b/tests/integration/__golden__/status-parity/mood.json index cc25c6355..26e2c36b8 100644 --- a/tests/integration/__golden__/status-parity/mood.json +++ b/tests/integration/__golden__/status-parity/mood.json @@ -25,6 +25,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -228,6 +229,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -418,6 +420,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -647,6 +650,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -1058,6 +1062,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -1196,6 +1201,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -1399,6 +1405,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", diff --git a/tests/integration/__golden__/status-parity/pulse.json b/tests/integration/__golden__/status-parity/pulse.json index a7a371fbf..ff8534ab4 100644 --- a/tests/integration/__golden__/status-parity/pulse.json +++ b/tests/integration/__golden__/status-parity/pulse.json @@ -25,6 +25,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -227,6 +228,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -416,6 +418,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -644,6 +647,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -1054,6 +1058,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -1191,6 +1196,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -1393,6 +1399,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", diff --git a/tests/integration/__golden__/status-parity/weight.json b/tests/integration/__golden__/status-parity/weight.json index 5ad00c1be..92333687b 100644 --- a/tests/integration/__golden__/status-parity/weight.json +++ b/tests/integration/__golden__/status-parity/weight.json @@ -25,6 +25,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -231,6 +232,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -431,6 +433,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -670,6 +673,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", @@ -1113,6 +1117,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "Du schreibst eine kurze, persönliche Einschätzung zu EINER Gesundheitsmetrik dieses Nutzers. Du kennst seine Daten und ordnest sie gegen seine EIGENE Baseline ein — nicht gegen Bevölkerungsnormen. Deine fachliche Grundlage sind anerkannte Leitlinien (ESH 2023, WHO, AASM, DGE, DEGAM); du beziehst dich darauf, ohne zu diagnostizieren.", "", @@ -1326,6 +1331,7 @@ ], "responseFormat": "json", "seed": 1618033, + "signal": {}, "system": [ "You are writing a short, personal assessment of ONE of this user's health metrics. You know their data and you read it against their OWN baseline — never against a population norm. Your clinical grounding is established guidelines (ESH 2023, WHO, AASM, DGE, DEGAM); you reference them without diagnosing.", "", diff --git a/tests/integration/ai-extraction-capability.test.ts b/tests/integration/ai-extraction-capability.test.ts index 6d57b5c2d..263eb2c48 100644 --- a/tests/integration/ai-extraction-capability.test.ts +++ b/tests/integration/ai-extraction-capability.test.ts @@ -105,6 +105,7 @@ vi.mock("@/lib/ai/provider-runner", async (importOriginal) => ({ }, workingProvider: { providerType: "anthropic" }, fallbackHops: [], + tokensSpent: 0, })), })); diff --git a/tests/integration/coach-chat.test.ts b/tests/integration/coach-chat.test.ts index 30af73e8d..00c6cb523 100644 --- a/tests/integration/coach-chat.test.ts +++ b/tests/integration/coach-chat.test.ts @@ -181,6 +181,7 @@ describe("POST /api/insights/chat — integration", () => { }, workingProvider: { providerType: "openai", instance: {} }, fallbackHops: [], + tokensSpent: 0, }); const { POST } = await import("@/app/api/insights/chat/route"); @@ -331,6 +332,7 @@ describe("POST /api/insights/chat — integration", () => { }, workingProvider: { providerType: "openai", instance: {} }, fallbackHops: [], + tokensSpent: 0, }); const { POST } = await import("@/app/api/insights/chat/route"); @@ -399,6 +401,7 @@ describe("POST /api/insights/chat — integration", () => { }, workingProvider: { providerType: "openai", instance: {} }, fallbackHops: [], + tokensSpent: 0, }); const { POST } = await import("@/app/api/insights/chat/route"); @@ -509,6 +512,7 @@ describe("POST /api/insights/chat — integration", () => { }, workingProvider: { providerType: "openai", instance: {} }, fallbackHops: [], + tokensSpent: 0, }); const { POST } = await import("@/app/api/insights/chat/route"); diff --git a/tests/integration/codex-credential-park.test.ts b/tests/integration/codex-credential-park.test.ts index e4d283eb4..1eb6c4c8c 100644 --- a/tests/integration/codex-credential-park.test.ts +++ b/tests/integration/codex-credential-park.test.ts @@ -262,6 +262,113 @@ describe("a refused Codex sign-in", () => { }); }); +/** + * #1279 — a fresh sign-in also closes the status notes' negative windows, + * as a save of the person's provider settings and of the operator's admin AI + * settings do. A window grows with the failures in a row (up to two hours), + * so without it a fixed connection still had every note wait out the window + * the refused one earned. + */ +describe("a fresh Codex sign-in and the notes' windows", () => { + async function openWindow(userId: string) { + await prisma.insightStatusCache.create({ + data: { + userId, + metric: "weight", + locale: "en", + dateKey: "2026-06-15", + retryAt: new Date(Date.now() + HOUR_MS), + negativeReason: "error", + failureStreak: 4, + }, + }); + } + + async function windowOf(userId: string) { + return prisma.insightStatusCache.findFirstOrThrow({ + where: { userId }, + select: { retryAt: true, failureStreak: true }, + }); + } + + function signedIn() { + vi.mocked(pollDeviceCode).mockResolvedValue({ + status: "connected", + creds: creds("new", 10 * HOUR_MS), + } as Awaited>); + } + + it("closes the person's windows on their own sign-in", async () => { + const user = await seedCodexUser(); + await openWindow(user.id); + signedIn(); + cookieJar.set( + "codex_device", + encrypt(JSON.stringify({ deviceAuthId: "device", userCode: "CODE" })), + ); + const { POST } = await import("@/app/api/auth/codex/device-poll/route"); + const res = await POST( + new NextRequest("http://localhost/api/auth/codex/device-poll", { + method: "POST", + }), + ); + expect(res.status).toBe(200); + expect(await windowOf(user.id)).toEqual({ + retryAt: null, + failureStreak: 0, + }); + }); + + it("closes every account's windows on the operator's central sign-in", async () => { + const admin = await prisma.user.create({ + data: { + username: "central-admin", + email: "central-admin@example.test", + role: "ADMIN", + }, + }); + const session = await prisma.session.create({ + data: { userId: admin.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + const member = await prisma.user.create({ + data: { + username: "central-member", + email: "central-member@example.test", + useCentralCodex: true, + }, + }); + await openWindow(member.id); + await postgresProviderHealthLedger.recordFailure( + member.id, + "admin-codex", + 401, + ); + signedIn(); + cookieJar.set( + "central_codex_device", + encrypt(JSON.stringify({ deviceAuthId: "device", userCode: "CODE" })), + ); + const { POST } = + await import("@/app/api/admin/central-codex/device-poll/route"); + const res = await POST( + new NextRequest("http://localhost/api/admin/central-codex/device-poll", { + method: "POST", + }), + ); + expect(res.status).toBe(200); + expect(await windowOf(member.id)).toEqual({ + retryAt: null, + failureStreak: 0, + }); + expect( + (await postgresProviderHealthLedger.getSkipHints(member.id)).has( + "admin-codex", + ), + ).toBe(false); + }); +}); + describe("two refreshes of one Codex token", () => { async function codexStatus(userId: string) { return ( diff --git a/tests/integration/query-budget-hot-routes.test.ts b/tests/integration/query-budget-hot-routes.test.ts index cb0390f26..0a50202d6 100644 --- a/tests/integration/query-budget-hot-routes.test.ts +++ b/tests/integration/query-budget-hot-routes.test.ts @@ -281,9 +281,15 @@ async function seed(): Promise { }), }); + // Both rollup tiers are minted here, the way a running account has them. + // Seeded rows without their rollups make the first read start a full + // recompute in the background, and that is a warm-up cost, not the read's. const { recomputeUserRollups } = await import("@/lib/rollups/measurement-rollups"); await recomputeUserRollups(USER_ID); + const { recomputeUserMoodRollups } = + await import("@/lib/rollups/mood-rollups"); + await recomputeUserMoodRollups(USER_ID); const session = await db.session.create({ data: { userId: USER_ID, expiresAt: new Date(Date.now() + 3_600_000) }, diff --git a/tests/integration/routes/parity/insights-generate.parity.test.ts b/tests/integration/routes/parity/insights-generate.parity.test.ts index 81d01d215..0cdb9d831 100644 --- a/tests/integration/routes/parity/insights-generate.parity.test.ts +++ b/tests/integration/routes/parity/insights-generate.parity.test.ts @@ -171,6 +171,7 @@ vi.mock("@/lib/ai/provider-runner", async (importOriginal) => { }, workingProvider: args.providers[0], fallbackHops: [], + tokensSpent: 0, }; }, }; diff --git a/tests/integration/settings-prefs-routes.test.ts b/tests/integration/settings-prefs-routes.test.ts index 61aaa5a00..2a18cdf04 100644 --- a/tests/integration/settings-prefs-routes.test.ts +++ b/tests/integration/settings-prefs-routes.test.ts @@ -1,13 +1,10 @@ /** - * The two per-user Settings preferences that live outside the account - * payload, end to end against Postgres: - * - * - `/api/settings/module-list-layout`: defaults without a write, one list - * changed per PUT with preserve-when-absent fields and untouched neighbours, - * the multi-issue 422 for unknown values, and the column carrying only what - * was chosen. - * - `/api/settings/preview-features`: all off by default, PATCH keeps the - * flags it does not name, an unknown flag or an empty body is refused. + * The per-user list layout, a Settings preference that lives outside the + * account payload, end to end against Postgres: + * `/api/settings/module-list-layout` answers defaults without a write, + * changes one list per PUT with preserve-when-absent fields and untouched + * neighbours, refuses unknown values with the multi-issue 422, and stores + * only what was chosen. */ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -173,51 +170,8 @@ describe("/api/settings/module-list-layout — integration", () => { }); }); -describe("/api/settings/preview-features — integration", () => { - const URL = "http://localhost/api/settings/preview-features"; - - it("is all off by default and keeps a flag once switched on", async () => { - const { userId } = await seedUser(); - const { GET, PATCH } = - await import("@/app/api/settings/preview-features/route"); - - const before = await (GET as Handler)(new Request(URL)); - expect(((await before.json()) as { data: unknown }).data).toEqual({ - topics: false, - }); - - const on = await (PATCH as Handler)(json("PATCH", URL, { topics: true })); - expect(on.status).toBe(200); - expect(((await on.json()) as { data: unknown }).data).toEqual({ - topics: true, - }); - - const after = await (GET as Handler)(new Request(URL)); - expect(((await after.json()) as { data: unknown }).data).toEqual({ - topics: true, - }); - const row = await getPrismaClient().user.findUnique({ - where: { id: userId }, - select: { previewFeaturesJson: true }, - }); - expect(row?.previewFeaturesJson).toEqual({ topics: true }); - }); - - it("refuses an unknown flag and an empty body", async () => { - await seedUser(); - const { PATCH } = await import("@/app/api/settings/preview-features/route"); - const unknown = await (PATCH as Handler)( - json("PATCH", URL, { topics: true, darkMagic: true }), - ); - expect(unknown.status).toBe(422); - const empty = await (PATCH as Handler)(json("PATCH", URL, {})); - expect(empty.status).toBe(422); - }); -}); - describe("settings presentation follows the person, also while acting for a record", () => { const LAYOUT = "http://localhost/api/settings/module-list-layout"; - const PREVIEW = "http://localhost/api/settings/preview-features"; async function actingForAnotherRecord() { const prisma = getPrismaClient(); @@ -251,10 +205,9 @@ describe("settings presentation follows the person, also while acting for a reco return { ownerId: owner.id, actorId: actor.id }; } - it("reads and saves the actor's own layout and preview switch", async () => { + it("reads and saves the actor's own layout", async () => { const { ownerId, actorId } = await actingForAnotherRecord(); const layout = await import("@/app/api/settings/module-list-layout/route"); - const preview = await import("@/app/api/settings/preview-features/route"); expect((await (layout.GET as Handler)(new Request(LAYOUT))).status).toBe( 200, @@ -263,20 +216,12 @@ describe("settings presentation follows the person, also while acting for a reco json("PUT", LAYOUT, { version: 1, list: "labs", view: "cards" }), ); expect(put.status).toBe(200); - expect((await (preview.GET as Handler)(new Request(PREVIEW))).status).toBe( - 200, - ); - const patch = await (preview.PATCH as Handler)( - json("PATCH", PREVIEW, { topics: true }), - ); - expect(patch.status).toBe(200); const rows = await getPrismaClient().user.findMany({ where: { id: { in: [ownerId, actorId] } }, select: { id: true, moduleListLayoutJson: true, - previewFeaturesJson: true, }, }); const actor = rows.find((r) => r.id === actorId)!; @@ -284,9 +229,7 @@ describe("settings presentation follows the person, also while acting for a reco expect(actor.moduleListLayoutJson).toMatchObject({ labs: { view: "cards" }, }); - expect(actor.previewFeaturesJson).toEqual({ topics: true }); expect(owner.moduleListLayoutJson).toBeNull(); - expect(owner.previewFeaturesJson).toBeNull(); }); }); diff --git a/tests/integration/settings-round-trip.test.ts b/tests/integration/settings-round-trip.test.ts index 8460aa4da..0045ba1a8 100644 --- a/tests/integration/settings-round-trip.test.ts +++ b/tests/integration/settings-round-trip.test.ts @@ -69,7 +69,6 @@ import * as medicationLayout from "@/app/api/medications/layout/route"; import * as moodTagLayout from "@/app/api/mood/tags/layout/route"; import * as documentsLayout from "@/app/api/documents/inbound/layout/route"; import * as moduleListLayout from "@/app/api/settings/module-list-layout/route"; -import * as previewFeatures from "@/app/api/settings/preview-features/route"; import * as activityGoals from "@/app/api/activity/goals/route"; import * as healthKit from "@/app/api/integrations/healthkit/route"; import { POST as exportHealthRecord } from "@/app/api/export/health-record/route"; @@ -888,21 +887,6 @@ function entries(user: Pick): Entry[] { }, expected: ["list", "alphaAsc"], }, - { - columns: ["previewFeaturesJson"], - write: { - handler: previewFeatures.PATCH, - method: "PATCH", - route: "/api/settings/preview-features", - body: { topics: true }, - }, - read: get( - previewFeatures.GET, - "/api/settings/preview-features", - "topics", - ), - expected: true, - }, { columns: ["activityGoalsJson"], write: { diff --git a/tests/integration/sync-mirror-walk.test.ts b/tests/integration/sync-mirror-walk.test.ts new file mode 100644 index 000000000..51a01ce3e --- /dev/null +++ b/tests/integration/sync-mirror-walk.test.ts @@ -0,0 +1,215 @@ +/** + * The `measurementMirror` walk of `/api/sync/changes`, against Postgres. + * + * The walk reads the switched tokens first and then each change stamp's + * readings by `(ingest_token_id, id)`. The page sequence a client sees must be + * exactly the one the earlier single query produced, which ordered every + * token-linked reading by `(token.mirrorChangedAt, measurement.id)` and + * resumed past `(stamp, id)`. That query is kept below as the reference, and + * every page of every walk is compared against it: tied stamps across tokens, + * a watermark inside a tie, soft-deleted rows, another account's token, a + * token never switched, readings with no token, and page sizes from one row + * up to the whole set. + */ +import { beforeEach, describe, expect, it } from "vitest"; + +import { readDomainRows } from "@/lib/sync/changes/read-rows"; +import type { SyncCursor } from "@/lib/sync/cursor"; + +import { makeMeasurement, makeToken, makeUser } from "../kit/factories"; +import { countStatements } from "./query-counter"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +const prisma = getPrismaClient(); + +interface Wm { + updatedAtMs: number; + id: string; +} + +/** The single-query walk this replaced, verbatim in its semantics. */ +async function referenceWalk( + userId: string, + wm: Wm | undefined, + limit: number, +) { + const at = wm ? new Date(wm.updatedAtMs) : null; + return prisma.measurement.findMany({ + where: { + userId, + deletedAt: null, + ingestToken: { mirrorChangedAt: { not: null } }, + ...(at && + wm && { + OR: [ + { ingestToken: { mirrorChangedAt: { gt: at } } }, + { + AND: [ + { ingestToken: { mirrorChangedAt: at } }, + { id: { gt: wm.id } }, + ], + }, + ], + }), + }, + orderBy: [{ ingestToken: { mirrorChangedAt: "asc" } }, { id: "asc" }], + take: limit + 1, + select: { + id: true, + source: true, + ingestToken: { + select: { mirrorToAppleHealth: true, mirrorChangedAt: true }, + }, + }, + }); +} + +async function walk(userId: string, wm: Wm | undefined, limit: number) { + const cursor = { v: 2, measurementMirror: wm } as unknown as SyncCursor; + const rows = await readDomainRows(userId, cursor, limit, false); + return rows.measurementMirror; +} + +/** Page through both walks from `start`, asserting each page is identical. */ +async function pageThrough(userId: string, limit: number, start?: Wm) { + let wm = start; + const seen: string[] = []; + for (let guard = 0; guard < 200; guard += 1) { + const [actual, expected] = await Promise.all([ + walk(userId, wm, limit), + referenceWalk(userId, wm, limit), + ]); + expect(actual, `limit ${limit}, page ${guard}`).toEqual(expected); + const page = actual.slice(0, limit); + seen.push(...page.map((r) => r.id)); + if (actual.length <= limit) return { seen, wm }; + const last = page[page.length - 1]; + wm = { + updatedAtMs: last.ingestToken!.mirrorChangedAt!.getTime(), + id: last.id, + }; + } + throw new Error("walk did not terminate"); +} + +let userId: string; +let otherId: string; +const T0 = new Date("2026-10-01T08:00:00.000Z"); +const T1 = new Date("2026-10-02T08:00:00.000Z"); +const T2 = new Date("2026-10-03T08:00:00.000Z"); + +async function rowsFor( + owner: { id: string }, + tokenId: string | null, + n: number, + base: number, + deleted: number[] = [], +) { + for (let i = 0; i < n; i += 1) { + await makeMeasurement(owner, { + type: "WEIGHT", + unit: "kg", + value: 80 + i / 10, + source: "EXTERNAL", + measuredAt: new Date(Date.UTC(2026, 8, 1, 0, base + i)), + ingestTokenId: tokenId, + deletedAt: deleted.includes(i) ? new Date() : null, + }); + } +} + +async function stamp(tokenId: string, at: Date, on = true) { + await prisma.apiToken.update({ + where: { id: tokenId }, + data: { mirrorChangedAt: at, mirrorToAppleHealth: on }, + }); +} + +beforeEach(async () => { + await truncateAllTables(prisma); + const user = await makeUser({ timezone: "UTC" }); + const other = await makeUser({ timezone: "UTC" }); + userId = user.id; + otherId = other.id; + const [a, b, c, d, never, othersOwn] = await Promise.all([ + makeToken(user, { name: "A" }), + makeToken(user, { name: "B" }), + makeToken(user, { name: "C" }), + makeToken(user, { name: "D" }), + makeToken(user, { name: "never" }), + makeToken(other, { name: "others-own" }), + ]); + await rowsFor(user, a.tokenId, 7, 0, [3]); + await rowsFor(user, b.tokenId, 5, 100, [0]); + await rowsFor(user, c.tokenId, 3, 200); + await rowsFor(user, never.tokenId, 3, 300); + await rowsFor(user, d.tokenId, 2, 400); + await rowsFor(other, othersOwn.tokenId, 3, 600); + await rowsFor(user, null, 4, 700); + // A and B share a stamp, so their rows interleave by id. + await stamp(a.tokenId, T0); + await stamp(b.tokenId, T0, false); + await stamp(c.tokenId, T1); + await stamp(d.tokenId, T2); + await stamp(othersOwn.tokenId, T1); +}); + +describe("the measurementMirror walk", () => { + it.each([1, 2, 3, 5, 8, 100])( + "pages exactly as the single query did, %i rows a page", + async (limit) => { + const { seen } = await pageThrough(userId, limit); + // 6 + 4 live rows on the tied stamp, 3 on C, 2 on D. Nothing from the + // never-switched token, nothing of the other account's, nothing + // deleted, nothing without a token. + expect(seen).toHaveLength(15); + expect(new Set(seen).size).toBe(15); + }, + ); + + it("resumes inside a tied stamp, and re-delivers a token switched again", async () => { + const all = await walk(userId, undefined, 100); + const tied = all.filter( + (r) => r.ingestToken!.mirrorChangedAt!.getTime() === T0.getTime(), + ); + // Both tied tokens' rows interleave by id, so a watermark three rows into + // the tie leaves rows of both still to come. + await pageThrough(userId, 2, { updatedAtMs: T0.getTime(), id: tied[2].id }); + + const last = all[all.length - 1]; + const caughtUp: Wm = { + updatedAtMs: last.ingestToken!.mirrorChangedAt!.getTime(), + id: last.id, + }; + expect(await walk(userId, caughtUp, 100)).toEqual([]); + + const c = await prisma.apiToken.findFirstOrThrow({ + where: { userId, name: "C" }, + }); + await stamp(c.id, new Date("2026-10-04T08:00:00.000Z"), false); + const { seen } = await pageThrough(userId, 2, caughtUp); + expect(seen).toHaveLength(3); + }); + + it("answers a poll with nothing switched past its watermark from the token table alone", async () => { + const wm = { updatedAtMs: T2.getTime() + 1, id: "" }; + const measured = await countStatements(() => walk(userId, wm, 100)); + expect(measured.result).toEqual([]); + // The walk's reading-side shape: id, source and the token link. + const readingWalks = measured.statements.filter((s) => + s.startsWith( + 'SELECT "public"."measurements"."id", "public"."measurements"."source"::text, "public"."measurements"."ingest_token_id"', + ), + ); + expect(readingWalks).toEqual([]); + const tokenLookups = measured.statements.filter((s) => + s.includes('"public"."api_tokens"."mirror_changed_at" FROM'), + ); + expect(tokenLookups).toHaveLength(1); + }); + + it("leaves the other account's walk to the other account", async () => { + const { seen } = await pageThrough(otherId, 2); + expect(seen).toHaveLength(3); + }); +});