Merge release/v1.39.0 into feat/ai-routes-insights-coach

Brings in the jobs, stored-text and consent package and the extraction
routes. Conflicts resolved:

- The digest keeps its capability-driven reaction line and check-in,
  now through the shared `aiCapabilityForRecord`; the local copy of it
  is removed.
- The route inventories classify the extraction routes (documents,
  labs OCR, medication text, fenced chat and attachments) as actions;
  a capability call counts as the gate for the module inventory.
- With every caller moved, `requireAssistantSurface` and
  `AssistantDisabledError` are deleted. The `assistant.disabled.*`
  codes stay on the capability refusal, and the guard now asserts the
  retired gate is gone.
- `aiCapabilityToServe` answers whether stored model text may be shown
  from the record's own state, apart from who is reading.

Refs #975

src/__tests__/ai-consent-enforcement-guard.test.ts

@@ -64,14 +64,14 @@ const RESOLUTION_HELPERS = [
   "resolveProviderForTest",
 ] as const;
 
-/** The two sanctioned consent APIs. Either satisfies the guard. */
+/** The sanctioned consent APIs in the consent guard. Any satisfies the guard. */
 const CONSENT_HELPERS = [
   // Chain-shaped: throw-form and predicate-form.
   "assertConsentForChain",
   "chainRequiresServerManagedConsent",
   "hasActiveConsentForSurface",
-  // Document-class, pick-shaped.
-  "assertDocumentEgressConsent",
+  // Document-class, pick-shaped (the receipt itself is read by the capability
+  // egress re-check below).
   "isExternalDocumentEgress",
 ] as const;
 
@@ -108,9 +108,23 @@ function resolvesAProvider(source: string): boolean {
   return RESOLUTION_HELPERS.some((h) => imported.includes(h));
 }
 
+/**
+ * The capability re-check at the wire. It answers the consent question for the
+ * providers a caller is about to send to, from the receipt kinds the capability
+ * table declares, so it is a consent helper in its own right.
+ */
+const CAPABILITY_EGRESS_HELPERS = [
+  "assertAiEgress",
+  "aiEgressRefusal",
+] as const;
+
 function importsAConsentHelper(source: string): boolean {
   const imported = importedNamesFrom(source, "@/lib/ai/consent-guard");
-  return CONSENT_HELPERS.some((h) => imported.includes(h));
+  const egress = importedNamesFrom(source, "@/lib/ai/capabilities/egress");
+  return (
+    CONSENT_HELPERS.some((h) => imported.includes(h)) ||
+    CAPABILITY_EGRESS_HELPERS.some((h) => egress.includes(h))
+  );
 }
 
 /**
@@ -125,8 +139,6 @@ function importsAConsentHelper(source: string): boolean {
  */
 const ALLOWLIST: Record<string, string> = {
   // --- Capability / config probes: resolve a chain, never call it. ---
-  "lib/labs/ocr-capability.ts":
-    "Resolves the chain to REPORT which entry could do vision/text OCR (mode, reason, pdfSupported). Makes no completion call. Every consumer that then egresses a document is gated at its own call site with assertDocumentEgressConsent.",
   "app/api/insights/provider-chain/route.ts":
     "Lists the user's configured chain and which entry is active, for the settings UI. Constructs clients but never calls generateCompletion. No health data in the request or the response.",
 

src/__tests__/ai-egress-capability-guard.test.ts

@@ -0,0 +1,199 @@
+/**
+ * Structural guard on the places a model is actually called.
+ *
+ * A capability is only as good as the last check before the wire. A job
+ * enqueued before an operator flipped a switch, or before the person withdrew
+ * consent or turned AI analysis off, must still stop there. So every file that
+ * hands a prompt to a provider (`.generateCompletion(`, the chain runners) is
+ * named here with the reason its call is admitted, and the ones that ARE a
+ * chokepoint must re-check a capability in the same file.
+ *
+ * Mirrors `ai-consent-enforcement-guard.test.ts` in shape and in its limits: it
+ * proves a reference exists, not that it sits on every branch or before the
+ * call. The integration suite with a provider spy
+ * (`tests/integration/ai-optional-jobs.test.ts`) is what proves behaviour.
+ * A future egress helper under a new name would slip the matcher; a reviewer
+ * adding one should add it here.
+ */
+import { describe, expect, it } from "vitest";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import type { runStatusCompletion } from "@/lib/insights/status-provider";
+
+import { walkSourceFiles } from "./helpers/source-files";
+
+const SRC = join(process.cwd(), "src");
+
+/** A provider call: a client's completion (plain or streamed), or a chain runner. */
+const EGRESS_CALL =
+  /\.generateCompletion(?:Stream)?\s*\(|\brunRawCompletionWithFallback\s*\(|\brunWithFallback\s*\(/;
+
+/**
+ * A capability read in the same file: one of the gates, or the wire re-check
+ * (`aiEgressRefusal` / `assertAiEgress`), which also answers the consent a
+ * picked provider needs.
+ */
+const CAPABILITY_CHECK =
+  /\b(?:aiCapabilityForJob|aiCapabilityForRecord|requireAiCapability|getAiCapability|aiEgressRefusal|assertAiEgress)\s*\(/;
+
+/** Comments name the helpers too; only code counts. */
+function stripComments(source: string): string {
+  return source
+    .replace(/\/\*[\s\S]*?\*\//g, "")
+    .replace(/(^|[^:])\/\/.*$/gm, "$1");
+}
+
+function sourceFiles(): string[] {
+  return walkSourceFiles(SRC, { floor: 3000 })
+    .filter((p) => !p.startsWith("generated/"))
+    .filter((p) => !p.includes("__tests__"))
+    .filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx"))
+    .sort();
+}
+
+function code(rel: string): string {
+  return stripComments(readFileSync(join(SRC, rel), "utf8"));
+}
+
+interface EgressSite {
+  /** True when this file is a chokepoint and must re-check a capability. */
+  recheck: boolean;
+  reason: string;
+}
+
+/**
+ * Every file that calls a provider, and why the call is admitted.
+ */
+const EGRESS_SITES: Record<string, EgressSite> = {
+  // ── Chokepoints: re-check at the wire ──────────────────────────────────
+  "lib/insights/status-provider.ts": {
+    recheck: true,
+    reason:
+      "runStatusCompletion: status notes, narratives, workout paragraphs, derived and biomarker assessments, Coach memory. Takes the capability as a required argument and re-checks it before the chain is resolved.",
+  },
+  "lib/jobs/reaction-line.ts": {
+    recheck: true,
+    reason:
+      "Reaction lines call the first chain entry directly; the worker resolves `reactionLines` before the digest and re-checks the chain at the wire (`aiEgressRefusal`).",
+  },
+  "lib/jobs/coach-nudge-ai.ts": {
+    recheck: true,
+    reason:
+      "AI-composed nudges call the first chain entry directly; the composer re-checks `coach` before the chain.",
+  },
+
+  // ── Admitted by the one caller that reaches them ───────────────────────
+  "lib/insights/briefing-provider.ts": {
+    recheck: false,
+    reason:
+      "Reached only through generateComprehensiveInsight, which re-checks `briefing` before the chain is resolved.",
+  },
+  "lib/ai/coach/tools/loop.ts": {
+    recheck: false,
+    reason:
+      "The Coach tool loop runs inside the chat stream, which the chat route gates on `coach`.",
+  },
+  "lib/ai/coach/self-context-questions.ts": {
+    recheck: false,
+    reason:
+      "About-me follow-up questions; reached from the about-me route, which gates `aboutMeQuestions`.",
+  },
+  "lib/documents/assist.ts": {
+    recheck: false,
+    reason:
+      "Document suggest/chat helpers; reached from the document routes and jobs, which resolve `documentAi` before a provider is picked.",
+  },
+  "lib/documents/describe.ts": {
+    recheck: false,
+    reason:
+      "Document summary; reached from the summary route and the summary job, which resolve `documentAi` first.",
+  },
+  "lib/documents/extract.ts": {
+    recheck: false,
+    reason:
+      "Document extraction/indexing; reached from the document routes and the index jobs, which resolve `documentAi` first.",
+  },
+  "lib/labs/ocr-extract.ts": {
+    recheck: false,
+    reason:
+      "Lab report scan; reached from the labs OCR route, which gates `labsOcr`.",
+  },
+  "app/api/medications/extract/route.ts": {
+    recheck: false,
+    reason:
+      "Medication text extraction; the route is the chokepoint for `medicationExtract`.",
+  },
+  "app/api/ai/test/route.ts": {
+    recheck: false,
+    reason:
+      "The connection test a person runs on a provider they are configuring; it sends a fixed probe prompt, no health data.",
+  },
+
+  // ── The machinery itself ────────────────────────────────────────────────
+  "lib/ai/provider-runner.ts": {
+    recheck: false,
+    reason: "The chain runners; every caller above is what admits a run.",
+  },
+  "lib/ai/openai-client.ts": {
+    recheck: false,
+    reason: "A wire client delegating to its own completion method.",
+  },
+  "lib/ai/local-client.ts": {
+    recheck: false,
+    reason: "A wire client delegating to its own completion method.",
+  },
+  "lib/ai/coach/eval/judge.ts": {
+    recheck: false,
+    reason: "The offline evaluation harness; never on a request or job path.",
+  },
+};
+
+describe("AI egress call sites", () => {
+  const found = sourceFiles().filter((rel) => EGRESS_CALL.test(code(rel)));
+
+  it("finds provider calls at all (an empty matcher is a failure, not a pass)", () => {
+    expect(found.length).toBeGreaterThan(5);
+  });
+
+  it("every provider call site is named, with a reason", () => {
+    const unlisted = found.filter((rel) => !(rel in EGRESS_SITES));
+    expect(
+      unlisted,
+      "A new file calls a provider. Add it to EGRESS_SITES with the reason its call is admitted, and re-check a capability in it if it is a chokepoint.",
+    ).toEqual([]);
+  });
+
+  it("the list carries no file that no longer calls a provider", () => {
+    const stale = Object.keys(EGRESS_SITES).filter(
+      (rel) => !found.includes(rel),
+    );
+    expect(stale).toEqual([]);
+  });
+
+  it("every chokepoint re-checks a capability in the same file", () => {
+    const missing = Object.entries(EGRESS_SITES)
+      .filter(([, site]) => site.recheck)
+      .filter(([rel]) => !CAPABILITY_CHECK.test(code(rel)))
+      .map(([rel]) => rel);
+    expect(missing).toEqual([]);
+  });
+});
+
+describe("the status chokepoint's capability argument", () => {
+  it("is required by the type (the @ts-expect-error below fails typecheck if it is not)", () => {
+    type Args = Parameters<typeof runStatusCompletion>[0];
+    // @ts-expect-error — `capability` is required: a caller that omits it does not compile.
+    const withoutCapability: Args = {
+      userId: "u",
+      cacheAction: "insights.weight-status.en",
+      systemPrompt: "s",
+      userPrompt: "p",
+    };
+    const withCapability: Args = {
+      ...withoutCapability,
+      capability: "statusText",
+    };
+    expect(withCapability.capability).toBe("statusText");
+  });
+});

src/__tests__/dense-measurement-read-guard.test.ts

@@ -0,0 +1,175 @@
+/**
+ * Structural guard: background and insight readers must not materialise a
+ * dense measurement stream.
+ *
+ * #1023: a single account syncing heart rate from a watch took the worker
+ * down within half an hour. The status-card read was bounded in time
+ * (`measuredAt >= now − 91 days`) but not in rows, and a stream sampled every
+ * minute or faster turns 91 days into six or seven figures of objects. The
+ * read also ran into the statement timeout. The cure was to fold per day in
+ * SQL (`readDayAggregates`, `readLiveBuckets`), so the reader's size tracks
+ * the number of days, not the sampling rate.
+ *
+ * What this guard pins, in the directories that run on the worker or feed an
+ * insight: every `measurement.findMany` without a `take` reads a type that is
+ * named as a literal and is not one a device streams. A read whose type is a
+ * variable could be handed a dense type by the next caller, so it must be
+ * listed below with the reason it is safe. The list is frozen both ways: a new
+ * unlisted read fails, and an entry whose read is gone fails too.
+ *
+ * It is a tripwire, not a proof: the matcher reads source text, and a read
+ * written some other way (a raw SELECT of rows, a helper in another
+ * directory) passes it. The floor below fails the guard if the matcher stops
+ * finding reads at all.
+ */
+import { describe, expect, it } from "vitest";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { walkSourceFiles } from "./helpers/source-files";
+import {
+  CUMULATIVE_HK_TYPES,
+  HIGH_FREQUENCY_MEAN_TYPES,
+} from "@/lib/measurements/apple-health-mapping";
+
+const SRC = join(process.cwd(), "src");
+
+const SCOPED_DIRS = [
+  "lib/insights/",
+  "lib/jobs/",
+  "lib/rollups/",
+  "lib/ai/",
+  "lib/analytics/",
+  "lib/dashboard/",
+  "lib/daily/",
+];
+
+/** Types a device writes at sampling rate rather than a few times a day. */
+const DENSE_TYPES: ReadonlySet<string> = new Set([
+  "PULSE",
+  "BLOOD_GLUCOSE",
+  "HEART_RATE_VARIABILITY",
+  "OXYGEN_SATURATION",
+  ...CUMULATIVE_HK_TYPES,
+  ...HIGH_FREQUENCY_MEAN_TYPES,
+]);
+
+/**
+ * Untaken reads whose type is not a sparse literal, keyed
+ * `<file>::<type expression>`, with why each stays bounded.
+ */
+const ALLOWED: Record<string, string> = {
+  "lib/insights/comprehensive-generate.ts::(shorthand)":
+    'Only reached for SLEEP_DURATION (guarded by `type === "SLEEP_DURATION"`); a few stage rows per night.',
+  "lib/insights/derived/vascular-age.ts::VASCULAR_AGE_TYPE":
+    "A constant for a type written at most a few times a day.",
+  "lib/insights/derived/fitness-age.ts::VO2_MAX_TYPE":
+    "A constant for VO2 max, written at most daily.",
+  "lib/insights/derived/six-minute-walk.ts::SIX_MINUTE_WALK_TYPE":
+    "A constant for six-minute-walk distance, written at most daily.",
+  "lib/insights/derived/wellness-scores.ts::measurementType":
+    "Daily computed / provider scores (recovery, strain, readiness), one row per day.",
+  "lib/jobs/reaction-line.ts::{ in: [...types]":
+    "Exact-instant match (`measuredAt: row.occurredAt`), not a window.",
+  "lib/jobs/step-consolidation-repair.ts::STEP_TYPE":
+    "Repair job over one user's step rows for a bounded day range; runs once per repair, not per status refresh.",
+  'lib/ai/coach/snapshot.ts::"BLOOD_GLUCOSE" as never':
+    "Clinical CGM metrics (time in range, SD / CV, J-index, LBGI / HBGI, reading span) are functions of the individual readings, which a per-day aggregate cannot reproduce. 30 days at a sensor's fixed rate (at most one a minute) stays below 45 000 rows.",
+  'lib/dashboard/snapshot.ts::"BLOOD_GLUCOSE"':
+    "Same 30-day clinical glucose panel as the coach snapshot (per-reading metrics), grouped by meal context.",
+  "lib/analytics/score/reader.ts::{ in: types":
+    "Health-score inputs: steps (drained to one row per day nightly), sleep, waist, weight, blood pressure, fasting glucose only.",
+};
+
+interface UntakenRead {
+  file: string;
+  line: number;
+  typeExpr: string;
+}
+
+function scopedFiles(): string[] {
+  return walkSourceFiles(SRC, { floor: 3000 })
+    .filter((p) => SCOPED_DIRS.some((d) => p.startsWith(d)))
+    .filter((p) => !p.includes("__tests__"))
+    .filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx"))
+    .sort();
+}
+
+function untakenReads(): { all: number; untaken: UntakenRead[] } {
+  let all = 0;
+  const untaken: UntakenRead[] = [];
+  for (const file of scopedFiles()) {
+    const src = readFileSync(join(SRC, file), "utf8");
+    // Whitespace-tolerant: a call split across lines still matches.
+    const re = /\.measurement\s*\.\s*findMany\s*\(/g;
+    for (let m = re.exec(src); m; m = re.exec(src)) {
+      all += 1;
+      let i = m.index + m[0].length;
+      let depth = 1;
+      while (depth > 0 && i < src.length) {
+        if (src[i] === "(") depth += 1;
+        else if (src[i] === ")") depth -= 1;
+        i += 1;
+      }
+      const call = src.slice(m.index, i);
+      if (/\btake\b/.test(call)) continue;
+      const typed = /\btype\s*:\s*([^,\n}]+)/.exec(call);
+      const typeExpr = typed
+        ? typed[1].trim()
+        : /\btype\s*,/.test(call)
+          ? "(shorthand)"
+          : "(none)";
+      untaken.push({
+        file,
+        line: src.slice(0, m.index).split("\n").length,
+        typeExpr,
+      });
+    }
+  }
+  return { all, untaken };
+}
+
+function sparseLiteral(typeExpr: string): boolean {
+  const lit = /^"([A-Z_0-9]+)"/.exec(typeExpr);
+  return lit !== null && !DENSE_TYPES.has(lit[1]);
+}
+
+describe("dense measurement reads stay bounded (#1023)", () => {
+  const { all, untaken } = untakenReads();
+
+  it("finds the reads it is meant to police", () => {
+    // Guard against a matcher that silently matches nothing.
+    expect(all).toBeGreaterThanOrEqual(40);
+    expect(untaken.length).toBeGreaterThanOrEqual(15);
+  });
+
+  it("every untaken read names a sparse literal type or is listed with its reason", () => {
+    const offenders = untaken
+      .filter((r) => !sparseLiteral(r.typeExpr))
+      .filter((r) => !(`${r.file}::${r.typeExpr}` in ALLOWED))
+      .map((r) => `${r.file}:${r.line} type ${r.typeExpr}`);
+    expect(offenders).toEqual([]);
+  });
+
+  it("every listed exception still exists", () => {
+    const present = new Set(untaken.map((r) => `${r.file}::${r.typeExpr}`));
+    expect(Object.keys(ALLOWED).filter((k) => !present.has(k))).toEqual([]);
+  });
+
+  it("the readers fixed for #1023 fold in SQL rather than reading rows", () => {
+    for (const file of [
+      "lib/insights/graded-series.ts",
+      "lib/rollups/tiered-context.ts",
+      "lib/jobs/coach-plan-review.ts",
+      "lib/measurements/daily-series-read.ts",
+    ]) {
+      const src = readFileSync(join(SRC, file), "utf8");
+      expect(src, file).not.toMatch(/\.measurement\s*\.\s*findMany\s*\(/);
+    }
+    const baseline = readFileSync(
+      join(SRC, "lib/insights/derived/baseline.ts"),
+      "utf8",
+    );
+    expect(baseline).not.toMatch(/\.measurement\s*\.\s*findMany\s*\(/);
+  });
+});

src/__tests__/helpers/provider-order-mock.ts

@@ -0,0 +1,67 @@
+/**
+ * A `@/lib/documents/provider-order` mock for the vault route suites.
+ *
+ * The suites steer the document pick through `resolveDocumentVisionProvider` /
+ * `resolveDocumentTextProvider` (a `{ chain, pick }` per case). The routes call
+ * the `require*` forms, which throw instead of returning an empty pick; these
+ * are built on the two mocked resolvers so a case that returns no pick sees
+ * the route answer the vault's own `documents.inbound.providerUnsupported`
+ * 422, and a case that returns a `withheld` refusal sees it thrown.
+ *
+ * Use inside a factory, where only a dynamic import is allowed:
+ *
+ *   vi.mock("@/lib/documents/provider-order", async () =>
+ *     (await import("@/__tests__/helpers/provider-order-mock")).providerOrderMock(),
+ *   );
+ */
+import { vi } from "vitest";
+
+import { AiUnavailableError } from "@/lib/ai/capabilities/refusal";
+
+interface MockPick {
+  pick: unknown;
+  withheld?: AiUnavailableError | null;
+}
+
+async function required(resolved: Promise<MockPick> | MockPick) {
+  const result = await resolved;
+  if (result?.withheld) throw result.withheld;
+  if (!result?.pick) {
+    throw new AiUnavailableError("documentAi", "no_provider", null, {
+      errorCode: "documents.inbound.providerUnsupported",
+      status: 422,
+    });
+  }
+  return result.pick;
+}
+
+export function providerOrderMock() {
+  const resolveDocumentVisionProvider = vi.fn();
+  const resolveDocumentTextProvider = vi.fn();
+  return {
+    resolveDocumentVisionProvider,
+    resolveDocumentTextProvider,
+    requireDocumentVisionProvider: vi.fn((userId: string) =>
+      required(resolveDocumentVisionProvider(userId)),
+    ),
+    requireDocumentTextProvider: vi.fn((userId: string) =>
+      required(resolveDocumentTextProvider(userId)),
+    ),
+  };
+}
+
+/** The capability gate, open. Pair with `vi.mock("@/lib/ai/capabilities/gate", …)`. */
+export function openCapabilityGateMock() {
+  return {
+    requireAiCapability: vi.fn(async () => ({
+      available: true,
+      reason: null,
+      onDeviceAllowed: true,
+    })),
+    getAiCapability: vi.fn(async () => ({
+      available: true,
+      reason: null,
+      onDeviceAllowed: true,
+    })),
+  };
+}

src/__tests__/i18n-reverse-coverage.test.ts

@@ -74,7 +74,7 @@ const DYNAMIC_ALLOWLIST_PREFIXES = [
  */
 const EXEMPT_TEMPLATES: Record<string, string> = {
   "insights.${scope}-status.${locale}":
-    "auditLog action name for the per-metric status cache (statusCacheAction), not an i18n key",
+    "the per-metric status note name (statusCacheAction), not an i18n key",
   "insights.${scope}.${locale}":
     "auditLog action name for the previous-insight memory read (getPreviousInsightContext), not an i18n key",
   "apiErrors.${errorCode}":