tom/HealthLog
1
0 Fork 0
mirror of https://github.com/MBombeck/HealthLog.git synced 2026-11-09 13:01:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Delete the citation-enforcement chain that never ran

Browse Source 
generateInsight() parsed against the strict aiInsightResponseSchema,
cross-checked that every recommendation's metricSource appeared in
citations[], and threw InsightSchemaError on a second failure. Its only
caller was runWithFallback(), which had no production caller — every
live surface runs runRawCompletionWithFallback. So none of it executed.

Wiring it into the live path was the alternative and it is not viable:
the live insightResultSchema has no citations[] field at all and treats
metricSource as optional, so the cross-check would have rejected
essentially every generation rather than tightening anything.

Removed: runWithFallback and its param/result types, generateInsight
and its private helpers, findUncitedRecommendations,
findRecommendationsMissingRationale, the never-thrown InsightSchemaError
and the isHardProviderFailure branch that excluded it, and
computeConfidence — a deterministic override that only ran inside the
dead wrapper.

Kept: buildRetryCorrectionMessage, which comprehensive-generate sends on
its own retry leg; the file it lived in is renamed to match what it now
holds. aiInsightResponseSchema stays as the shape that message
describes. Note the retry prompt asks for a stricter payload than the
live validator requires — that mismatch predates this and is left alone.

computeCitationCoverage is untouched and still live: it is a metric, not
a gate, and v1.30.24 wired it into the comprehensive post-parse step.

Chain-runner coverage the deleted tests carried (last-working cache,
empty-input guard) is ported onto runRawCompletionWithFallback, which
shares the machinery.
This commit is contained in:
Marc-André Bombeck committed 2026-07-19 05:50:57 +02:00
1 parent 576be8b746
commit 4183202ed3
20 files changed
+231 -1892
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/components/insights/confidence-meter.tsx
+1 -1
View File
@@ -10,7 +10,7 @@ export type ConfidenceBand = "high" | "medium" | "low" | "draft";
export type ConfidenceMeterVariant = "bars" | "ring";
export interface ConfidenceMeterProps {
/** Score from `computeConfidence()` — 0..100, integer expected. */
/** Score carried on the insight payload — 0..100, integer expected. */
value: number;
/** Visual variant. Defaults to `bars`. */
variant?: ConfidenceMeterVariant;
src/lib/ai/__tests__/citation-coverage-logging.test.ts
+1 -98
View File
@@ -1,7 +1,4 @@
import { describe, it, expect, beforeEach, vi } from "vitest";
import { generateInsight } from "../generate-insight";
import { MockAIProvider } from "../mock-client";
import { singleUserTurn } from "../types";
import { describe, it, expect, vi } from "vitest";
import {
detectsNormativeClaim,
computeCitationCoverage,
@@ -236,97 +233,3 @@ describe("computeCitationCoverage()", () => {
expect(result.uncitedNormativeRecommendationIds).toEqual(["uncited-1"]);
});
});
describe("generateInsight() — citation-coverage annotation", () => {
beforeEach(() => {
annotateMock.mockReset();
});
/**
* v1.4.16 phase B5d — `applyConfidenceOverride()` also calls
* `annotate()` (action: "confidence.override.applied"), so the
* citation-coverage assertions filter by meta-key rather than by
* call count.
*/
function findCoverageCall(calls: Array<unknown[]>): Record<string, unknown> {
for (const call of calls) {
const arg = call[0] as { meta?: Record<string, unknown> } | undefined;
const meta = arg?.meta;
if (meta && "ai_total_recommendations" in meta) {
return meta;
}
}
throw new Error("citation-coverage annotate() call not found");
}
it("emits an annotate() call with the coverage breakdown on success", async () => {
const provider = new MockAIProvider({
responses: JSON.stringify({
summary: "x",
recommendations: [
{
id: "rec-cited",
text: "Aim for a target below 140/90",
severity: "important",
metricSource: baseMetricSource,
rationale: baseRationale,
referenceId: knownRefId,
},
],
citations: [baseCitation],
warnings: [],
}),
});
await generateInsight(
provider,
singleUserTurn({ system: "system", user: "user" }),
);
const meta = findCoverageCall(annotateMock.mock.calls);
expect(meta).toMatchObject({
ai_total_recommendations: 1,
ai_normative_recommendations: 1,
ai_cited_normative_recommendations: 1,
ai_uncited_normative_recommendation_ids: [],
});
});
it("annotation flags the uncited normative rec", async () => {
const provider = new MockAIProvider({
responses: JSON.stringify({
summary: "x",
recommendations: [
{
id: "rec-naked",
text: "Your BP should stay below 140/90",
severity: "important",
metricSource: baseMetricSource,
rationale: baseRationale,
},
],
citations: [baseCitation],
warnings: [],
}),
});
await generateInsight(
provider,
singleUserTurn({ system: "system", user: "user" }),
);
const meta = findCoverageCall(annotateMock.mock.calls);
expect(meta).toMatchObject({
ai_normative_recommendations: 1,
ai_cited_normative_recommendations: 0,
ai_uncited_normative_recommendation_ids: ["rec-naked"],
});
});
it("does NOT annotate when generation throws (only on success)", async () => {
// Provider returns invalid JSON twice → InsightSchemaError surfaces.
const provider = new MockAIProvider({
responses: ["not json", "still not json"],
});
await expect(
generateInsight(provider, singleUserTurn({ system: "s", user: "u" })),
).rejects.toThrow();
expect(annotateMock).not.toHaveBeenCalled();
});
});
src/lib/ai/__tests__/citation-enforcement.test.ts
-149
View File
@@ -1,14 +1,10 @@
import { describe, it, expect } from "vitest";
import {
aiInsightResponseSchema,
findUncitedRecommendations,
metricSourceSchema,
aiRecommendationSchema,
type AIInsightResponse,
} from "../schema";
import { generateInsight } from "../generate-insight";
import { MockAIProvider } from "../mock-client";
import { singleUserTurn } from "../types";
/**
* Phase C1 — citation-from-data hard requirement.
@@ -148,148 +144,3 @@ describe("citation-from-data — schema-level enforcement", () => {
expect(result.success).toBe(true);
});
});
describe("citation-from-data — cross-check enforcement", () => {
it("recommendation citing a (type, timeRange) absent from citations[] is flagged", () => {
const orphan: AIInsightResponse = {
...baseValid,
recommendations: [
{
id: "rec-orphan",
text: "Cut sodium.",
severity: "suggestion",
metricSource: {
type: "diet",
timeRange: "last7days",
summary: "no diet logging available",
},
rationale: {
dataWindow: "last7days",
comparedTo: "general DGE intake guideline",
deviation: "n/a — no salt tracking present",
},
},
],
citations: [baseCitation], // diet citation missing
};
const missing = findUncitedRecommendations(orphan);
expect(missing).toHaveLength(1);
expect(missing[0].recommendationId).toBe("rec-orphan");
expect(missing[0].missing).toEqual({
type: "diet",
timeRange: "last7days",
});
});
it("partial-key match on type alone is NOT enough — timeRange must also match", () => {
const orphan: AIInsightResponse = {
...baseValid,
recommendations: [
{
id: "rec-mismatch",
text: "X",
severity: "info",
metricSource: {
type: "bloodPressure",
timeRange: "last30days", // citation only covers last7days
summary: "avg over 30 days",
},
rationale: {
dataWindow: "last30days",
comparedTo: "your 90-day median",
deviation: "+2 mmHg over 30-day window",
},
},
],
citations: [baseCitation],
};
const missing = findUncitedRecommendations(orphan);
expect(missing).toHaveLength(1);
});
it("multiple recommendations sharing the same citation pass once that one citation is present", () => {
const ok: AIInsightResponse = {
...baseValid,
recommendations: [
baseValid.recommendations[0],
{
id: "rec-2",
text: "Continue daily logging.",
severity: "info",
metricSource: { ...baseMetricSource },
rationale: baseRationale,
},
],
citations: [baseCitation],
};
const missing = findUncitedRecommendations(ok);
expect(missing).toEqual([]);
});
});
describe("citation-from-data — wrapper end-to-end", () => {
it("uncited recommendation triggers retry; second try with proper citation passes", async () => {
const orphan = {
...baseValid,
recommendations: [
{
id: "rec-orphan",
text: "Cut sodium.",
severity: "suggestion",
metricSource: {
type: "diet",
timeRange: "last7days",
summary: "no diet logging available",
},
},
],
citations: [],
};
const provider = new MockAIProvider({
responses: [JSON.stringify(orphan), JSON.stringify(baseValid)],
});
const outcome = await generateInsight(
provider,
singleUserTurn({ system: "system", user: "user" }),
);
expect(outcome.retried).toBe(true);
expect(outcome.parsed.recommendations[0].id).toBe("rec-1");
});
it("retry-attempt that drops the recommendation rather than fixing the citation also passes", async () => {
const orphan = {
...baseValid,
recommendations: [
{
id: "rec-orphan",
text: "Cut sodium.",
severity: "suggestion",
metricSource: {
type: "diet",
timeRange: "last7days",
summary: "no diet logging available",
},
},
],
citations: [],
};
// Second attempt — model decides to drop the unsupported recommendation
// entirely. This is the explicitly-allowed "OMIT it" path from the
// retry-correction message.
const droppedRecs = {
summary: "Insufficient data for actionable recommendations.",
recommendations: [],
citations: [],
warnings: [],
};
const provider = new MockAIProvider({
responses: [JSON.stringify(orphan), JSON.stringify(droppedRecs)],
});
const outcome = await generateInsight(
provider,
singleUserTurn({ system: "system", user: "user" }),
);
expect(outcome.retried).toBe(true);
expect(outcome.parsed.recommendations).toHaveLength(0);
});
});
src/lib/ai/__tests__/confidence-schema.test.ts
+1 -1
View File
@@ -5,7 +5,7 @@ import { aiInsightResponseSchema, aiRecommendationSchema } from "../schema";
* v1.4.16 phase B5d — `recommendation.confidence` schema field.
*
* The wrapper overrides the model's confidence with a deterministic
* server-computed value (`computeConfidence()`), but the schema needs
* value the model emits, but the schema needs
* to ACCEPT a model-supplied number so payloads round-trip cleanly.
* The field is `.optional()` because:
* - The wrapper fills it post-validation regardless.
src/lib/ai/__tests__/confidence-wrapper.test.ts
-203
View File
@@ -1,203 +0,0 @@
import { describe, it, expect } from "vitest";
import { generateInsight } from "../generate-insight";
import { MockAIProvider } from "../mock-client";
import { singleUserTurn } from "../types";
/**
* v1.4.16 phase B5d — wrapper post-validation override.
*
* The model's self-reported `confidence` is DISCARDED. After zod
* parse succeeds, `generateInsight()` derives `ConfidenceInputs` per
* recommendation and overwrites `rec.confidence` with the value
* `computeConfidence()` produces.
*
* Default-on per research §7 question 1 (maintainer-acked). The override
* happens whether or not the caller supplies a bucket-series-aware
* resolver — when the resolver is missing we fall back to inputs
* derivable from the parsed payload alone (`metricSource.n`,
* recencyDays=0, deviationStdRatio=null) so the contract is "the
* confidence on the response NEVER comes from the LLM".
*/
const validResponseWithModelConfidence = {
summary: "Your blood pressure is trending high.",
recommendations: [
{
id: "rec-1",
text: "Discuss lifestyle and possible medication review with your physician.",
severity: "important",
// Model claims 99 — wrapper must overwrite this.
confidence: 99,
metricSource: {
type: "bloodPressure",
timeRange: "last7days",
summary: "avg 142/88 across 12 readings",
n: 12,
},
rationale: {
dataWindow: "last7days",
comparedTo: "your 90-day median (128/82)",
deviation: "+14/+6 mmHg above baseline over 7 of 7 days",
},
},
],
citations: [
{
type: "bloodPressure",
timeRange: "last7days",
summary: "avg 142/88 across 12 readings",
},
],
warnings: [],
};
describe("generateInsight() — confidence override", () => {
it("overrides model-supplied confidence with deterministic computation (default fallback)", async () => {
const provider = new MockAIProvider({
responses: JSON.stringify(validResponseWithModelConfidence),
});
const outcome = await generateInsight(
provider,
singleUserTurn({ system: "sys", user: "user" }),
);
const rec = outcome.parsed.recommendations[0];
// Model claimed 99; wrapper must have replaced it.
// n=12, recencyDays=0 (default), ratio=null (default) →
// nScore = 10 + 10*log10(12) ≈ 20.79
// recency = 30 * (1 - 0/30) = 30
// signal = null → 15
// total ≈ 65.79 → 66
expect(rec.confidence).toBe(66);
expect(rec.confidence).not.toBe(99);
});
it("fills in confidence even when the model omitted it", async () => {
const noConfidence = {
...validResponseWithModelConfidence,
recommendations: [
{
...validResponseWithModelConfidence.recommendations[0],
confidence: undefined,
},
],
};
// strip the undefined to emulate the model not emitting the key
const stringified = JSON.stringify(noConfidence);
const provider = new MockAIProvider({ responses: stringified });
const outcome = await generateInsight(
provider,
singleUserTurn({ system: "sys", user: "u" }),
);
expect(outcome.parsed.recommendations[0].confidence).toBe(66);
});
it("derives n from metricSource.n on the parsed payload (no caller injection)", async () => {
// v1.4.16 phase D reconcile (simplify F2) — the wrapper used to
// accept an optional `confidenceContext` resolver, but no production
// caller ever passed one. Inputs are derived from the payload alone
// today; v1.4.17's feedback ratchet will re-introduce a resolver
// injection when a real second caller exists.
const fourteenN = {
...validResponseWithModelConfidence,
recommendations: [
{
...validResponseWithModelConfidence.recommendations[0],
metricSource: {
...validResponseWithModelConfidence.recommendations[0].metricSource,
n: 14,
},
},
],
};
const provider = new MockAIProvider({
responses: JSON.stringify(fourteenN),
});
const outcome = await generateInsight(
provider,
singleUserTurn({ system: "s", user: "u" }),
);
// n=14, recencyDays=0, ratio=null → 10 + 10*log10(14) + 30 + 15 ≈ 66.46
expect(outcome.parsed.recommendations[0].confidence).toBe(66);
});
it("hard-caps confidence to <=15 when n<3 (small-data shield)", async () => {
const tinyN = {
...validResponseWithModelConfidence,
recommendations: [
{
...validResponseWithModelConfidence.recommendations[0],
metricSource: {
...validResponseWithModelConfidence.recommendations[0].metricSource,
n: 1,
},
},
],
};
const provider = new MockAIProvider({ responses: JSON.stringify(tinyN) });
const outcome = await generateInsight(
provider,
singleUserTurn({ system: "s", user: "u" }),
);
expect(outcome.parsed.recommendations[0].confidence).toBeLessThanOrEqual(
15,
);
});
it("treats missing metricSource.n as n=0 (hard-cap)", async () => {
const noN = {
...validResponseWithModelConfidence,
recommendations: [
{
...validResponseWithModelConfidence.recommendations[0],
metricSource: {
type: "bloodPressure",
timeRange: "last7days",
summary: "avg 142/88 across 12 readings",
// n omitted
},
},
],
};
const provider = new MockAIProvider({ responses: JSON.stringify(noN) });
const outcome = await generateInsight(
provider,
singleUserTurn({ system: "s", user: "u" }),
);
// n=0 → hard-cap floor 10
expect(outcome.parsed.recommendations[0].confidence).toBe(10);
});
it("overrides confidence on every recommendation in payload order", async () => {
const twoRecs = {
...validResponseWithModelConfidence,
recommendations: [
{
...validResponseWithModelConfidence.recommendations[0],
id: "rec-a",
confidence: 99,
},
{
...validResponseWithModelConfidence.recommendations[0],
id: "rec-b",
confidence: 1,
metricSource: {
...validResponseWithModelConfidence.recommendations[0].metricSource,
n: 5,
},
},
],
};
const provider = new MockAIProvider({
responses: JSON.stringify(twoRecs),
});
const outcome = await generateInsight(
provider,
singleUserTurn({ system: "s", user: "u" }),
);
// Model claimed (99, 1); both must be discarded and replaced with
// the deterministic value (n=12 → 66, n=5 → 62 from the saturating
// formula 10 + 10*log10(5) + 30 + 15 ≈ 61.99).
expect(outcome.parsed.recommendations[0].confidence).toBe(66);
expect(outcome.parsed.recommendations[1].confidence).toBe(62);
});
});
src/lib/ai/__tests__/confidence.test.ts
-150
View File
@@ -1,150 +0,0 @@
import { describe, it, expect } from "vitest";
import { computeConfidence, type ConfidenceInputs } from "../confidence";
/**
* v1.4.16 phase B5d — `computeConfidence()` deterministic scorer.
*
* The wrapper feeds three inputs:
* - n: sample count behind the cited data window
* (from `metricSource.n` when available; 0 otherwise).
* - recencyDays: days since the most recent sample in the window.
* - deviationStdRatio: |deviation| / stdev of the user's 90-day
* baseline. `null` when the baseline is too thin to compute.
*
* Score breakdown (max 100):
* - n contribution: log-saturating curve, max 40 at n=30 ish.
* n<3 hard-caps the whole score at ≤15 (5 * n).
* - recencyScore: 30 at <2d, decays linearly to 0 at 30d, floored at 0.
* - signalScore: 30 at |z|≥1.5, 0 at z=0, 15 when ratio is null.
*
* 12 fixtures cover the input matrix per research §2.C: small-n hard
* cap, mid-n + recency interplay, saturated-n + signal extremes,
* null-ratio neutral path.
*/
interface Fixture {
name: string;
inputs: ConfidenceInputs;
expected: number;
}
const FIXTURES: Fixture[] = [
// n hard-cap: n<3 always returns max(10, 5*n)
{
name: "n=0 → hard-cap 10 floor",
inputs: { n: 0, recencyDays: 0, deviationStdRatio: 2 },
expected: 10,
},
{
name: "n=1 → hard-cap 10 floor",
inputs: { n: 1, recencyDays: 0, deviationStdRatio: 2 },
expected: 10,
},
{
name: "n=2 → hard-cap 10 (5*2)",
inputs: { n: 2, recencyDays: 0, deviationStdRatio: 2 },
expected: 10,
},
// n=3: nScore = 10 + 10*log10(3) ≈ 14.77;
// recency=1 → 30 * (1 - 1/30) = 29; signal=null → 15
// total ≈ 14.77 + 29 + 15 = 58.77 → 59
{
name: "n=3 fresh sample (1d old), null ratio → 59",
inputs: { n: 3, recencyDays: 1, deviationStdRatio: null },
expected: 59,
},
// n=14, recencyDays=1, ratio=2 → research-§2C calibration target ~94
// nScore = 10 + 10*log10(14) ≈ 21.46
// recencyScore = 30 * (1 - 1/30) = 29
// signalScore = min(30, 30 * 2/1.5) = 30
// total ≈ 80.46 → 80
{
name: "n=14, fresh, strong signal → ~80",
inputs: { n: 14, recencyDays: 1, deviationStdRatio: 2 },
expected: 80,
},
// n=30, recencyDays=20, null ratio → research-§2C calibration target ~52
// nScore = min(40, 10 + 10*log10(30)) = min(40, 24.77) = 24.77
// recencyScore = max(0, 30 * (1 - 20/30)) = 10
// signalScore = 15 (null)
// total ≈ 49.77 → 50
{
name: "n=30, stale (20d), null ratio → ~50",
inputs: { n: 30, recencyDays: 20, deviationStdRatio: null },
expected: 50,
},
// n=100 → nScore = min(40, 10 + 10*log10(100)) = 30 (curve, not cap)
// recency=0 → 30, signal=z=1.5 → 30, total 90
{
name: "n=100, fresh, strong signal → 90",
inputs: { n: 100, recencyDays: 0, deviationStdRatio: 1.5 },
expected: 90,
},
// n=10000 → log10 saturates at the 40 cap → max possible total 100
{
name: "n=10000 saturates the n curve at 40 → 100",
inputs: { n: 10000, recencyDays: 0, deviationStdRatio: 2 },
expected: 100,
},
// n=10, recency=30d → recency floors at 0; ratio=0 → signal=0
// nScore = 10 + 10*log10(10) = 20; recency=0; signal=0
// total = 20
{
name: "n=10, ancient (30d), zero deviation → 20",
inputs: { n: 10, recencyDays: 30, deviationStdRatio: 0 },
expected: 20,
},
// n=10, recency=45d → recencyScore floor 0 (negative clamped)
{
name: "n=10, very ancient (45d) clamps recency to 0",
inputs: { n: 10, recencyDays: 45, deviationStdRatio: null },
expected: 35, // 20 + 0 + 15
},
// n=20, fresh, mild signal (z=0.75 → half of cap)
// nScore = 10 + 10*log10(20) ≈ 23.01
// recency=0 → 30
// signal = 30 * 0.75/1.5 = 15
// total ≈ 68.01 → 68
{
name: "n=20, fresh, mild signal → ~68",
inputs: { n: 20, recencyDays: 0, deviationStdRatio: 0.75 },
expected: 68,
},
// Negative ratio: |z|=2 still hits cap of 30
{
name: "n=14, fresh, strong negative signal (z=-2) → 80",
inputs: { n: 14, recencyDays: 1, deviationStdRatio: -2 },
expected: 80,
},
// Edge: n=2, fresh, strong signal — hard-cap still applies
{
name: "n=2 hard-cap survives even with strong signal + fresh",
inputs: { n: 2, recencyDays: 0, deviationStdRatio: 2 },
expected: 10,
},
];
describe("computeConfidence()", () => {
it.each(FIXTURES)("$name", ({ inputs, expected }) => {
expect(computeConfidence(inputs)).toBe(expected);
});
it("returns an integer in [0, 100] for all fixtures", () => {
for (const f of FIXTURES) {
const v = computeConfidence(f.inputs);
expect(Number.isInteger(v)).toBe(true);
expect(v).toBeGreaterThanOrEqual(0);
expect(v).toBeLessThanOrEqual(100);
}
});
it("is monotonic in n when other inputs fixed (more data → never lower)", () => {
const baseInputs = { recencyDays: 1, deviationStdRatio: 1 };
let prev = -Infinity;
for (const n of [5, 10, 20, 50, 100]) {
const v = computeConfidence({ ...baseInputs, n });
expect(v).toBeGreaterThanOrEqual(prev);
prev = v;
}
});
});
src/lib/ai/__tests__/generate-insight.test.ts
-262
View File
@@ -1,262 +0,0 @@
import { describe, it, expect } from "vitest";
import {
aiInsightResponseSchema,
findUncitedRecommendations,
InsightSchemaError,
} from "../schema";
import { generateInsight } from "../generate-insight";
import { MockAIProvider } from "../mock-client";
import { singleUserTurn, type CompletionParams } from "../types";
/** Read the last user message's text out of a recorded CompletionParams. */
function lastUserText(params: CompletionParams): string {
for (let i = params.messages.length - 1; i >= 0; i -= 1) {
const m = params.messages[i];
if (m.role !== "user") continue;
return typeof m.content === "string"
? m.content
: m.content
.filter((p): p is { type: "text"; text: string } => p.type === "text")
.map((p) => p.text)
.join("");
}
return "";
}
const validResponse = {
summary: "Your blood pressure is trending high.",
recommendations: [
{
id: "rec-1",
text: "Discuss lifestyle and possible medication review with your physician.",
severity: "important",
metricSource: {
type: "bloodPressure",
timeRange: "last7days",
summary: "avg 142/88 across 12 readings",
n: 12,
},
rationale: {
dataWindow: "last7days",
comparedTo: "your 90-day median (128/82)",
deviation: "+14/+6 mmHg above baseline over 7 of 7 days",
},
},
],
citations: [
{
type: "bloodPressure",
timeRange: "last7days",
summary: "avg 142/88 across 12 readings",
},
],
warnings: [
{
topic: "blood_pressure",
message: "Stage 1 hypertension threshold crossed.",
severity: "important",
},
],
};
describe("aiInsightResponseSchema", () => {
it("accepts a well-formed strict response", () => {
const result = aiInsightResponseSchema.safeParse(validResponse);
expect(result.success).toBe(true);
});
it("permits legacy passthrough fields (back-compat)", () => {
const withLegacy = {
...validResponse,
classification: "grenzwertig",
findings: [{ label: "x", value: "y", assessment: "neutral" }],
disclaimer: "consult your doctor",
};
const result = aiInsightResponseSchema.safeParse(withLegacy);
expect(result.success).toBe(true);
});
it("rejects when summary is empty", () => {
const result = aiInsightResponseSchema.safeParse({
...validResponse,
summary: "",
});
expect(result.success).toBe(false);
});
it("rejects a recommendation without metricSource.summary", () => {
const broken = {
...validResponse,
recommendations: [
{
id: "rec-1",
text: "Walk more",
severity: "suggestion",
metricSource: {
type: "activity",
timeRange: "last7days",
summary: "", // empty — should fail
},
},
],
};
const result = aiInsightResponseSchema.safeParse(broken);
expect(result.success).toBe(false);
});
it("rejects a recommendation with missing severity enum", () => {
const broken = {
...validResponse,
recommendations: [
{
id: "rec-1",
text: "Walk more",
severity: "kinda-bad", // not in enum
metricSource: {
type: "activity",
timeRange: "last7days",
summary: "5,000 steps avg",
},
},
],
};
const result = aiInsightResponseSchema.safeParse(broken);
expect(result.success).toBe(false);
});
it("findUncitedRecommendations flags recommendations whose metricSource is not in citations", () => {
const orphan = {
...validResponse,
recommendations: [
validResponse.recommendations[0],
{
id: "rec-2",
text: "Reduce salt intake",
severity: "suggestion",
metricSource: {
type: "diet",
timeRange: "last30days",
summary: "no salt logging available",
},
rationale: {
dataWindow: "last30days",
comparedTo: "general DGE intake guideline",
deviation: "n/a — no salt tracking present",
},
},
],
};
const parsed = aiInsightResponseSchema.parse(orphan);
const missing = findUncitedRecommendations(parsed);
expect(missing).toHaveLength(1);
expect(missing[0].recommendationId).toBe("rec-2");
expect(missing[0].missing.type).toBe("diet");
});
});
describe("generateInsight wrapper", () => {
it("returns the parsed response on first-attempt success", async () => {
const provider = new MockAIProvider({
responses: JSON.stringify(validResponse),
});
const outcome = await generateInsight(
provider,
singleUserTurn({ system: "sys", user: "user" }),
);
expect(outcome.attempts).toBe(1);
expect(outcome.retried).toBe(false);
expect(outcome.parsed.summary).toMatch(/blood pressure/);
expect(provider.callCount).toBe(1);
});
it("retries once with correction message and succeeds on second attempt", async () => {
const provider = new MockAIProvider({
responses: ["definitely not json", JSON.stringify(validResponse)],
});
const outcome = await generateInsight(
provider,
singleUserTurn({ system: "sys", user: "Original user prompt." }),
);
expect(outcome.attempts).toBe(2);
expect(outcome.retried).toBe(true);
expect(provider.callCount).toBe(2);
// The retry call must include the original user prompt PLUS the
// corrective suffix referencing the schema.
const retryUserPrompt = lastUserText(provider.calls[1]);
expect(retryUserPrompt).toContain("Original user prompt.");
expect(retryUserPrompt).toContain("did not satisfy");
expect(retryUserPrompt).toContain("metricSource");
});
it("throws InsightSchemaError(422) when both attempts fail", async () => {
const provider = new MockAIProvider({
responses: ["nope", "still nope"],
});
let caught: unknown;
try {
await generateInsight(
provider,
singleUserTurn({ system: "sys", user: "user" }),
);
} catch (e) {
caught = e;
}
expect(caught).toBeInstanceOf(InsightSchemaError);
const err = caught as InsightSchemaError;
expect(err.httpStatus).toBe(422);
expect(err.attempts).toBe(2);
expect(provider.callCount).toBe(2);
});
it("retries when first attempt has uncited recommendations", async () => {
const orphanResponse = {
...validResponse,
recommendations: [
{
id: "rec-orphan",
text: "Cut sugar",
severity: "suggestion",
metricSource: {
type: "diet",
timeRange: "last7days",
summary: "no sugar tracking present",
},
rationale: {
dataWindow: "last7days",
comparedTo: "general DGE intake guideline",
deviation: "n/a — no sugar tracking present",
},
},
],
citations: [], // empty — fails cross-check
};
const provider = new MockAIProvider({
responses: [
JSON.stringify(orphanResponse),
JSON.stringify(validResponse),
],
});
const outcome = await generateInsight(
provider,
singleUserTurn({ system: "sys", user: "u" }),
);
expect(outcome.attempts).toBe(2);
expect(outcome.retried).toBe(true);
expect(lastUserText(provider.calls[1])).toContain(
"metricSources not in citations",
);
});
it("propagates provider-level errors without retrying", async () => {
const provider = new MockAIProvider({
rejectWith: Object.assign(new Error("OpenAI request failed (500)"), {
httpStatus: 500,
}),
});
await expect(
generateInsight(provider, singleUserTurn({ system: "s", user: "u" })),
).rejects.toThrow("OpenAI request failed (500)");
// Provider only called once — wrapper does not retry on provider errors.
expect(provider.callCount).toBe(1);
});
});
src/lib/ai/__tests__/insight-generator-prompt.test.ts
+1 -27
View File
@@ -5,10 +5,7 @@ import {
OUT_OF_SCOPE_REFUSAL_EN,
OUT_OF_SCOPE_REFUSAL_DE,
} from "../prompts/insight-generator";
import { aiInsightResponseSchema, findUncitedRecommendations } from "../schema";
import { generateInsight } from "../generate-insight";
import { MockAIProvider } from "../mock-client";
import { singleUserTurn } from "../types";
import { aiInsightResponseSchema } from "../schema";
/**
* Phase C1 — scope-hardened system prompt assertions.
@@ -137,27 +134,4 @@ describe("out-of-scope refusal payloads", () => {
const result = aiInsightResponseSchema.safeParse(OUT_OF_SCOPE_REFUSAL_DE);
expect(result.success).toBe(true);
});
it("refusal payload has no orphan citations (trivially)", () => {
const parsed = aiInsightResponseSchema.parse(OUT_OF_SCOPE_REFUSAL_EN);
expect(findUncitedRecommendations(parsed)).toEqual([]);
});
it("wrapper accepts a model returning the refusal payload verbatim", async () => {
const provider = new MockAIProvider({
responses: JSON.stringify(OUT_OF_SCOPE_REFUSAL_EN),
});
const outcome = await generateInsight(
provider,
singleUserTurn({
system: getStrictInsightsSystemPrompt("en"),
// Deliberately out-of-scope user prompt — model is instructed to
// return the refusal rather than fabricate health metrics.
user: "What's the weather in Berlin tomorrow and how do I write Python?",
}),
);
expect(outcome.attempts).toBe(1);
expect(outcome.parsed.recommendations).toEqual([]);
expect(outcome.parsed.summary).toContain("only summarise");
});
});
src/lib/ai/__tests__/insight-response-schema.test.ts
+103
View File
@@ -0,0 +1,103 @@
import { describe, it, expect } from "vitest";
import { aiInsightResponseSchema } from "../schema";
const validResponse = {
summary: "Your blood pressure is trending high.",
recommendations: [
{
id: "rec-1",
text: "Discuss lifestyle and possible medication review with your physician.",
severity: "important",
metricSource: {
type: "bloodPressure",
timeRange: "last7days",
summary: "avg 142/88 across 12 readings",
n: 12,
},
rationale: {
dataWindow: "last7days",
comparedTo: "your 90-day median (128/82)",
deviation: "+14/+6 mmHg above baseline over 7 of 7 days",
},
},
],
citations: [
{
type: "bloodPressure",
timeRange: "last7days",
summary: "avg 142/88 across 12 readings",
},
],
warnings: [
{
topic: "blood_pressure",
message: "Stage 1 hypertension threshold crossed.",
severity: "important",
},
],
};
describe("aiInsightResponseSchema", () => {
it("accepts a well-formed strict response", () => {
const result = aiInsightResponseSchema.safeParse(validResponse);
expect(result.success).toBe(true);
});
it("permits legacy passthrough fields (back-compat)", () => {
const withLegacy = {
...validResponse,
classification: "grenzwertig",
findings: [{ label: "x", value: "y", assessment: "neutral" }],
disclaimer: "consult your doctor",
};
const result = aiInsightResponseSchema.safeParse(withLegacy);
expect(result.success).toBe(true);
});
it("rejects when summary is empty", () => {
const result = aiInsightResponseSchema.safeParse({
...validResponse,
summary: "",
});
expect(result.success).toBe(false);
});
it("rejects a recommendation without metricSource.summary", () => {
const broken = {
...validResponse,
recommendations: [
{
id: "rec-1",
text: "Walk more",
severity: "suggestion",
metricSource: {
type: "activity",
timeRange: "last7days",
summary: "", // empty — should fail
},
},
],
};
const result = aiInsightResponseSchema.safeParse(broken);
expect(result.success).toBe(false);
});
it("rejects a recommendation with missing severity enum", () => {
const broken = {
...validResponse,
recommendations: [
{
id: "rec-1",
text: "Walk more",
severity: "kinda-bad", // not in enum
metricSource: {
type: "activity",
timeRange: "last7days",
summary: "5,000 steps avg",
},
},
],
};
const result = aiInsightResponseSchema.safeParse(broken);
expect(result.success).toBe(false);
});
});
src/lib/ai/__tests__/provider-runner.test.ts
+8 -151
View File
@@ -33,7 +33,6 @@ import {
isHardProviderFailure,
runRawCompletionWithFallback,
runStreamingRawCompletionWithFallback,
runWithFallback,
} from "../provider-runner";
import {
AUTH_FAILURE_COOLDOWN_MS,
@@ -129,15 +128,6 @@ describe("isHardProviderFailure", () => {
expect(isHardProviderFailure(err(404))).toBe(true);
});
it("does NOT flag an InsightSchemaError as hard — the strict JSON 422 surface still bubbles", async () => {
const { InsightSchemaError } = await import("../schema");
expect(
isHardProviderFailure(
new InsightSchemaError("bad json", { attempts: 2 }),
),
).toBe(false);
});
it("flags network errors (no httpStatus) as hard", () => {
expect(isHardProviderFailure(new Error("ECONNRESET"))).toBe(true);
});
@@ -151,140 +141,7 @@ describe("isHardProviderFailure", () => {
});
});
describe("runWithFallback — happy path", () => {
it("returns the parsed response from the first working provider", async () => {
const codex = new ScriptedProvider({
type: "codex",
script: [{ ok: true }],
});
const result = await runWithFallback({
userId: "u1",
providers: [{ providerType: "codex", instance: codex }],
params: singleUserTurn({ system: "s", user: "u" }),
});
expect(result.parsed.summary).toBe("ok");
expect(result.raw.providerType).toBe("codex");
expect(result.fallbackHops).toEqual([]);
expect(codex.callCount).toBe(1);
});
it("attaches the provider type to the outcome for B5e feedback attribution", async () => {
const openai = new ScriptedProvider({
type: "admin-key",
script: [{ ok: true }],
});
const result = await runWithFallback({
userId: "u1",
providers: [{ providerType: "openai", instance: openai }],
params: singleUserTurn({ system: "s", user: "u" }),
});
expect(result.workingProvider.providerType).toBe("openai");
expect(result.raw.providerType).toBe("admin-key");
});
});
describe("runWithFallback — primary fails 401, secondary succeeds", () => {
it("falls through to the secondary and surfaces the failure reason in the hop log", async () => {
const codex = new ScriptedProvider({
type: "codex",
script: [{ ok: false, error: err(401, "OAuth expired") }],
});
const openai = new ScriptedProvider({
type: "admin-key",
script: [{ ok: true }],
});
const result = await runWithFallback({
userId: "u1",
providers: [
{ providerType: "codex", instance: codex },
{ providerType: "openai", instance: openai },
],
params: singleUserTurn({ system: "s", user: "u" }),
});
expect(result.parsed.summary).toBe("ok");
expect(result.workingProvider.providerType).toBe("openai");
expect(result.fallbackHops).toHaveLength(1);
expect(result.fallbackHops[0]).toMatchObject({
providerType: "codex",
attempt: 1,
});
expect(result.fallbackHops[0].failureReason).toMatch(/401/);
});
});
describe("runWithFallback — all-fail cascade", () => {
it("throws AllProvidersFailedError when every chain entry fails hard", async () => {
const codex = new ScriptedProvider({
type: "codex",
script: [{ ok: false, error: err(0, "ECONNRESET") }],
});
const openai = new ScriptedProvider({
type: "admin-key",
script: [{ ok: false, error: err(503, "upstream down") }],
});
let caught: unknown;
try {
await runWithFallback({
userId: "u1",
providers: [
{ providerType: "codex", instance: codex },
{ providerType: "openai", instance: openai },
],
params: singleUserTurn({ system: "s", user: "u" }),
});
} catch (e) {
caught = e;
}
expect(caught).toBeInstanceOf(AllProvidersFailedError);
const e = caught as AllProvidersFailedError;
expect(e.httpStatus).toBe(503);
expect(e.attempts).toHaveLength(2);
expect(e.attempts.map((a) => a.providerType)).toEqual(["codex", "openai"]);
});
});
describe("runWithFallback — non-hard errors bubble unchanged", () => {
it("re-throws a 422 schema error from the first provider rather than walking", async () => {
// schema-mismatch is the wrapper's job, not the runner's. The runner
// only walks on transport / auth / 5xx — bad JSON from the first
// provider must still propagate as the wrapper's
// InsightSchemaError so the user gets the existing 422 surface.
const breaking = new ScriptedProvider({
type: "codex",
// Both wrapper attempts return invalid JSON → wrapper raises
// InsightSchemaError(422). The runner does not walk to provider
// 2 because that error is not a "hard provider failure".
script: [
{ ok: true, content: "not-json" },
{ ok: true, content: "still-not-json" },
],
});
const openai = new ScriptedProvider({
type: "admin-key",
script: [{ ok: true }],
});
let caught: unknown;
try {
await runWithFallback({
userId: "u1",
providers: [
{ providerType: "codex", instance: breaking },
{ providerType: "openai", instance: openai },
],
params: singleUserTurn({ system: "s", user: "u" }),
});
} catch (e) {
caught = e;
}
// Wrapper threw — fallback runner should NOT have escalated to
// openai because schema-mismatch is not a hard provider failure.
expect(caught).toBeDefined();
expect((caught as Error).name).toBe("InsightSchemaError");
expect(openai.callCount).toBe(0);
});
});
describe("runWithFallback — last-working cache", () => {
describe("runRawCompletionWithFallback — last-working cache", () => {
it("caches the working provider so the next call starts there", async () => {
const codex = new ScriptedProvider({
type: "codex",
@@ -296,7 +153,7 @@ describe("runWithFallback — last-working cache", () => {
});
// First call: codex fails, openai succeeds.
await runWithFallback({
await runRawCompletionWithFallback({
userId: "u-cache",
providers: [
{ providerType: "codex", instance: codex },
@@ -309,7 +166,7 @@ describe("runWithFallback — last-working cache", () => {
// Second call: cache reorders so openai is tried first; codex is
// never re-invoked even though it appears first in the input chain.
await runWithFallback({
await runRawCompletionWithFallback({
userId: "u-cache",
providers: [
{ providerType: "codex", instance: codex },
@@ -339,7 +196,7 @@ describe("runWithFallback — last-working cache", () => {
});
// First call: codex fails, openai succeeds, cache = openai.
await runWithFallback({
await runRawCompletionWithFallback({
userId: "u-ttl",
providers: [
{ providerType: "codex", instance: codex },
@@ -355,7 +212,7 @@ describe("runWithFallback — last-working cache", () => {
// Next call walks the original priority order again (codex first,
// and this time codex's second scripted entry succeeds).
await runWithFallback({
await runRawCompletionWithFallback({
userId: "u-ttl",
providers: [
{ providerType: "codex", instance: codex },
@@ -371,7 +228,7 @@ describe("runWithFallback — last-working cache", () => {
type: "admin-key",
script: [{ ok: true }],
});
await runWithFallback({
await runRawCompletionWithFallback({
userId: "u-clear",
providers: [{ providerType: "openai", instance: ok }],
params: singleUserTurn({ system: "s", user: "u" }),
@@ -502,11 +359,11 @@ describe("runRawCompletionWithFallback — legacy route shim", () => {
});
});
describe("runWithFallback — empty input", () => {
describe("runRawCompletionWithFallback — empty input", () => {
it("throws AllProvidersFailedError with httpStatus 422 when zero providers configured", async () => {
let caught: unknown;
try {
await runWithFallback({
await runRawCompletionWithFallback({
userId: "u-empty",
providers: [],
params: singleUserTurn({ system: "s", user: "u" }),
src/lib/ai/__tests__/rationale-schema.test.ts
-75
View File
@@ -3,7 +3,6 @@ import {
aiInsightResponseSchema,
aiRecommendationRationaleSchema,
aiRecommendationSchema,
findRecommendationsMissingRationale,
type AIInsightResponse,
} from "../schema";
@@ -195,77 +194,3 @@ describe("aiInsightResponseSchema — rationale integration", () => {
expect(parsed.recommendations[0].rationale).toEqual(baseRationale);
});
});
describe("findRecommendationsMissingRationale()", () => {
it("returns an empty array for a payload where every rec has rationale", () => {
expect(findRecommendationsMissingRationale(baseValid)).toEqual([]);
});
it("flags a legacy payload whose recs predate B5c (rationale absent)", () => {
// Legacy payloads from before B5c had no rationale field. Once
// the strict parser flips to mandatory rationale, those payloads
// would fail parse — but we keep `.passthrough()` for one
// milestone and use `findRecommendationsMissingRationale()` to
// *detect* legacy shape so the UI can prompt regeneration.
//
// Cast through unknown because the strict input type already
// requires rationale; this helper exists to handle the legacy
// mismatch defensively at runtime.
const legacy = {
summary: "Older insight from v1.4.15.",
recommendations: [
{
id: "rec-old-1",
text: "Walk more",
severity: "suggestion",
metricSource: {
type: "activity",
timeRange: "last7days",
summary: "5,000 steps avg",
},
// no rationale
},
{
id: "rec-old-2",
text: "Hydrate",
severity: "info",
metricSource: {
type: "fluid",
timeRange: "last7days",
summary: "low intake logged",
},
// no rationale
},
],
citations: [],
warnings: [],
} as unknown as AIInsightResponse;
const missing = findRecommendationsMissingRationale(legacy);
expect(missing).toEqual(["rec-old-1", "rec-old-2"]);
});
it("flags only the recs that are missing, not the well-formed ones", () => {
const partial = {
summary: "Mixed payload",
recommendations: [
baseRec,
{
id: "rec-bad",
text: "Hydrate",
severity: "info",
metricSource: {
type: "fluid",
timeRange: "last7days",
summary: "low intake logged",
},
// no rationale
},
],
citations: [baseCitation],
warnings: [],
} as unknown as AIInsightResponse;
expect(findRecommendationsMissingRationale(partial)).toEqual(["rec-bad"]);
});
});
src/lib/ai/__tests__/rationale-wrapper.test.ts
-172
View File
@@ -1,172 +0,0 @@
import { describe, it, expect, beforeEach, vi } from "vitest";
import { generateInsight } from "../generate-insight";
import { MockAIProvider } from "../mock-client";
import { singleUserTurn } from "../types";
const annotateMock = vi.fn();
vi.mock("@/lib/logging/context", () => ({
annotate: (fields: { meta?: Record<string, unknown> }) =>
annotateMock(fields),
getEvent: () => null,
}));
/**
* v1.4.16 phase B5c — wrapper-level rationale handling.
*
* 1. The corrective retry message lists the rationale fields so a
* first-attempt response without rationale gets a targeted
* reprompt (instead of the model guessing what's wrong).
* 2. The wrapper's post-validation `annotate()` carries a
* `ai_rationale_coverage_*` breakdown so the admin AI quality
* dashboard can chart per-payload coverage of the new field.
* Mirrors the citation-coverage annotations from B5a.
*/
const baseRationale = {
dataWindow: "last7days" as const,
comparedTo: "your 90-day median (122/78)",
deviation: "+16/+8 mmHg above baseline over 9 of 9 readings",
};
const baseMetricSource = {
type: "bloodPressure",
timeRange: "last7days",
summary: "avg 138/86 across 9 readings",
};
const baseCitation = {
type: "bloodPressure",
timeRange: "last7days",
summary: "avg 138/86 across 9 readings",
};
const validResponse = {
summary: "BP runs slightly above your 90-day median.",
recommendations: [
{
id: "rec-1",
text: "Discuss home BP log with your physician.",
severity: "important",
metricSource: baseMetricSource,
rationale: baseRationale,
},
],
citations: [baseCitation],
warnings: [],
};
describe("generateInsight() — corrective retry mentions rationale", () => {
beforeEach(() => {
annotateMock.mockReset();
});
it("retries with a corrective message naming dataWindow / comparedTo / deviation when rationale missing", async () => {
const recWithoutRationale = {
id: "rec-1",
text: "Walk more",
severity: "suggestion",
metricSource: baseMetricSource,
// rationale missing
};
const broken = {
...validResponse,
recommendations: [recWithoutRationale],
};
const provider = new MockAIProvider({
responses: [JSON.stringify(broken), JSON.stringify(validResponse)],
});
const outcome = await generateInsight(
provider,
singleUserTurn({ system: "sys", user: "Original user prompt." }),
);
expect(outcome.attempts).toBe(2);
expect(outcome.retried).toBe(true);
const retryUserPrompt = provider.calls[1].messages[0].content as string;
expect(retryUserPrompt).toContain("Original user prompt.");
expect(retryUserPrompt).toContain("rationale");
expect(retryUserPrompt).toContain("dataWindow");
expect(retryUserPrompt).toContain("comparedTo");
expect(retryUserPrompt).toContain("deviation");
});
it("retries when first attempt has empty rationale.deviation", async () => {
const recWithEmptyDeviation = {
id: "rec-1",
text: "Walk more",
severity: "suggestion",
metricSource: baseMetricSource,
rationale: {
dataWindow: "last7days",
comparedTo: "your 90-day median",
deviation: "", // empty — should fail
},
};
const broken = {
...validResponse,
recommendations: [recWithEmptyDeviation],
};
const provider = new MockAIProvider({
responses: [JSON.stringify(broken), JSON.stringify(validResponse)],
});
const outcome = await generateInsight(
provider,
singleUserTurn({ system: "sys", user: "user" }),
);
expect(outcome.attempts).toBe(2);
expect(outcome.retried).toBe(true);
});
});
describe("generateInsight() — rationale-coverage annotation", () => {
beforeEach(() => {
annotateMock.mockReset();
});
it("annotates ai_rationale_coverage_* fields on a successful generation", async () => {
const provider = new MockAIProvider({
responses: JSON.stringify(validResponse),
});
await generateInsight(
provider,
singleUserTurn({ system: "sys", user: "user" }),
);
expect(annotateMock).toHaveBeenCalled();
const meta = annotateMock.mock.calls[0][0]?.meta as Record<string, unknown>;
expect(meta).toMatchObject({
ai_total_recommendations: 1,
ai_rationale_total_recommendations: 1,
ai_rationale_with_rationale: 1,
ai_rationale_missing_recommendation_ids: [],
});
});
it("rationale-coverage = total recommendations when every rec carries one", async () => {
const twoRecs = {
...validResponse,
recommendations: [
validResponse.recommendations[0],
{
id: "rec-2",
text: "Continue daily logging.",
severity: "info",
metricSource: baseMetricSource,
rationale: baseRationale,
},
],
};
const provider = new MockAIProvider({
responses: JSON.stringify(twoRecs),
});
await generateInsight(
provider,
singleUserTurn({ system: "sys", user: "user" }),
);
const meta = annotateMock.mock.calls[0][0]?.meta as Record<string, unknown>;
expect(meta).toMatchObject({
ai_rationale_total_recommendations: 2,
ai_rationale_with_rationale: 2,
ai_rationale_missing_recommendation_ids: [],
});
});
});
src/lib/ai/citation-coverage.ts
+15 -7
View File
@@ -1,12 +1,20 @@
/**
* Reachability note. This module was written for `generateInsight()`, whose
* only caller is `runWithFallback()` — and `runWithFallback()` has no
* production caller left (every live surface routes through
* `runRawCompletionWithFallback` instead). The coverage annotation therefore
* never executed in production: an audit surface that implied enforcement it
* did not perform. Rather than delete working, tested logic, the input type is
* widened to the structural shape both payloads share and the comprehensive
* post-parse step now calls it, so the admin quality dashboard sees real data.
* only caller was `runWithFallback()` — and that had no production caller,
* because every live surface routes through `runRawCompletionWithFallback`.
* The coverage annotation therefore never executed: an audit surface that
* implied enforcement it did not perform.
*
* v1.30.24 kept the logic and widened the input type to the structural shape
* both payloads share, so the comprehensive post-parse step could call it and
* the admin quality dashboard could see real data. The dead wrapper and chain
* runner around it have since been removed outright, so this module now has
* exactly one caller and it is a live one.
*
* Observational by design: an uncited normative recommendation is an
* annotation, never a parse failure. It is a METRIC, not a gate — the citation
* ENFORCEMENT that once sat beside it is gone rather than dormant, so nothing
* here should be read as withholding output.
*/
/**
src/lib/ai/confidence.ts
-102
View File
@@ -1,102 +0,0 @@
/**
* v1.4.16 phase B5d — deterministic confidence-score computation per
* recommendation.
*
* The maintainer's mandate: the model's self-reported confidence is unreliable
* (calibrated probabilities are not a small-LLM strength), so the
* wrapper OVERRIDES with a server-computed value derived from three
* inputs that the cited data window actually exposes:
*
* - n sample count behind the cited data window
* (`metricSource.n`).
* - recencyDays days since the most recent sample in the window;
* a 7-day rec built off data 25 days old should
* not feel as confident as the same rec built off
* data from this morning.
* - deviationStdRatio |deviation| / stdev of the user's 90-day
* baseline. `null` when the baseline is too thin
* to compute a stdev — we fall back to a neutral
* mid-band score rather than zero.
*
* Score components (sum, max 100):
*
* - n contribution: log-saturating curve, capped at 40.
* n<3 hard-caps the whole score at ≤15 (5 * n) — three samples is
* the hard floor for any quantitative claim per research §2.A
* (Apple's "be quiet when unsure" stance).
*
* - recency contribution: 30 at <2d old, decays linearly to 0 at
* 30d, floored at 0. The shape matches the Oura "Pay Attention"
* band — old data is still data, just stale.
*
* - signal contribution: 30 at |z|≥1.5 (clearly out-of-band),
* scaled linearly down to 0 at z=0. When ratio is null (no
* baseline yet), we award 15 — neither punishing the user for
* thin history nor over-claiming a strong deviation we can't
* measure.
*
* The formula is pre-feedback. v1.4.17 will fit weights to thumbs-up
* rates (B5e feedback ratchet); for v1.4.16 the formula is fixed and
* deterministic so admin observability + the "draft pill below 25"
* UI affordance are reproducible across runs.
*/
export interface ConfidenceInputs {
/** Sample count behind the cited data window (from metricSource.n). */
n: number;
/** Days since the most recent sample in the window. */
recencyDays: number;
/** |deviation| / stdev of the user's 90-day baseline, when computable. */
deviationStdRatio: number | null;
}
const HARD_CAP_FLOOR = 10;
const N_CONTRIBUTION_MAX = 40;
const RECENCY_CONTRIBUTION_MAX = 30;
const RECENCY_DECAY_DAYS = 30;
const SIGNAL_CONTRIBUTION_MAX = 30;
const SIGNAL_SATURATION_RATIO = 1.5;
const SIGNAL_NULL_NEUTRAL = 15;
/**
* Compute the deterministic confidence score (0..100) for a single
* recommendation. Pure function — no I/O, no clock; safe to call from
* test fixtures and from `generateInsight()` post-validation alike.
*/
export function computeConfidence(inputs: ConfidenceInputs): number {
const { n, recencyDays, deviationStdRatio } = inputs;
// n hard cap: <3 samples means we cannot make a quantitative claim,
// so the whole score is bounded at 5*n (max 15 at n=2). Floor at
// 10 so n=0/n=1 still produce a non-zero "draft" signal — the UI
// shows the rec but tags it as low-confidence rather than hiding it.
if (n < 3) {
return Math.max(HARD_CAP_FLOOR, 5 * n);
}
// n contribution: log10 saturates so a 100-sample window doesn't
// outweigh a 30-sample window by 3×; both are "enough data" for the
// user-level signal we care about.
const nScore = Math.min(N_CONTRIBUTION_MAX, 10 + 10 * Math.log10(n));
// Recency contribution: <2d ≈ full score (30 * (1 - 0/30) = 30,
// 30 * (1 - 1/30) = 29; both round into the meter's top band).
// 30d+ floors to 0 so an ancient window cannot inflate the score.
const recencyScore = Math.max(
0,
RECENCY_CONTRIBUTION_MAX * (1 - recencyDays / RECENCY_DECAY_DAYS),
);
// Signal contribution: null ratio (no baseline) → neutral 15.
// Otherwise scale |z| linearly to the SIGNAL_SATURATION_RATIO cap.
const signalScore =
deviationStdRatio === null
? SIGNAL_NULL_NEUTRAL
: Math.min(
SIGNAL_CONTRIBUTION_MAX,
SIGNAL_CONTRIBUTION_MAX *
(Math.abs(deviationStdRatio) / SIGNAL_SATURATION_RATIO),
);
return Math.round(nScore + recencyScore + signalScore);
}
src/lib/ai/generate-insight.ts
-279
View File
@@ -1,279 +0,0 @@
import type { z } from "zod/v4";
import {
appendToLastUserMessage,
type AIProvider,
type CompletionParams,
type CompletionResult,
} from "./types";
import {
aiInsightResponseSchema,
findRecommendationsMissingRationale,
findUncitedRecommendations,
InsightSchemaError,
type AIInsightResponse,
} from "./schema";
import { computeCitationCoverage } from "./citation-coverage";
import { computeConfidence } from "./confidence";
import { annotate } from "@/lib/logging/context";
/**
* Schema-enforced wrapper around `AIProvider.generateCompletion()`.
*
* Phase C1 (v1.4.15): the maintainer's mandate ("zero hallucinations, must
* ground on user data") means a free-form LLM response is no longer
* acceptable. This wrapper:
*
* 1. Calls the provider once.
* 2. Parses the response as JSON.
* 3. Validates it against `aiInsightResponseSchema` AND cross-checks
* that every recommendation's `metricSource` appears in
* `citations[]`.
* 4. On failure: retries ONCE, prepending a corrective system
* message that includes the violated zod issues.
* 5. On second failure: throws `InsightSchemaError` (httpStatus 422).
*
* The wrapper does not concern itself with provider-level retries
* (HTTP 5xx, 401 token-refresh, etc.) — each provider client handles
* those internally. This wrapper retries only on schema-violation.
*/
export interface GenerateInsightOutcome {
parsed: AIInsightResponse;
raw: CompletionResult;
/** Number of provider calls made (1 on success, 2 on retry). */
attempts: number;
/** True iff the retry attempt produced the parsed response. */
retried: boolean;
}
/**
* Build the corrective user-prompt suffix that explains to the model
* what went wrong on the first attempt. Includes the violated zod
* issues verbatim (truncated to 1KB) so the model can self-correct.
*/
export function buildRetryCorrectionMessage(
reason: string,
details: string,
): string {
return `
Your previous response did not satisfy the required JSON schema.
Reason: ${reason}
Details: ${details.slice(0, 1024)}
You MUST return a JSON object matching this schema exactly. Do NOT
include prose, markdown fences, or commentary outside the JSON.
Required top-level fields:
- summary: string (2-3 sentences)
- recommendations: array of objects with
{ id: string,
text: string,
severity: "info" | "suggestion" | "important" | "urgent",
metricSource: { type: string, timeRange: string, summary: string, n?: number },
rationale: { dataWindow: "last7days" | "last30days" | "last90days" | "allTime",
comparedTo: non-empty string,
deviation: non-empty string } }
- citations: array of objects with { type: string, timeRange: string, summary: string }
- warnings: array of objects with { topic: string, message: string, severity?: same enum }
Every recommendation's metricSource MUST also appear in citations[]
(same type + timeRange). Every recommendation MUST carry a
rationale object — dataWindow, comparedTo, and deviation are all
required and rationale.dataWindow MUST equal metricSource.timeRange.
If you cannot ground a recommendation in user data from the
snapshot you were given, OMIT it. An empty recommendations[] is
acceptable.
`;
}
interface TryOutcome {
ok: boolean;
parsed?: AIInsightResponse;
raw: CompletionResult;
reason?: string;
details?: string;
zodIssues?: z.ZodIssue[];
}
async function tryOnce(
provider: AIProvider,
params: CompletionParams,
): Promise<TryOutcome> {
const raw = await provider.generateCompletion(params);
// Parse JSON.
let json: unknown;
try {
json = JSON.parse(raw.content);
} catch (e) {
return {
ok: false,
raw,
reason: "Response was not valid JSON",
details: (e as Error).message,
};
}
// Schema validate.
const result = aiInsightResponseSchema.safeParse(json);
if (!result.success) {
return {
ok: false,
raw,
reason: "Response did not match the required JSON schema",
details: result.error.issues
.slice(0, 8)
.map((i) => `${i.path.join(".")}: ${i.message}`)
.join("; "),
zodIssues: result.error.issues,
};
}
// Cross-check citations.
const uncited = findUncitedRecommendations(result.data);
if (uncited.length > 0) {
const list = uncited
.slice(0, 4)
.map(
(u) =>
`${u.recommendationId} → ${u.missing.type}/${u.missing.timeRange}`,
)
.join(", ");
return {
ok: false,
raw,
reason: "Recommendations referenced metricSources not in citations[]",
details: `Missing citations: ${list}`,
};
}
return { ok: true, parsed: result.data, raw };
}
/**
* Forward the citation-coverage breakdown into the active Wide-Event
* via `annotate()` (no-op outside a request context). Keys are
* snake_case to match the rest of the AI Wide-Event vocabulary.
*
* v1.4.16 phase B5c — additionally emit `ai_rationale_*` keys so the
* admin AI quality dashboard can chart rationale-coverage per
* payload alongside citation-coverage. Rationale is required by the
* strict schema; the missing-ids list will only ever populate when
* a legacy payload sneaks through `.passthrough()` — useful as an
* observability tripwire for the migration window.
*/
function annotateCitationCoverage(parsed: AIInsightResponse): void {
const coverage = computeCitationCoverage(parsed);
const rationaleMissing = findRecommendationsMissingRationale(parsed);
annotate({
meta: {
ai_total_recommendations: coverage.totalRecommendations,
ai_normative_recommendations: coverage.normativeRecommendations,
ai_cited_normative_recommendations:
coverage.citedNormativeRecommendations,
ai_uncited_normative_recommendation_ids:
coverage.uncitedNormativeRecommendationIds,
ai_rationale_total_recommendations: coverage.totalRecommendations,
ai_rationale_with_rationale:
coverage.totalRecommendations - rationaleMissing.length,
ai_rationale_missing_recommendation_ids: rationaleMissing,
},
});
}
/**
* Apply the deterministic confidence override to every recommendation.
* The model's value (if any) is REPLACED. Inputs are derived from the
* parsed payload alone (n from `metricSource.n`, recencyDays=0,
* deviationStdRatio=null) — the route layer doesn't pass bucket-series
* data through here today. v1.4.17's feedback ratchet will need a
* resolver injection; re-introduce one when the second caller exists.
*
* Emits a Wide-Event with per-rec model-vs-computed pairs so admin
* observability can chart drift between what the LLM thinks and what
* the formula produces.
*/
function applyConfidenceOverride(parsed: AIInsightResponse): void {
const overrides: Array<{
id: string;
model: number | null;
computed: number;
}> = [];
for (const rec of parsed.recommendations) {
const computed = computeConfidence({
n: rec.metricSource.n ?? 0,
recencyDays: 0,
deviationStdRatio: null,
});
const modelValue =
typeof rec.confidence === "number" ? rec.confidence : null;
rec.confidence = computed;
overrides.push({ id: rec.id, model: modelValue, computed });
}
if (overrides.length > 0) {
annotate({
meta: {
ai_confidence_override_event: "confidence.override.applied",
ai_confidence_override_count: overrides.length,
ai_confidence_override_pairs: overrides,
},
});
}
}
/**
* Run a schema-enforced insight generation. Up to 2 provider calls.
* Throws `InsightSchemaError` (httpStatus 422) if both attempts fail
* the schema; provider-level errors bubble untouched.
*
* v1.4.16 phase B5d — after a successful parse, every recommendation's
* `confidence` field is overwritten with the deterministic
* `computeConfidence()` value. The model's number (if any) is
* discarded; admin observability sees both via the
* `confidence.override.applied` Wide-Event annotation.
*/
export async function generateInsight(
provider: AIProvider,
params: CompletionParams,
): Promise<GenerateInsightOutcome> {
const first = await tryOnce(provider, params);
if (first.ok && first.parsed) {
annotateCitationCoverage(first.parsed);
applyConfidenceOverride(first.parsed);
return {
parsed: first.parsed,
raw: first.raw,
attempts: 1,
retried: false,
};
}
// Retry-once with corrective context appended to the user prompt.
const correction = buildRetryCorrectionMessage(
first.reason ?? "schema mismatch",
first.details ?? "",
);
const retryParams: CompletionParams = appendToLastUserMessage(
params,
`\n\n${correction}`,
);
const second = await tryOnce(provider, retryParams);
if (second.ok && second.parsed) {
annotateCitationCoverage(second.parsed);
applyConfidenceOverride(second.parsed);
return {
parsed: second.parsed,
raw: second.raw,
attempts: 2,
retried: true,
};
}
throw new InsightSchemaError(
second.reason ?? first.reason ?? "Schema validation failed twice",
{
issues: second.zodIssues ?? first.zodIssues,
attempts: 2,
},
);
}
src/lib/ai/prompts/insight-generator.ts
+1 -1
View File
@@ -986,7 +986,7 @@ const INSIGHTS_LOCALE_REPLY_FOOTER_FALLBACK: Record<
/**
* Returns the active scope-hardened system prompt for a given locale.
* The route consumes this via `generateInsight()`.
* Consumed by the comprehensive generation path.
*/
export function getStrictInsightsSystemPrompt(locale: Locale): string {
if (locale === "de") return SYSTEM_PROMPT_DE;
src/lib/ai/provider-runner.ts
+17 -120
View File
@@ -1,9 +1,4 @@
import type { AIProvider, CompletionParams, CompletionResult } from "./types";
import {
generateInsight,
type GenerateInsightOutcome,
} from "./generate-insight";
import { InsightSchemaError } from "./schema";
import type { ProviderChainType } from "./provider-chain";
import {
postgresProviderHealthLedger,
@@ -25,19 +20,17 @@ const NEVER_SKIP: ReadonlySet<ProviderChainType> = new Set(["local"]);
/**
* v1.4.16 phase B5b — multi-provider redundancy runner.
*
* The chain runner wraps `generateInsight()` with a hard-failure
* The chain runner wraps a provider completion with a hard-failure
* walkthrough across an ordered list of providers. The motivation is
* the maintainer's "ein Provider-Ausfall darf die Insights nicht zerschiessen"
* mandate: a 401 from Codex (OAuth expired), a 503 from OpenAI
* (upstream brown-out), or a network reset MUST cascade to the next
* configured provider rather than 422-ing the whole request.
*
* Hard failures (`isHardProviderFailure`) cascade. Schema failures
* (the existing 422 surface) do NOT cascade — they bubble through
* `InsightSchemaError` so the user still sees the helpful "the model
* returned malformed JSON" diagnostic. Walking on schema errors would
* mask provider-specific prompt-following issues that the v1.4.15
* citation-enforcement guard exists to surface.
* Hard failures (`isHardProviderFailure`) cascade. A malformed-JSON
* reply is NOT a hard failure — it is a prompt-following issue, so it
* bubbles to the caller's own validator rather than masking itself as
* provider unavailability and walking the chain.
*
* Cross-feature coupling:
* - B5a's `MEDICAL_REFERENCES` is shared via the system prompt; each
@@ -120,9 +113,9 @@ function rememberWorkingProvider(
* - Network errors (no `httpStatus`) — DNS, ECONNRESET, timeout.
*
* No:
* - `InsightSchemaError` — surfaces the wrapper's existing 422 so the
* user sees "model JSON malformed" rather than getting silent
* provider drift. This is the ONLY non-hard class.
* - Anything with a non-error status. Callers that parse a structured
* reply raise their own validation error, which carries no
* `httpStatus` from a provider wire and is theirs to surface.
*
* v1.21.3 — a 4xx that the PROVIDER WIRE returned (tagged `upstream`,
* e.g. a Codex `400` rejecting the tool-call / structured request shape)
@@ -132,12 +125,9 @@ function rememberWorkingProvider(
* provider error rethrew as an HTTP 500. Treating every upstream 4xx as
* hard means it cascades to the next provider and, on exhaustion, is
* wrapped in `AllProvidersFailedError` so the route surfaces a graceful
* `coach.provider.*` frame instead of a 500. `InsightSchemaError`
* (no `upstream` tag) is still excluded, so the strict JSON surface keeps
* its 422 "model JSON malformed" diagnostic.
* `coach.provider.*` frame instead of a 500.
*/
export function isHardProviderFailure(error: unknown): boolean {
if (error instanceof InsightSchemaError) return false;
const err = error as { httpStatus?: number; name?: string } | null;
const status = err?.httpStatus;
if (typeof status !== "number" || status <= 0) {
@@ -169,28 +159,6 @@ export interface FallbackHop {
httpStatus: number | null;
}
export interface RunWithFallbackParams {
userId: string;
providers: ProviderChainResolved[];
params: CompletionParams;
/**
* v1.11.0 W1 — durable provider-health ledger. Defaults to the
* Postgres-backed implementation in production; unit tests of the pure
* chain pass an in-memory / no-op ledger to avoid standing up a DB.
* The ledger is read-through (skip-hint reorder) + write-through
* (record outcome); it never gates generation — a ledger error always
* fails open to today's behaviour.
*/
ledger?: ProviderHealthLedger;
}
export interface RunWithFallbackResult extends GenerateInsightOutcome {
/** Provider that actually produced the parsed response. */
workingProvider: ProviderChainResolved;
/** Hops that failed before the working provider succeeded. */
fallbackHops: FallbackHop[];
}
/**
* Hard-fail wrapper around `AllProvidersFailedError.attempts` so the
* route layer can pick a single representative status (the worst of
@@ -392,78 +360,6 @@ export function classifyErrorBody(body: unknown): string | null {
return `unclassified:${body.length}chars`;
}
/**
* Run an insight generation across an ordered chain of providers,
* walking past hard provider failures. Schema-class failures (422)
* bubble unchanged from the underlying wrapper.
*/
export async function runWithFallback(
args: RunWithFallbackParams,
): Promise<RunWithFallbackResult> {
const { userId, providers, params } = args;
const ledger = args.ledger ?? postgresProviderHealthLedger;
if (providers.length === 0) {
throw new AllProvidersFailedError([]);
}
const ordered = await resolveChainOrder(userId, providers, ledger);
const hops: FallbackHop[] = [];
for (let i = 0; i < ordered.length; i += 1) {
const candidate = ordered[i];
try {
const outcome = await generateInsight(candidate.instance, params);
rememberWorkingProvider(userId, candidate.providerType);
void ledger.recordSuccess(userId, candidate.providerType);
annotate({
meta: {
ai_chain_working_provider: candidate.providerType,
ai_chain_fallback_count: hops.length,
},
});
return {
...outcome,
workingProvider: candidate,
fallbackHops: hops,
};
} catch (error) {
if (!isHardProviderFailure(error)) {
// Schema/validation error — bubble immediately. NOT recorded in
// the health ledger: a malformed-JSON reply is a prompt-following
// issue, not a provider-availability failure.
throw error;
}
const summary = summariseError(error);
void ledger.recordFailure(userId, candidate.providerType, summary.status);
const hop: FallbackHop = {
providerType: candidate.providerType,
attempt: i + 1,
failureReason: summary.reason,
httpStatus: summary.status,
};
hops.push(hop);
annotate({
meta: {
[`ai_chain_hop_${i + 1}_provider`]: candidate.providerType,
[`ai_chain_hop_${i + 1}_status`]: summary.status,
[`ai_chain_hop_${i + 1}_reason`]: summary.reason.slice(0, 240),
[`ai_chain_hop_${i + 1}_body`]: summary.bodyExcerpt,
},
});
}
}
// Every chain entry hard-failed.
annotate({
meta: {
ai_chain_outcome: "all-failed",
ai_chain_fallback_count: hops.length,
},
});
throw new AllProvidersFailedError(hops);
}
/**
* Shared chain walker for the two RAW runners (streaming + buffered). Owns the
* one set of chain / fallback / health-ledger semantics — empty-chain guard,
@@ -593,14 +489,15 @@ export interface RunRawWithFallbackResult {
}
/**
* Sibling of `runWithFallback()` that calls `provider.generateCompletion`
* directly (no strict-schema wrapper). Used by the legacy route at
* `/api/insights/generate` which still consumes the rich legacy shape.
* Buffered runner: calls `provider.generateCompletion` directly and hands the
* raw reply back for the caller to validate. This is the ONLY buffered chain
* runner — a strict-schema sibling (`runWithFallback`) once sat beside it but
* had no production caller and was removed.
*
* Fallback policy is identical: hard provider failures cascade,
* everything else (e.g. a 4xx-but-non-auth from a custom provider)
* bubbles to the caller. The cache + structured logging are shared
* with the strict variant.
* Fallback policy: hard provider failures cascade, everything else (e.g. a
* 4xx-but-non-auth from a custom provider) bubbles to the caller. The
* last-working cache + structured logging are shared with the streaming
* sibling via `runRawChain`.
*/
export async function runRawCompletionWithFallback(args: {
userId: string;
src/lib/ai/retry-correction.ts
+60
View File
@@ -0,0 +1,60 @@
/**
* The corrective retry prompt for a structured-output generation that came
* back malformed.
*
* History worth keeping, because the shape of this module is a consequence of
* it: this file used to hold `generateInsight()`, a schema-enforced wrapper
* that parsed a reply against the strict `aiInsightResponseSchema`,
* cross-checked that every recommendation's `metricSource` appeared in
* `citations[]`, retried once with the message below, and threw
* `InsightSchemaError` on a second failure. That wrapper's only caller was
* `runWithFallback()`, which itself had no production caller — every live
* surface runs `runRawCompletionWithFallback` and validates with its own
* schema. So the enforcement never executed. It was removed rather than wired
* in: the live `insightResultSchema` has no `citations[]` field at all and
* treats `metricSource` as optional, so applying the strict cross-check to a
* live payload would have rejected essentially every generation.
*
* What survived is the part that was always reachable — the corrective
* message, which `comprehensive-generate.ts` sends on its own JSON-retry leg.
*/
/**
* Build the corrective user-prompt suffix that explains to the model
* what went wrong on the first attempt. Includes the violated zod
* issues verbatim (truncated to 1KB) so the model can self-correct.
*/
export function buildRetryCorrectionMessage(
reason: string,
details: string,
): string {
return `
Your previous response did not satisfy the required JSON schema.
Reason: ${reason}
Details: ${details.slice(0, 1024)}
You MUST return a JSON object matching this schema exactly. Do NOT
include prose, markdown fences, or commentary outside the JSON.
Required top-level fields:
- summary: string (2-3 sentences)
- recommendations: array of objects with
{ id: string,
text: string,
severity: "info" | "suggestion" | "important" | "urgent",
metricSource: { type: string, timeRange: string, summary: string, n?: number },
rationale: { dataWindow: "last7days" | "last30days" | "last90days" | "allTime",
comparedTo: non-empty string,
deviation: non-empty string } }
- citations: array of objects with { type: string, timeRange: string, summary: string }
- warnings: array of objects with { topic: string, message: string, severity?: same enum }
Every recommendation's metricSource MUST also appear in citations[]
(same type + timeRange). Every recommendation MUST carry a
rationale object — dataWindow, comparedTo, and deviation are all
required and rationale.dataWindow MUST equal metricSource.timeRange.
If you cannot ground a recommendation in user data from the
snapshot you were given, OMIT it. An empty recommendations[] is
acceptable.
`;
}
src/lib/ai/schema.ts
+20 -91
View File
@@ -18,7 +18,7 @@ import { MEDICAL_REFERENCE_IDS } from "./medical-references";
* - A `citations[]` array referencing the metrics the response drew
* on. May be empty when `recommendations[]` is empty (no data → no
* citation), but every recommendation's `metricSource` MUST also
* appear in `citations[]` (cross-check enforced by `validateInsightResponse`).
* appear in `citations[]`.
* - A `warnings[]` array for guideline-flagged values (high BP,
* critical pulse, etc.). Free-form for now; v1.4.16 adds severity
* enum + medical-reference grounding.
@@ -137,9 +137,8 @@ export const aiRecommendationSchema = z
/**
* v1.4.16 phase B5c — per-recommendation explainability rationale.
* Required on every rec so the UI's expand-card always has WHY +
* WINDOW + COMPARED-TO content to render. Legacy payloads (pre-
* B5c) ride through `findRecommendationsMissingRationale()` for
* the regenerate-CTA migration; the strict parser rejects them.
* WINDOW + COMPARED-TO content to render. The strict parser
* rejects a legacy (pre-B5c) payload that omits it.
*/
rationale: aiRecommendationRationaleSchema,
/**
@@ -159,11 +158,9 @@ export const aiRecommendationSchema = z
*
* Optional at parse-time:
* - The LLM may emit a value (we accept it so the payload round-
* trips cleanly), but `generateInsight()` OVERRIDES with the
* server-computed `computeConfidence()` post-validation. The
* model's number is discarded — calibrated probabilities are
* not a small-LLM strength and the deterministic path keeps
* the v1.4.17 feedback ratchet reproducible.
* trips cleanly). A deterministic server-side override once
* replaced it post-validation, but that path never ran in
* production and was removed with the rest of the wrapper.
* - Legacy cached payloads from before B5d landed have no
* confidence field; the meter falls back to a "draft" pill.
*
@@ -507,87 +504,19 @@ export const aiInsightResponseSchema = z
export type AIInsightResponse = z.infer<typeof aiInsightResponseSchema>;
/**
* Tagged error thrown by the schema-enforcement wrapper when even the
* retry attempt fails to produce a valid payload. The route catches
* this and surfaces a 422 to the client.
*/
export class InsightSchemaError extends Error {
readonly httpStatus = 422;
readonly issues: z.ZodIssue[] | null;
readonly attempts: number;
constructor(
message: string,
options: { issues?: z.ZodIssue[]; attempts: number },
) {
super(message);
this.name = "InsightSchemaError";
this.issues = options.issues ?? null;
this.attempts = options.attempts;
}
}
/**
* v1.4.16 phase B5c — legacy-payload detector.
* The strict-contract ENFORCEMENT that used to live here is gone: an
* `InsightSchemaError` class no caller threw, a `findUncitedRecommendations`
* cross-check (every recommendation's `metricSource` must appear in
* `citations[]`) and a `findRecommendationsMissingRationale` legacy-payload
* detector. All three existed only for `generateInsight()`, whose sole caller
* `runWithFallback()` had no production caller — so none of them ever ran.
*
* The strict schema now requires `rationale` on every recommendation.
* Cached payloads from v1.4.14/v1.4.15 predate the field — they would
* fail `aiInsightResponseSchema.safeParse()` outright. Rather than
* auto-regenerating on every read (expensive + surprising), we let
* the route call this helper against the legacy-shape JSON so the
* UI can show a "Insights updated — regenerate for new explainability
* features" CTA. User-initiated regeneration stays the trigger.
*
* Input is intentionally typed loosely (the runtime shape is the
* canonical AIInsightResponse, but legacy payloads omit `rationale`
* which the static type now requires). Returns the ids of
* recommendations missing rationale; an empty array means the
* payload is well-shaped under B5c.
* The schema above stays because it is not a guard: it is the shape the
* corrective retry prompt (`retry-correction.ts`) describes to the model, and
* it is the reference definition for the strict payload. NOTE the live
* comprehensive path validates against the looser `insightResultSchema` in
* `types.ts`, which has no `citations[]` and treats `metricSource` as
* optional — so the retry prompt asks for a stricter shape than the live
* validator requires. That mismatch predates this cleanup and is left alone
* deliberately; changing the retry prompt changes model output.
*/
export function findRecommendationsMissingRationale(
parsed: AIInsightResponse,
): string[] {
const missing: string[] = [];
for (const rec of parsed.recommendations) {
// Defensive runtime check — the static type asserts rationale is
// present, but the helper exists precisely for legacy payloads
// where it isn't.
const r = (rec as { rationale?: unknown }).rationale;
if (r === undefined || r === null) {
missing.push(rec.id);
}
}
return missing;
}
/**
* Cross-validation beyond zod: every recommendation's `metricSource`
* must also appear in `citations[]` (same `type` + `timeRange`).
* Returns the list of missing-citation issues; an empty list means
* the response is internally consistent.
*/
export function findUncitedRecommendations(parsed: AIInsightResponse): Array<{
recommendationId: string;
missing: { type: string; timeRange: string };
}> {
const cited = new Set(
parsed.citations.map((c) => `${c.type}::${c.timeRange}`),
);
const missing: Array<{
recommendationId: string;
missing: { type: string; timeRange: string };
}> = [];
for (const rec of parsed.recommendations) {
const key = `${rec.metricSource.type}::${rec.metricSource.timeRange}`;
if (!cited.has(key)) {
missing.push({
recommendationId: rec.id,
missing: {
type: rec.metricSource.type,
timeRange: rec.metricSource.timeRange,
},
});
}
}
return missing;
}
src/lib/insights/comprehensive-generate.ts
+3 -3
View File
@@ -53,7 +53,7 @@ import {
buildSystemPromptWithReferences,
buildBriefingPersonalisationBlock,
} from "@/lib/ai/prompts/insight-generator";
import { buildRetryCorrectionMessage } from "@/lib/ai/generate-insight";
import { buildRetryCorrectionMessage } from "@/lib/ai/retry-correction";
import { singleUserTurn } from "@/lib/ai/types";
import {
buildAboutMeInsightBlock,
@@ -1042,8 +1042,8 @@ export async function generateComprehensiveInsight(
// Citation coverage. This annotation is what the admin AI-quality dashboard
// slices to track how many normative recommendations carry a curated
// medical-reference id. It used to hang off `generateInsight()`, which has no
// production caller, so the dashboard was reading a metric nothing emitted.
// medical-reference id. It used to hang off a wrapper that had no production
// caller, so the dashboard was reading a metric nothing emitted.
// Observational only — an uncited normative rec is a logged warning, never a
// reason to fail the generation.
if (insights && typeof insights === "object") {