diff --git a/src/lib/ai/__tests__/retry-correction.test.ts b/src/lib/ai/__tests__/retry-correction.test.ts new file mode 100644 index 000000000..681a2b062 --- /dev/null +++ b/src/lib/ai/__tests__/retry-correction.test.ts @@ -0,0 +1,87 @@ +/** + * The corrective retry prompt and the validator that grades the retry reply + * must describe ONE contract. + * + * The prompt used to demand the dead strict wrapper's shape — `citations[]`, + * `warnings[]`, and a mandatory `metricSource` / `rationale` / `id` / + * `severity` on every recommendation — while naming none of the five fields + * `insightResultSchema` actually requires. A model that obeyed the correction + * produced a reply the validator could not accept, so the one corrective pass + * spent budget steering the generation away from its own contract. + * + * These tests pin both directions: the example the prompt hands the model + * validates, and the prompt names every required key without naming a key the + * validator does not have. + */ +import { describe, it, expect } from "vitest"; +import { z } from "zod/v4"; + +import { + buildRetryCorrectionMessage, + RETRY_CONTRACT_EXAMPLE, +} from "@/lib/ai/retry-correction"; +import { insightResultSchema } from "@/lib/ai/types"; + +/** Top-level keys `insightResultSchema` requires (no `.optional()`). */ +function requiredSchemaKeys(): string[] { + const shape = (insightResultSchema as unknown as z.ZodObject).shape; + return Object.entries(shape) + .filter(([, field]) => !(field as z.ZodType).safeParse(undefined).success) + .map(([key]) => key); +} + +/** Every top-level key the schema knows about, required or optional. */ +function allSchemaKeys(): string[] { + return Object.keys((insightResultSchema as unknown as z.ZodObject).shape); +} + +describe("buildRetryCorrectionMessage — prompt matches the validator", () => { + it("hands the model an example that the validator accepts", () => { + const parsed = insightResultSchema.safeParse(RETRY_CONTRACT_EXAMPLE); + expect(parsed.success).toBe(true); + }); + + it("embeds that example verbatim in the corrective message", () => { + const message = buildRetryCorrectionMessage("reason", "details"); + expect(message).toContain(JSON.stringify(RETRY_CONTRACT_EXAMPLE, null, 2)); + }); + + it("names every field the validator requires", () => { + const message = buildRetryCorrectionMessage("reason", "details"); + for (const key of requiredSchemaKeys()) { + expect(message, `retry prompt omits required field "${key}"`).toContain( + key, + ); + } + }); + + it("names no top-level field the validator does not have", () => { + const message = buildRetryCorrectionMessage("reason", "details"); + const known = new Set(allSchemaKeys()); + // The dead wrapper's two invented top-level arrays. Asserted by name + // rather than by diffing prose, so the check cannot go vacuous. + for (const stale of ["citations", "warnings"]) { + expect(known.has(stale)).toBe(false); + expect( + message, + `retry prompt still demands the removed "${stale}" field`, + ).not.toContain(stale); + } + }); + + it("marks the optional recommendation fields as optional", () => { + const message = buildRetryCorrectionMessage("reason", "details"); + // `metricSource` and `rationale` are `.optional()` on the validator; the + // prompt must not demand them or a grounded-but-sparse reply gets a + // needless second round trip. + expect(message).toMatch(/OPTIONAL/); + expect(message).not.toMatch(/Every recommendation MUST carry/); + }); + + it("passes the caller's reason and truncated details through", () => { + const message = buildRetryCorrectionMessage("bad-json", "x".repeat(2000)); + expect(message).toContain("bad-json"); + expect(message).toContain("x".repeat(1024)); + expect(message).not.toContain("x".repeat(1025)); + }); +}); diff --git a/src/lib/ai/retry-correction.ts b/src/lib/ai/retry-correction.ts index 26c912b49..d3206decc 100644 --- a/src/lib/ai/retry-correction.ts +++ b/src/lib/ai/retry-correction.ts @@ -17,8 +17,47 @@ * * What survived is the part that was always reachable — the corrective * message, which `comprehensive-generate.ts` sends on its own JSON-retry leg. + * The message kept describing the DEAD wrapper's contract long after the + * wrapper went: it demanded `citations[]` and `warnings[]` (fields the live + * `insightResultSchema` does not have), demanded `id` / `severity` / + * `metricSource` / `rationale` on every recommendation (all optional in the + * live schema), and named NONE of the five fields the live schema actually + * requires — `classification`, `findings`, `correlations`, `dataQuality`, + * `disclaimer`. So the one corrective pass steered the model away from the + * contract it was being corrected towards: the retry reply could not validate, + * fell through `parseComprehensiveResult`'s raw-object escape hatch, and the + * downstream surfaces read an unvalidated payload. + * + * The message below now describes `insightResultSchema` (`src/lib/ai/types.ts`) + * and nothing else. `RETRY_CONTRACT_EXAMPLE` is the same contract as data, and + * a test validates it against the schema — so the prompt and the validator + * cannot drift apart again without the suite going red. */ +/** + * A minimal payload that satisfies `insightResultSchema` exactly: every + * required field present, no optional field set. Embedded verbatim in the + * corrective message so the model has an unambiguous target, and asserted + * against the schema in `__tests__/retry-correction.test.ts`. + */ +export const RETRY_CONTRACT_EXAMPLE = { + summary: "Two to three sentences describing the period.", + classification: "gut", + findings: [ + { label: "Metric name", value: "Value as text", assessment: "neutral" }, + ], + correlations: [ + { factor: "Factor name", effect: "Observed effect", confidence: "mittel" }, + ], + recommendations: [{ text: "One grounded, actionable sentence." }], + dataQuality: { + coverage: "How much of the window carried readings.", + gaps: ["A named gap, or an empty array."], + confidence: "mittel", + }, + disclaimer: "The standard non-diagnostic disclaimer.", +} as const; + /** * Build the corrective user-prompt suffix that explains to the model * what went wrong on the first attempt. Includes the violated zod @@ -33,28 +72,34 @@ Your previous response did not satisfy the required JSON schema. Reason: ${reason} Details: ${details.slice(0, 1024)} -You MUST return a JSON object matching this schema exactly. Do NOT -include prose, markdown fences, or commentary outside the JSON. +You MUST return a single JSON object. Do NOT include prose, markdown +fences, or commentary outside the JSON. -Required top-level fields: +Required top-level fields — all seven MUST be present: - summary: string (2-3 sentences) - - recommendations: array of objects with - { id: string, - text: string, - severity: "info" | "suggestion" | "important" | "urgent", - metricSource: { type: string, timeRange: string, summary: string, n?: number }, - rationale: { dataWindow: "last7days" | "last30days" | "last90days" | "allTime", - comparedTo: non-empty string, - deviation: non-empty string } } - - citations: array of objects with { type: string, timeRange: string, summary: string } - - warnings: array of objects with { topic: string, message: string, severity?: same enum } + - classification: exactly one of "optimal" | "gut" | "grenzwertig" + | "erhoht" | "kritisch" + - findings: array of { label: string, value: string, + assessment: "positive" | "neutral" | "attention" | "warning" } + - correlations: array of { factor: string, effect: string, + confidence: "hoch" | "mittel" | "gering" } + - recommendations: array of { text: string } + - dataQuality: { coverage: string, gaps: array of string, + confidence: "hoch" | "mittel" | "gering" } + - disclaimer: string -Every recommendation's metricSource MUST also appear in citations[] -(same type + timeRange). Every recommendation MUST carry a -rationale object — dataWindow, comparedTo, and deviation are all -required and rationale.dataWindow MUST equal metricSource.timeRange. -If you cannot ground a recommendation in user data from the -snapshot you were given, OMIT it. An empty recommendations[] is -acceptable. +An empty array is acceptable for findings, correlations, and +recommendations. If you cannot ground a recommendation in the user +data from the snapshot you were given, OMIT that recommendation. + +A recommendation object MAY additionally carry any of: id, severity +("info" | "suggestion" | "important" | "urgent"), metricSource +({ type, timeRange, summary }), rationale ({ dataWindow, comparedTo, +deviation }), confidence (integer 0-100). All of these are OPTIONAL — +omit any you cannot ground rather than inventing a value. Do NOT add +top-level fields beyond the seven listed above. + +A minimal valid response looks exactly like this: +${JSON.stringify(RETRY_CONTRACT_EXAMPLE, null, 2)} `; }