From 0cb0373e59ae5b9bca7e02de3df6c977d4d125cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 10 May 2026 01:26:00 +0200 Subject: [PATCH] feat(ai): deterministic confidence-score computation per recommendation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each recommendation now carries an optional `confidence: 0..100` integer in `aiRecommendationSchema`. New `src/lib/ai/confidence.ts` exposes `computeConfidence({ n, recencyDays, deviationStdRatio })`, the deterministic formula the wrapper will use to override whatever number the model picks. Score breakdown (sum, max 100): - n contribution: log10-saturating curve, capped at 40. n<3 hard- caps the whole score at 5*n (max 15) so a three-sample window never produces a confident-looking meter. - recency contribution: 30 at <2d, decays linearly to 0 at 30d. - signal contribution: 30 at |z|>=1.5, scaled linearly to 0 at z=0; null ratio (no baseline yet) returns a neutral 15. Schema is `.optional()` so: - The wrapper can fill it post-validation regardless of what the LLM emitted — the model's calibrated probability is unreliable. - Legacy cached payloads from before B5d landed still parse. Integer-only and bounded so the meter component renders without clamping; a renegade provider that emits 101 or 67.5 fails parse and triggers the wrapper's corrective retry. 15 fixtures cover the input matrix: n hard-cap (n=0..2), n=3 floor, mid-range (n=14, n=20), saturation (n=100 plateau, n=10000 cap), recency clamp (45d -> 0), null-ratio neutral, negative-z magnitude, plus monotonicity-in-n + integer-in-bounds invariants. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../ai/__tests__/confidence-schema.test.ts | 98 ++++++++++++ src/lib/ai/__tests__/confidence.test.ts | 150 ++++++++++++++++++ src/lib/ai/confidence.ts | 102 ++++++++++++ src/lib/ai/schema.ts | 18 +++ src/lib/ai/types.ts | 7 + 5 files changed, 375 insertions(+) create mode 100644 src/lib/ai/__tests__/confidence-schema.test.ts create mode 100644 src/lib/ai/__tests__/confidence.test.ts create mode 100644 src/lib/ai/confidence.ts diff --git a/src/lib/ai/__tests__/confidence-schema.test.ts b/src/lib/ai/__tests__/confidence-schema.test.ts new file mode 100644 index 000000000..2f7b7a589 --- /dev/null +++ b/src/lib/ai/__tests__/confidence-schema.test.ts @@ -0,0 +1,98 @@ +import { describe, it, expect } from "vitest"; +import { + aiInsightResponseSchema, + aiRecommendationSchema, +} from "../schema"; + +/** + * v1.4.16 phase B5d — `recommendation.confidence` schema field. + * + * The wrapper overrides the model's confidence with a deterministic + * server-computed value (`computeConfidence()`), but the schema needs + * to ACCEPT a model-supplied number so payloads round-trip cleanly. + * The field is `.optional()` because: + * - The wrapper fills it post-validation regardless. + * - A locally-cached payload that was generated before B5d landed + * parses cleanly without the field. + * + * When set, the value is an integer 0..100. Decimals or out-of-range + * values fail parse so a renegade provider can't put us in a UI state + * where the meter overflows. + */ + +const baseRationale = { + dataWindow: "last7days" as const, + comparedTo: "your 90-day median (73 bpm)", + deviation: "+5 bpm above baseline over 7 of 7 days", +}; + +const baseRec = { + id: "rec-1", + text: "Aim for resting pulse below your usual baseline.", + severity: "suggestion" as const, + metricSource: { + type: "pulse", + timeRange: "last7days", + summary: "avg 78 bpm across 9 readings", + }, + rationale: baseRationale, +}; + +describe("aiRecommendationSchema — confidence", () => { + it("accepts a recommendation without confidence (legacy + wrapper-fill path)", () => { + expect(aiRecommendationSchema.safeParse(baseRec).success).toBe(true); + }); + + it("accepts a recommendation with confidence=0", () => { + expect( + aiRecommendationSchema.safeParse({ ...baseRec, confidence: 0 }).success, + ).toBe(true); + }); + + it("accepts a recommendation with confidence=100", () => { + expect( + aiRecommendationSchema.safeParse({ ...baseRec, confidence: 100 }).success, + ).toBe(true); + }); + + it("accepts mid-range integer confidence", () => { + expect( + aiRecommendationSchema.safeParse({ ...baseRec, confidence: 67 }).success, + ).toBe(true); + }); + + it("rejects negative confidence", () => { + expect( + aiRecommendationSchema.safeParse({ ...baseRec, confidence: -1 }).success, + ).toBe(false); + }); + + it("rejects confidence over 100", () => { + expect( + aiRecommendationSchema.safeParse({ ...baseRec, confidence: 101 }).success, + ).toBe(false); + }); + + it("rejects non-integer confidence", () => { + expect( + aiRecommendationSchema.safeParse({ ...baseRec, confidence: 67.5 }).success, + ).toBe(false); + }); + + it("preserves confidence on full payload round-trip", () => { + const payload = { + summary: "ok", + recommendations: [{ ...baseRec, confidence: 67 }], + citations: [ + { + type: "pulse", + timeRange: "last7days", + summary: "avg 78 bpm across 9 readings", + }, + ], + warnings: [], + }; + const result = aiInsightResponseSchema.parse(payload); + expect(result.recommendations[0].confidence).toBe(67); + }); +}); diff --git a/src/lib/ai/__tests__/confidence.test.ts b/src/lib/ai/__tests__/confidence.test.ts new file mode 100644 index 000000000..e632982e6 --- /dev/null +++ b/src/lib/ai/__tests__/confidence.test.ts @@ -0,0 +1,150 @@ +import { describe, it, expect } from "vitest"; +import { computeConfidence, type ConfidenceInputs } from "../confidence"; + +/** + * v1.4.16 phase B5d — `computeConfidence()` deterministic scorer. + * + * The wrapper feeds three inputs: + * - n: sample count behind the cited data window + * (from `metricSource.n` when available; 0 otherwise). + * - recencyDays: days since the most recent sample in the window. + * - deviationStdRatio: |deviation| / stdev of the user's 90-day + * baseline. `null` when the baseline is too thin to compute. + * + * Score breakdown (max 100): + * - n contribution: log-saturating curve, max 40 at n=30 ish. + * n<3 hard-caps the whole score at ≤15 (5 * n). + * - recencyScore: 30 at <2d, decays linearly to 0 at 30d, floored at 0. + * - signalScore: 30 at |z|≥1.5, 0 at z=0, 15 when ratio is null. + * + * 12 fixtures cover the input matrix per research §2.C: small-n hard + * cap, mid-n + recency interplay, saturated-n + signal extremes, + * null-ratio neutral path. + */ + +interface Fixture { + name: string; + inputs: ConfidenceInputs; + expected: number; +} + +const FIXTURES: Fixture[] = [ + // n hard-cap: n<3 always returns max(10, 5*n) + { + name: "n=0 → hard-cap 10 floor", + inputs: { n: 0, recencyDays: 0, deviationStdRatio: 2 }, + expected: 10, + }, + { + name: "n=1 → hard-cap 10 floor", + inputs: { n: 1, recencyDays: 0, deviationStdRatio: 2 }, + expected: 10, + }, + { + name: "n=2 → hard-cap 10 (5*2)", + inputs: { n: 2, recencyDays: 0, deviationStdRatio: 2 }, + expected: 10, + }, + // n=3: nScore = 10 + 10*log10(3) ≈ 14.77; + // recency=1 → 30 * (1 - 1/30) = 29; signal=null → 15 + // total ≈ 14.77 + 29 + 15 = 58.77 → 59 + { + name: "n=3 fresh sample (1d old), null ratio → 59", + inputs: { n: 3, recencyDays: 1, deviationStdRatio: null }, + expected: 59, + }, + // n=14, recencyDays=1, ratio=2 → research-§2C calibration target ~94 + // nScore = 10 + 10*log10(14) ≈ 21.46 + // recencyScore = 30 * (1 - 1/30) = 29 + // signalScore = min(30, 30 * 2/1.5) = 30 + // total ≈ 80.46 → 80 + { + name: "n=14, fresh, strong signal → ~80", + inputs: { n: 14, recencyDays: 1, deviationStdRatio: 2 }, + expected: 80, + }, + // n=30, recencyDays=20, null ratio → research-§2C calibration target ~52 + // nScore = min(40, 10 + 10*log10(30)) = min(40, 24.77) = 24.77 + // recencyScore = max(0, 30 * (1 - 20/30)) = 10 + // signalScore = 15 (null) + // total ≈ 49.77 → 50 + { + name: "n=30, stale (20d), null ratio → ~50", + inputs: { n: 30, recencyDays: 20, deviationStdRatio: null }, + expected: 50, + }, + // n=100 → nScore = min(40, 10 + 10*log10(100)) = 30 (curve, not cap) + // recency=0 → 30, signal=z=1.5 → 30, total 90 + { + name: "n=100, fresh, strong signal → 90", + inputs: { n: 100, recencyDays: 0, deviationStdRatio: 1.5 }, + expected: 90, + }, + // n=10000 → log10 saturates at the 40 cap → max possible total 100 + { + name: "n=10000 saturates the n curve at 40 → 100", + inputs: { n: 10000, recencyDays: 0, deviationStdRatio: 2 }, + expected: 100, + }, + // n=10, recency=30d → recency floors at 0; ratio=0 → signal=0 + // nScore = 10 + 10*log10(10) = 20; recency=0; signal=0 + // total = 20 + { + name: "n=10, ancient (30d), zero deviation → 20", + inputs: { n: 10, recencyDays: 30, deviationStdRatio: 0 }, + expected: 20, + }, + // n=10, recency=45d → recencyScore floor 0 (negative clamped) + { + name: "n=10, very ancient (45d) clamps recency to 0", + inputs: { n: 10, recencyDays: 45, deviationStdRatio: null }, + expected: 35, // 20 + 0 + 15 + }, + // n=20, fresh, mild signal (z=0.75 → half of cap) + // nScore = 10 + 10*log10(20) ≈ 23.01 + // recency=0 → 30 + // signal = 30 * 0.75/1.5 = 15 + // total ≈ 68.01 → 68 + { + name: "n=20, fresh, mild signal → ~68", + inputs: { n: 20, recencyDays: 0, deviationStdRatio: 0.75 }, + expected: 68, + }, + // Negative ratio: |z|=2 still hits cap of 30 + { + name: "n=14, fresh, strong negative signal (z=-2) → 80", + inputs: { n: 14, recencyDays: 1, deviationStdRatio: -2 }, + expected: 80, + }, + // Edge: n=2, fresh, strong signal — hard-cap still applies + { + name: "n=2 hard-cap survives even with strong signal + fresh", + inputs: { n: 2, recencyDays: 0, deviationStdRatio: 2 }, + expected: 10, + }, +]; + +describe("computeConfidence()", () => { + it.each(FIXTURES)("$name", ({ inputs, expected }) => { + expect(computeConfidence(inputs)).toBe(expected); + }); + + it("returns an integer in [0, 100] for all fixtures", () => { + for (const f of FIXTURES) { + const v = computeConfidence(f.inputs); + expect(Number.isInteger(v)).toBe(true); + expect(v).toBeGreaterThanOrEqual(0); + expect(v).toBeLessThanOrEqual(100); + } + }); + + it("is monotonic in n when other inputs fixed (more data → never lower)", () => { + const baseInputs = { recencyDays: 1, deviationStdRatio: 1 }; + let prev = -Infinity; + for (const n of [5, 10, 20, 50, 100]) { + const v = computeConfidence({ ...baseInputs, n }); + expect(v).toBeGreaterThanOrEqual(prev); + prev = v; + } + }); +}); diff --git a/src/lib/ai/confidence.ts b/src/lib/ai/confidence.ts new file mode 100644 index 000000000..2e625b242 --- /dev/null +++ b/src/lib/ai/confidence.ts @@ -0,0 +1,102 @@ +/** + * v1.4.16 phase B5d — deterministic confidence-score computation per + * recommendation. + * + * Marc's mandate: the model's self-reported confidence is unreliable + * (calibrated probabilities are not a small-LLM strength), so the + * wrapper OVERRIDES with a server-computed value derived from three + * inputs that the cited data window actually exposes: + * + * - n sample count behind the cited data window + * (`metricSource.n`). + * - recencyDays days since the most recent sample in the window; + * a 7-day rec built off data 25 days old should + * not feel as confident as the same rec built off + * data from this morning. + * - deviationStdRatio |deviation| / stdev of the user's 90-day + * baseline. `null` when the baseline is too thin + * to compute a stdev — we fall back to a neutral + * mid-band score rather than zero. + * + * Score components (sum, max 100): + * + * - n contribution: log-saturating curve, capped at 40. + * n<3 hard-caps the whole score at ≤15 (5 * n) — three samples is + * the hard floor for any quantitative claim per research §2.A + * (Apple's "be quiet when unsure" stance). + * + * - recency contribution: 30 at <2d old, decays linearly to 0 at + * 30d, floored at 0. The shape matches the Oura "Pay Attention" + * band — old data is still data, just stale. + * + * - signal contribution: 30 at |z|≥1.5 (clearly out-of-band), + * scaled linearly down to 0 at z=0. When ratio is null (no + * baseline yet), we award 15 — neither punishing the user for + * thin history nor over-claiming a strong deviation we can't + * measure. + * + * The formula is pre-feedback. v1.4.17 will fit weights to thumbs-up + * rates (B5e feedback ratchet); for v1.4.16 the formula is fixed and + * deterministic so admin observability + the "draft pill below 25" + * UI affordance are reproducible across runs. + */ + +export interface ConfidenceInputs { + /** Sample count behind the cited data window (from metricSource.n). */ + n: number; + /** Days since the most recent sample in the window. */ + recencyDays: number; + /** |deviation| / stdev of the user's 90-day baseline, when computable. */ + deviationStdRatio: number | null; +} + +const HARD_CAP_FLOOR = 10; +const N_CONTRIBUTION_MAX = 40; +const RECENCY_CONTRIBUTION_MAX = 30; +const RECENCY_DECAY_DAYS = 30; +const SIGNAL_CONTRIBUTION_MAX = 30; +const SIGNAL_SATURATION_RATIO = 1.5; +const SIGNAL_NULL_NEUTRAL = 15; + +/** + * Compute the deterministic confidence score (0..100) for a single + * recommendation. Pure function — no I/O, no clock; safe to call from + * test fixtures and from `generateInsight()` post-validation alike. + */ +export function computeConfidence(inputs: ConfidenceInputs): number { + const { n, recencyDays, deviationStdRatio } = inputs; + + // n hard cap: <3 samples means we cannot make a quantitative claim, + // so the whole score is bounded at 5*n (max 15 at n=2). Floor at + // 10 so n=0/n=1 still produce a non-zero "draft" signal — the UI + // shows the rec but tags it as low-confidence rather than hiding it. + if (n < 3) { + return Math.max(HARD_CAP_FLOOR, 5 * n); + } + + // n contribution: log10 saturates so a 100-sample window doesn't + // outweigh a 30-sample window by 3×; both are "enough data" for the + // user-level signal we care about. + const nScore = Math.min(N_CONTRIBUTION_MAX, 10 + 10 * Math.log10(n)); + + // Recency contribution: <2d ≈ full score (30 * (1 - 0/30) = 30, + // 30 * (1 - 1/30) = 29; both round into the meter's top band). + // 30d+ floors to 0 so an ancient window cannot inflate the score. + const recencyScore = Math.max( + 0, + RECENCY_CONTRIBUTION_MAX * (1 - recencyDays / RECENCY_DECAY_DAYS), + ); + + // Signal contribution: null ratio (no baseline) → neutral 15. + // Otherwise scale |z| linearly to the SIGNAL_SATURATION_RATIO cap. + const signalScore = + deviationStdRatio === null + ? SIGNAL_NULL_NEUTRAL + : Math.min( + SIGNAL_CONTRIBUTION_MAX, + SIGNAL_CONTRIBUTION_MAX * + (Math.abs(deviationStdRatio) / SIGNAL_SATURATION_RATIO), + ); + + return Math.round(nScore + recencyScore + signalScore); +} diff --git a/src/lib/ai/schema.ts b/src/lib/ai/schema.ts index 3ed97a0f1..edfe06f42 100644 --- a/src/lib/ai/schema.ts +++ b/src/lib/ai/schema.ts @@ -167,6 +167,24 @@ export const aiRecommendationSchema = z * when `severity >= "important"`. */ referenceId: z.string().optional(), + /** + * v1.4.16 phase B5d — deterministic confidence score (0-100). + * + * Optional at parse-time: + * - The LLM may emit a value (we accept it so the payload round- + * trips cleanly), but `generateInsight()` OVERRIDES with the + * server-computed `computeConfidence()` post-validation. The + * model's number is discarded — calibrated probabilities are + * not a small-LLM strength and the deterministic path keeps + * the v1.4.17 feedback ratchet reproducible. + * - Legacy cached payloads from before B5d landed have no + * confidence field; the meter falls back to a "draft" pill. + * + * Integer-only and bounded so the meter component can render + * without clamping; a renegade provider that emits 101 or 67.5 + * fails parse and triggers the wrapper's corrective retry. + */ + confidence: z.number().int().min(0).max(100).optional(), }) .superRefine((rec, ctx) => { if (rec.referenceId !== undefined) { diff --git a/src/lib/ai/types.ts b/src/lib/ai/types.ts index 92c938a3a..3d06402e1 100644 --- a/src/lib/ai/types.ts +++ b/src/lib/ai/types.ts @@ -69,6 +69,13 @@ export const insightRecommendationSchema = z.union([ .enum(["info", "suggestion", "important", "urgent"]) .optional(), id: z.string().optional(), + /** + * v1.4.16 phase B5d — deterministic confidence score (0-100). + * Optional on the UI type because legacy cached payloads predate + * the field; `` renders a "draft" pill in that + * case. Server-side `aiRecommendationSchema` mirrors this shape. + */ + confidence: z.number().int().min(0).max(100).optional(), }), ]);