HealthLog/src/lib/ai/__tests__/confidence-wrapper.test.ts

import { describe, it, expect } from "vitest";
import { generateInsight } from "../generate-insight";
import { MockAIProvider } from "../mock-client";
import { singleUserTurn } from "../types";

/**
 * v1.4.16 phase B5d — wrapper post-validation override.
 *
 * The model's self-reported `confidence` is DISCARDED. After zod
 * parse succeeds, `generateInsight()` derives `ConfidenceInputs` per
 * recommendation and overwrites `rec.confidence` with the value
 * `computeConfidence()` produces.
 *
 * Default-on per research §7 question 1 (maintainer-acked). The override
 * happens whether or not the caller supplies a bucket-series-aware
 * resolver — when the resolver is missing we fall back to inputs
 * derivable from the parsed payload alone (`metricSource.n`,
 * recencyDays=0, deviationStdRatio=null) so the contract is "the
 * confidence on the response NEVER comes from the LLM".
 */

const validResponseWithModelConfidence = {
  summary: "Your blood pressure is trending high.",
  recommendations: [
    {
      id: "rec-1",
      text: "Discuss lifestyle and possible medication review with your physician.",
      severity: "important",
      // Model claims 99 — wrapper must overwrite this.
      confidence: 99,
      metricSource: {
        type: "bloodPressure",
        timeRange: "last7days",
        summary: "avg 142/88 across 12 readings",
        n: 12,
      },
      rationale: {
        dataWindow: "last7days",
        comparedTo: "your 90-day median (128/82)",
        deviation: "+14/+6 mmHg above baseline over 7 of 7 days",
      },
    },
  ],
  citations: [
    {
      type: "bloodPressure",
      timeRange: "last7days",
      summary: "avg 142/88 across 12 readings",
    },
  ],
  warnings: [],
};

describe("generateInsight() — confidence override", () => {
  it("overrides model-supplied confidence with deterministic computation (default fallback)", async () => {
    const provider = new MockAIProvider({
      responses: JSON.stringify(validResponseWithModelConfidence),
    });
    const outcome = await generateInsight(
      provider,
      singleUserTurn({ system: "sys", user: "user" }),
    );
    const rec = outcome.parsed.recommendations[0];
    // Model claimed 99; wrapper must have replaced it.
    // n=12, recencyDays=0 (default), ratio=null (default) →
    // nScore = 10 + 10*log10(12) ≈ 20.79
    // recency = 30 * (1 - 0/30) = 30
    // signal = null → 15
    // total ≈ 65.79 → 66
    expect(rec.confidence).toBe(66);
    expect(rec.confidence).not.toBe(99);
  });

  it("fills in confidence even when the model omitted it", async () => {
    const noConfidence = {
      ...validResponseWithModelConfidence,
      recommendations: [
        {
          ...validResponseWithModelConfidence.recommendations[0],
          confidence: undefined,
        },
      ],
    };
    // strip the undefined to emulate the model not emitting the key
    const stringified = JSON.stringify(noConfidence);
    const provider = new MockAIProvider({ responses: stringified });
    const outcome = await generateInsight(
      provider,
      singleUserTurn({ system: "sys", user: "u" }),
    );
    expect(outcome.parsed.recommendations[0].confidence).toBe(66);
  });

  it("derives n from metricSource.n on the parsed payload (no caller injection)", async () => {
    // v1.4.16 phase D reconcile (simplify F2) — the wrapper used to
    // accept an optional `confidenceContext` resolver, but no production
    // caller ever passed one. Inputs are derived from the payload alone
    // today; v1.4.17's feedback ratchet will re-introduce a resolver
    // injection when a real second caller exists.
    const fourteenN = {
      ...validResponseWithModelConfidence,
      recommendations: [
        {
          ...validResponseWithModelConfidence.recommendations[0],
          metricSource: {
            ...validResponseWithModelConfidence.recommendations[0].metricSource,
            n: 14,
          },
        },
      ],
    };
    const provider = new MockAIProvider({
      responses: JSON.stringify(fourteenN),
    });
    const outcome = await generateInsight(
      provider,
      singleUserTurn({ system: "s", user: "u" }),
    );
    // n=14, recencyDays=0, ratio=null → 10 + 10*log10(14) + 30 + 15 ≈ 66.46
    expect(outcome.parsed.recommendations[0].confidence).toBe(66);
  });

  it("hard-caps confidence to <=15 when n<3 (small-data shield)", async () => {
    const tinyN = {
      ...validResponseWithModelConfidence,
      recommendations: [
        {
          ...validResponseWithModelConfidence.recommendations[0],
          metricSource: {
            ...validResponseWithModelConfidence.recommendations[0].metricSource,
            n: 1,
          },
        },
      ],
    };
    const provider = new MockAIProvider({ responses: JSON.stringify(tinyN) });
    const outcome = await generateInsight(
      provider,
      singleUserTurn({ system: "s", user: "u" }),
    );
    expect(outcome.parsed.recommendations[0].confidence).toBeLessThanOrEqual(
      15,
    );
  });

  it("treats missing metricSource.n as n=0 (hard-cap)", async () => {
    const noN = {
      ...validResponseWithModelConfidence,
      recommendations: [
        {
          ...validResponseWithModelConfidence.recommendations[0],
          metricSource: {
            type: "bloodPressure",
            timeRange: "last7days",
            summary: "avg 142/88 across 12 readings",
            // n omitted
          },
        },
      ],
    };
    const provider = new MockAIProvider({ responses: JSON.stringify(noN) });
    const outcome = await generateInsight(
      provider,
      singleUserTurn({ system: "s", user: "u" }),
    );
    // n=0 → hard-cap floor 10
    expect(outcome.parsed.recommendations[0].confidence).toBe(10);
  });

  it("overrides confidence on every recommendation in payload order", async () => {
    const twoRecs = {
      ...validResponseWithModelConfidence,
      recommendations: [
        {
          ...validResponseWithModelConfidence.recommendations[0],
          id: "rec-a",
          confidence: 99,
        },
        {
          ...validResponseWithModelConfidence.recommendations[0],
          id: "rec-b",
          confidence: 1,
          metricSource: {
            ...validResponseWithModelConfidence.recommendations[0].metricSource,
            n: 5,
          },
        },
      ],
    };
    const provider = new MockAIProvider({
      responses: JSON.stringify(twoRecs),
    });
    const outcome = await generateInsight(
      provider,
      singleUserTurn({ system: "s", user: "u" }),
    );
    // Model claimed (99, 1); both must be discarded and replaced with
    // the deterministic value (n=12 → 66, n=5 → 62 from the saturating
    // formula 10 + 10*log10(5) + 30 + 15 ≈ 61.99).
    expect(outcome.parsed.recommendations[0].confidence).toBe(66);
    expect(outcome.parsed.recommendations[1].confidence).toBe(62);
  });
});