diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index fa0e0a551..c18490084 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -29420,6 +29420,12 @@ components: properties: sourceText: type: string + anchored: + type: boolean + sourceOffset: + anyOf: + - type: number + - type: "null" page: anyOf: - type: number @@ -29428,6 +29434,8 @@ components: type: number required: - sourceText + - anchored + - sourceOffset - page - confidence additionalProperties: false diff --git a/src/lib/documents/__tests__/anchor-source-text.test.ts b/src/lib/documents/__tests__/anchor-source-text.test.ts new file mode 100644 index 000000000..bdeeb4ab0 --- /dev/null +++ b/src/lib/documents/__tests__/anchor-source-text.test.ts @@ -0,0 +1,97 @@ +/** + * Provenance anchoring — the model's quote is a lookup key, never content. + * + * Both branches matter equally. A quote that IS in the document must come back + * as the document's own characters (not the model's rendering of them), and a + * quote that is NOT in the document must come back unanchored with no text at + * all — never a paraphrase presented to a reviewer as a verbatim excerpt. + */ +import { describe, expect, it } from "vitest"; + +import { anchorSourceText } from "@/lib/documents/anchor-source-text"; + +const OCR = [ + "Befundbericht 2026-01-02", + "Dx: Type 2 diabetes mellitus (E11.9)", + "Hämoglobin 14.2 g/dL (13.5-17.5)", + "Metformin 500 mg twice daily", +].join("\n"); + +describe("anchorSourceText — located spans", () => { + it("returns the document's own span for an exact echo", () => { + const anchored = anchorSourceText( + "Dx: Type 2 diabetes mellitus (E11.9)", + OCR, + ); + expect(anchored.anchored).toBe(true); + expect(anchored.sourceText).toBe("Dx: Type 2 diabetes mellitus (E11.9)"); + expect(anchored.sourceOffset).toBe(OCR.indexOf("Dx:")); + }); + + it("stores the source spacing, not the model's normalised spacing", () => { + // The model collapsed the OCR column padding to single spaces. The stored + // span must be what the document carries, padding included. + const anchored = anchorSourceText("Hämoglobin 14.2 g/dL (13.5-17.5)", OCR); + expect(anchored.anchored).toBe(true); + expect(anchored.sourceText).toBe("Hämoglobin 14.2 g/dL (13.5-17.5)"); + expect(OCR.slice(anchored.sourceOffset!)).toContain(anchored.sourceText); + }); + + it("tolerates case drift and reflowed line breaks", () => { + const anchored = anchorSourceText("metformin 500 MG twice daily", OCR); + expect(anchored.anchored).toBe(true); + expect(anchored.sourceText).toBe("Metformin 500 mg twice daily"); + }); + + it("anchors a span the offset can be read back from", () => { + const anchored = anchorSourceText("Befundbericht", OCR); + expect(anchored.sourceOffset).not.toBeNull(); + expect( + OCR.slice( + anchored.sourceOffset!, + anchored.sourceOffset! + anchored.sourceText.length, + ), + ).toBe(anchored.sourceText); + }); +}); + +describe("anchorSourceText — unlocatable quotes are unanchored", () => { + it("drops a plausible-looking paraphrase the document never contained", () => { + // Every word here appears somewhere in the document; the SENTENCE does not. + const anchored = anchorSourceText( + "Patient has diabetes and takes Metformin for it", + OCR, + ); + expect(anchored.anchored).toBe(false); + expect(anchored.sourceText).toBe(""); + expect(anchored.sourceOffset).toBeNull(); + }); + + it("drops an outright hallucinated value", () => { + const anchored = anchorSourceText("Hämoglobin 9.1 g/dL", OCR); + expect(anchored.anchored).toBe(false); + expect(anchored.sourceText).toBe(""); + }); + + it("refuses to anchor a quote too short to mean anything", () => { + // "g" occurs many times; a match that short is noise, not provenance. + const anchored = anchorSourceText("g", OCR); + expect(anchored.anchored).toBe(false); + expect(anchored.sourceText).toBe(""); + }); + + it("is unanchored when there is no extracted text to verify against", () => { + // Vision mode: the model read the rendered page, nothing can confirm it. + const anchored = anchorSourceText( + "Dx: Type 2 diabetes mellitus", + undefined, + ); + expect(anchored.anchored).toBe(false); + expect(anchored.sourceText).toBe(""); + expect(anchored.sourceOffset).toBeNull(); + }); + + it("is unanchored for an empty quote", () => { + expect(anchorSourceText(" ", OCR).anchored).toBe(false); + }); +}); diff --git a/src/lib/documents/__tests__/auto-stage-labs.test.ts b/src/lib/documents/__tests__/auto-stage-labs.test.ts index 566fcc826..aca4ac148 100644 --- a/src/lib/documents/__tests__/auto-stage-labs.test.ts +++ b/src/lib/documents/__tests__/auto-stage-labs.test.ts @@ -120,7 +120,13 @@ beforeEach(() => { referenceHigh: 17, effectiveDate: "2026-07-10", }, - provenance: { sourceText: "Hämoglobin 14.2", page: 0, confidence: 0.9 }, + provenance: { + sourceText: "Hämoglobin 14.2", + anchored: true, + sourceOffset: 0, + page: 0, + confidence: 0.9, + }, }, ], }); diff --git a/src/lib/documents/__tests__/extract.test.ts b/src/lib/documents/__tests__/extract.test.ts index f1d86aee1..6e398accf 100644 --- a/src/lib/documents/__tests__/extract.test.ts +++ b/src/lib/documents/__tests__/extract.test.ts @@ -145,6 +145,95 @@ describe("runInboundExtraction — stated-status-only mapping", () => { expect(med.statusStated).toBe("ongoing"); }); + it("stores the document's real span, not the model's echo of it", async () => { + // The OCR text says "Hämoglobin 14.2 g/dL"; the model echoes a tidied + // rendering of it. The stored provenance must be the document's characters. + const ocrText = "Labor 2026-01-02\nHämoglobin 14.2 g/dL (13.5-17.5)"; + const provider = fakeProvider( + JSON.stringify({ + facts: [ + { + type: "OBSERVATION", + label: "Hämoglobin", + value: 14.2, + unit: "g/dL", + confidence: 0.9, + sourceText: "Hämoglobin 14.2 g/dL (13.5-17.5)", + }, + ], + }), + ); + + const result = await runInboundExtraction({ + provider, + providerType: "mock", + ocrText, + }); + const { provenance } = result.facts[0]!; + expect(provenance.anchored).toBe(true); + expect(provenance.sourceText).toBe("Hämoglobin 14.2 g/dL (13.5-17.5)"); + expect( + ocrText.slice( + provenance.sourceOffset!, + provenance.sourceOffset! + provenance.sourceText.length, + ), + ).toBe(provenance.sourceText); + }); + + it("marks a fact unanchored rather than storing a quote it cannot locate", async () => { + const ocrText = "Labor 2026-01-02\nHämoglobin 14.2 g/dL"; + const provider = fakeProvider( + JSON.stringify({ + facts: [ + { + type: "OBSERVATION", + label: "Hämoglobin", + value: 14.2, + unit: "g/dL", + confidence: 0.9, + // A quote the document never carried. + sourceText: "Haemoglobin was measured at 14.2 and is normal", + }, + ], + }), + ); + + const result = await runInboundExtraction({ + provider, + providerType: "mock", + ocrText, + }); + const { provenance } = result.facts[0]!; + expect(provenance.anchored).toBe(false); + expect(provenance.sourceText).toBe(""); + expect(provenance.sourceOffset).toBeNull(); + // The fact itself still stages — only its provenance claim is withheld. + expect(result.facts).toHaveLength(1); + }); + + it("marks vision-mode facts unanchored — there is no extracted text to verify", async () => { + const provider = fakeProvider( + JSON.stringify({ + facts: [ + { + type: "CONDITION", + label: "Type 2 diabetes mellitus", + confidence: 0.9, + sourceText: "Dx: Type 2 diabetes mellitus", + }, + ], + }), + ); + + const result = await runInboundExtraction({ + provider, + providerType: "mock", + }); + const { provenance } = result.facts[0]!; + expect(provenance.anchored).toBe(false); + expect(provenance.sourceText).toBe(""); + }); + it("throws InboundExtractError on unparseable provider output", async () => { const provider = fakeProvider("not json at all"); await expect( diff --git a/src/lib/documents/anchor-source-text.ts b/src/lib/documents/anchor-source-text.ts new file mode 100644 index 000000000..1cd1f5670 --- /dev/null +++ b/src/lib/documents/anchor-source-text.ts @@ -0,0 +1,130 @@ +/** + * Resolve a fact's provenance quote against the REAL extracted document text. + * + * Why this exists: the extraction prompt asks the model to echo "the exact + * verbatim span of the document the fact came from" and the pipeline used to + * store that echo as-is. A model can paraphrase it, normalise it, or invent it + * outright, and the stored quote is then an anchor that points at nothing — or, + * worse, at a span the document never contained, presented to a human reviewer + * as if it were a verbatim transcription of their clinical record. + * + * The model's echo is therefore treated as a LOOKUP KEY, never as content. We + * search the extracted text for it and store the span the DOCUMENT actually + * carries. When the echo cannot be located, the fact is marked unanchored and + * carries no quote at all: "we could not verify this against the source" is an + * honest answer, an unverifiable quote is not. + * + * Two passes, cheapest first: + * 1. exact substring — the common case, the model echoed the span verbatim; + * 2. whitespace-collapsed + case-folded substring — covers OCR line wrapping, + * column merges, and case drift, which are reflow artefacts rather than + * paraphrase. The offset map walks the match back to the ORIGINAL text so + * the stored span keeps the document's own characters and spacing. + * + * A paraphrase fails both passes by construction, which is the point. + */ + +/** + * Minimum normalised length a quote must reach before we try to locate it. A + * one- or two-character key matches somewhere in almost any document, so a + * "match" that short would be noise dressed up as provenance. + */ +const MIN_ANCHORABLE_LENGTH = 3; + +/** Longest span we store, mirroring the provenance column's own cap. */ +const MAX_SPAN_LENGTH = 2000; + +export interface AnchoredSource { + /** + * The verbatim span as it appears in the extracted text, or "" when the + * quote could not be located (or there was no extracted text to search). + */ + sourceText: string; + /** True only when `sourceText` was read back out of the document text. */ + anchored: boolean; + /** Character offset of the span in the extracted text; null when unanchored. */ + sourceOffset: number | null; +} + +/** The unanchored result — no quote, no offset, no claim. */ +const UNANCHORED: AnchoredSource = { + sourceText: "", + anchored: false, + sourceOffset: null, +}; + +/** + * Whitespace-collapsed, case-folded projection of `text`, plus a map from each + * projected character back to its index in the original string. Folding is + * skipped for any character whose lowercase form is not exactly one code unit, + * so the map stays 1:1 and an offset can never drift. + */ +function project(text: string): { normalised: string; offsets: number[] } { + const chars: string[] = []; + const offsets: number[] = []; + let pendingSpace = false; + for (let i = 0; i < text.length; i += 1) { + const ch = text[i]!; + if (/\s/.test(ch)) { + // Collapse a whitespace run to one space, and never lead with one. + if (chars.length > 0) pendingSpace = true; + continue; + } + if (pendingSpace) { + chars.push(" "); + offsets.push(i); + pendingSpace = false; + } + const lower = ch.toLowerCase(); + chars.push(lower.length === 1 ? lower : ch); + offsets.push(i); + } + return { normalised: chars.join(""), offsets }; +} + +/** + * Locate `quote` inside `sourceText` and return the span the document actually + * carries. `sourceText` is the text the extraction ran against — the OCR output + * in text mode. Pass `undefined`/`""` for vision mode, where the model read the + * rendered page directly and there is no extracted text to verify against: the + * result is unanchored, because nothing in that flow can confirm the quote. + */ +export function anchorSourceText( + quote: string, + sourceText: string | undefined, +): AnchoredSource { + if (!sourceText) return UNANCHORED; + const trimmed = quote.trim(); + if (!trimmed) return UNANCHORED; + + // The length floor gates BOTH passes: a one-character key matches the exact + // pass just as readily as the folded one, and a hit that short is noise + // either way. + const needle = project(trimmed); + if (needle.normalised.length < MIN_ANCHORABLE_LENGTH) return UNANCHORED; + + // Pass 1 — the model echoed the span verbatim. + const exact = sourceText.indexOf(trimmed); + if (exact !== -1) { + return { + sourceText: sourceText.slice(exact, exact + trimmed.length), + anchored: true, + sourceOffset: exact, + }; + } + + // Pass 2 — same span, reflowed or case-drifted by OCR. + const haystack = project(sourceText); + const hit = haystack.normalised.indexOf(needle.normalised); + if (hit === -1) return UNANCHORED; + + const start = haystack.offsets[hit]!; + const lastIndex = haystack.offsets[hit + needle.normalised.length - 1]!; + return { + sourceText: sourceText + .slice(start, lastIndex + 1) + .slice(0, MAX_SPAN_LENGTH), + anchored: true, + sourceOffset: start, + }; +} diff --git a/src/lib/documents/extract.ts b/src/lib/documents/extract.ts index 8b6ea3d51..e33508cdb 100644 --- a/src/lib/documents/extract.ts +++ b/src/lib/documents/extract.ts @@ -15,8 +15,17 @@ * assertion about the patient, never as the app's conclusion. The mandatory * human review screen is the hard backstop — nothing commits without per-fact * confirmation, and a low-confidence fact fails closed. + * + * Provenance is resolved, not echoed. The model returns a quote of the span it + * read a fact from; that quote is used to LOOK UP the real span in the extracted + * text (`anchorSourceText`) and the document's own characters are what gets + * stored. A quote that matches nothing is dropped and the fact is flagged + * unanchored — an anchor that points at nothing, or at the wrong span, is worse + * than an honest "unverified" because the review screen presents it to a human + * as a verbatim transcription of their record. */ import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; +import { anchorSourceText } from "@/lib/documents/anchor-source-text"; import { DOCUMENT_TEXT_FENCE_START, DOCUMENT_TEXT_FENCE_END, @@ -263,17 +272,28 @@ export async function runInboundExtraction( const reportDate = normaliseDate(envelope.reportDate); const facts: StagedFactInput[] = []; + let unanchored = 0; for (const raw of envelope.facts.slice(0, INBOUND_MAX_FACTS)) { const data = mapFactData(raw); if (!data) continue; const confidence = raw.confidence; + // The model's `sourceText` is a LOOKUP KEY, never content: we search the + // extracted text for it and store the span the document actually carries. + // A quote we cannot locate is dropped and the fact is marked unanchored — + // the reviewer is told "unverified" rather than shown a quote the document + // may never have contained. Vision mode has no extracted text at all, so + // every fact from it is unanchored by construction. + const anchor = anchorSourceText(raw.sourceText, args.ocrText); + if (!anchor.anchored) unanchored += 1; facts.push({ factType: raw.type, confidence, needsReview: confidence < INBOUND_CONFIDENCE_FLOOR, data, provenance: { - sourceText: raw.sourceText.trim().slice(0, 2000), + sourceText: anchor.sourceText, + anchored: anchor.anchored, + sourceOffset: anchor.sourceOffset, page: raw.page, confidence, }, @@ -286,6 +306,7 @@ export async function runInboundExtraction( mode: isTextMode ? "text" : "vision", providerType: args.providerType, facts: facts.length, + unanchored, conditions: facts.filter((f) => f.factType === "CONDITION").length, observations: facts.filter((f) => f.factType === "OBSERVATION").length, medications: facts.filter((f) => f.factType === "MEDICATION_STATEMENT") diff --git a/src/lib/documents/store.ts b/src/lib/documents/store.ts index 7f063122a..b1a1253be 100644 --- a/src/lib/documents/store.ts +++ b/src/lib/documents/store.ts @@ -158,9 +158,28 @@ export function encryptFactProvenance( return encryptToBytes(JSON.stringify(provenance)); } -/** Decrypt a staged fact's provenance back to its DTO shape. */ +/** + * Decrypt a staged fact's provenance back to its DTO shape. + * + * Rows written before provenance was anchored carry the model's own echo of the + * span with no record of whether it ever matched the document. That is exactly + * the unverified case, so they resolve to `anchored: false`. The stored text is + * preserved rather than blanked — it is the only provenance those rows have — + * but the flag stops a consumer presenting it as a checked transcription. + */ export function decryptFactProvenance(buf: Uint8Array): FactProvenance { - return JSON.parse(decryptFromBytes(buf)) as FactProvenance; + const raw = JSON.parse(decryptFromBytes(buf)) as Partial; + const anchored = raw.anchored === true; + return { + sourceText: typeof raw.sourceText === "string" ? raw.sourceText : "", + anchored, + sourceOffset: + anchored && typeof raw.sourceOffset === "number" + ? raw.sourceOffset + : null, + page: typeof raw.page === "number" ? raw.page : null, + confidence: typeof raw.confidence === "number" ? raw.confidence : 0, + }; } /** diff --git a/src/lib/labs/vault-link.ts b/src/lib/labs/vault-link.ts index 9d6df818c..4d4a42d5d 100644 --- a/src/lib/labs/vault-link.ts +++ b/src/lib/labs/vault-link.ts @@ -97,8 +97,12 @@ export async function linkOcrLabsToVaultDocument( confidence: 1, needsReview: false, dataEncrypted: encryptFactData(observationFromLab(lab)), + // No document span to point at: this fact is minted from an already + // committed lab row, not transcribed from the document text. provenanceEncrypted: encryptFactProvenance({ sourceText: "", + anchored: false, + sourceOffset: null, page: null, confidence: 1, }), diff --git a/src/lib/openapi/routes/documents.ts b/src/lib/openapi/routes/documents.ts index 2915f0170..88c6ea852 100644 --- a/src/lib/openapi/routes/documents.ts +++ b/src/lib/openapi/routes/documents.ts @@ -96,6 +96,8 @@ documentBulkSchema.meta({ const factProvenance = z.object({ sourceText: z.string(), + anchored: z.boolean(), + sourceOffset: z.number().nullable(), page: z.number().nullable(), confidence: z.number(), }); diff --git a/src/lib/validations/inbound-documents.ts b/src/lib/validations/inbound-documents.ts index 0b7d381da..0e49daaae 100644 --- a/src/lib/validations/inbound-documents.ts +++ b/src/lib/validations/inbound-documents.ts @@ -179,8 +179,22 @@ export type InboundExtraction = z.infer; /** Per-field provenance carried on every staged fact. */ export interface FactProvenance { - /** The verbatim source span the value was transcribed from. */ + /** + * The verbatim source span the value was transcribed from, read back out of + * the extracted document text — NOT the model's echo of it. Empty string when + * the span could not be located (see `anchored`). + */ sourceText: string; + /** + * True only when `sourceText` was resolved against the extracted document + * text. False means the quote is unverifiable and none is stored: either the + * model's echo matched nothing in the source, or the extraction ran in vision + * mode, where there is no extracted text to verify against. A reviewer must + * read an unanchored fact against the original document itself. + */ + anchored: boolean; + /** Character offset of the span in the extracted text; null when unanchored. */ + sourceOffset: number | null; /** Optional 0-based page index. */ page: number | null; /** The model's self-reported confidence (0..1). */