mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-17 07:42:23 +00:00
fix(documents): anchor extracted-fact provenance to the real document span
Every staged fact carried a sourceText the model echoed back, stored verbatim as 'the exact verbatim span of the document the fact came from'. Nothing checked that the span existed. A paraphrase, a normalised rendering, or an outright invention was persisted and shown to the reviewer as a transcription of their own clinical record — an anchor pointing at nothing, or at a span the document never contained. Treat the echo as a lookup key instead. anchorSourceText resolves it against the extracted text (exact, then whitespace-collapsed and case-folded to absorb OCR reflow) and stores the span the document actually carries, with its offset. A quote that matches nothing is dropped and the fact is flagged unanchored; vision mode has no extracted text at all, so its facts are unanchored by construction. Rows written before this resolve to unanchored on read — their quote is preserved but never presented as checked.
This commit is contained in:
1 parent
d40cb3eef3
commit
8e938a4f45
10 files changed
+395
-5
No files matched your search
@@ -29420,6 +29420,12 @@ components:
|
||||
properties:
|
||||
sourceText:
|
||||
type: string
|
||||
anchored:
|
||||
type: boolean
|
||||
sourceOffset:
|
||||
anyOf:
|
||||
- type: number
|
||||
- type: "null"
|
||||
page:
|
||||
anyOf:
|
||||
- type: number
|
||||
@@ -29428,6 +29434,8 @@ components:
|
||||
type: number
|
||||
required:
|
||||
- sourceText
|
||||
- anchored
|
||||
- sourceOffset
|
||||
- page
|
||||
- confidence
|
||||
additionalProperties: false
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
/**
|
||||
* Provenance anchoring — the model's quote is a lookup key, never content.
|
||||
*
|
||||
* Both branches matter equally. A quote that IS in the document must come back
|
||||
* as the document's own characters (not the model's rendering of them), and a
|
||||
* quote that is NOT in the document must come back unanchored with no text at
|
||||
* all — never a paraphrase presented to a reviewer as a verbatim excerpt.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { anchorSourceText } from "@/lib/documents/anchor-source-text";
|
||||
|
||||
const OCR = [
|
||||
"Befundbericht 2026-01-02",
|
||||
"Dx: Type 2 diabetes mellitus (E11.9)",
|
||||
"Hämoglobin 14.2 g/dL (13.5-17.5)",
|
||||
"Metformin 500 mg twice daily",
|
||||
].join("\n");
|
||||
|
||||
describe("anchorSourceText — located spans", () => {
|
||||
it("returns the document's own span for an exact echo", () => {
|
||||
const anchored = anchorSourceText(
|
||||
"Dx: Type 2 diabetes mellitus (E11.9)",
|
||||
OCR,
|
||||
);
|
||||
expect(anchored.anchored).toBe(true);
|
||||
expect(anchored.sourceText).toBe("Dx: Type 2 diabetes mellitus (E11.9)");
|
||||
expect(anchored.sourceOffset).toBe(OCR.indexOf("Dx:"));
|
||||
});
|
||||
|
||||
it("stores the source spacing, not the model's normalised spacing", () => {
|
||||
// The model collapsed the OCR column padding to single spaces. The stored
|
||||
// span must be what the document carries, padding included.
|
||||
const anchored = anchorSourceText("Hämoglobin 14.2 g/dL (13.5-17.5)", OCR);
|
||||
expect(anchored.anchored).toBe(true);
|
||||
expect(anchored.sourceText).toBe("Hämoglobin 14.2 g/dL (13.5-17.5)");
|
||||
expect(OCR.slice(anchored.sourceOffset!)).toContain(anchored.sourceText);
|
||||
});
|
||||
|
||||
it("tolerates case drift and reflowed line breaks", () => {
|
||||
const anchored = anchorSourceText("metformin 500 MG twice daily", OCR);
|
||||
expect(anchored.anchored).toBe(true);
|
||||
expect(anchored.sourceText).toBe("Metformin 500 mg twice daily");
|
||||
});
|
||||
|
||||
it("anchors a span the offset can be read back from", () => {
|
||||
const anchored = anchorSourceText("Befundbericht", OCR);
|
||||
expect(anchored.sourceOffset).not.toBeNull();
|
||||
expect(
|
||||
OCR.slice(
|
||||
anchored.sourceOffset!,
|
||||
anchored.sourceOffset! + anchored.sourceText.length,
|
||||
),
|
||||
).toBe(anchored.sourceText);
|
||||
});
|
||||
});
|
||||
|
||||
describe("anchorSourceText — unlocatable quotes are unanchored", () => {
|
||||
it("drops a plausible-looking paraphrase the document never contained", () => {
|
||||
// Every word here appears somewhere in the document; the SENTENCE does not.
|
||||
const anchored = anchorSourceText(
|
||||
"Patient has diabetes and takes Metformin for it",
|
||||
OCR,
|
||||
);
|
||||
expect(anchored.anchored).toBe(false);
|
||||
expect(anchored.sourceText).toBe("");
|
||||
expect(anchored.sourceOffset).toBeNull();
|
||||
});
|
||||
|
||||
it("drops an outright hallucinated value", () => {
|
||||
const anchored = anchorSourceText("Hämoglobin 9.1 g/dL", OCR);
|
||||
expect(anchored.anchored).toBe(false);
|
||||
expect(anchored.sourceText).toBe("");
|
||||
});
|
||||
|
||||
it("refuses to anchor a quote too short to mean anything", () => {
|
||||
// "g" occurs many times; a match that short is noise, not provenance.
|
||||
const anchored = anchorSourceText("g", OCR);
|
||||
expect(anchored.anchored).toBe(false);
|
||||
expect(anchored.sourceText).toBe("");
|
||||
});
|
||||
|
||||
it("is unanchored when there is no extracted text to verify against", () => {
|
||||
// Vision mode: the model read the rendered page, nothing can confirm it.
|
||||
const anchored = anchorSourceText(
|
||||
"Dx: Type 2 diabetes mellitus",
|
||||
undefined,
|
||||
);
|
||||
expect(anchored.anchored).toBe(false);
|
||||
expect(anchored.sourceText).toBe("");
|
||||
expect(anchored.sourceOffset).toBeNull();
|
||||
});
|
||||
|
||||
it("is unanchored for an empty quote", () => {
|
||||
expect(anchorSourceText(" ", OCR).anchored).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -120,7 +120,13 @@ beforeEach(() => {
|
||||
referenceHigh: 17,
|
||||
effectiveDate: "2026-07-10",
|
||||
},
|
||||
provenance: { sourceText: "Hämoglobin 14.2", page: 0, confidence: 0.9 },
|
||||
provenance: {
|
||||
sourceText: "Hämoglobin 14.2",
|
||||
anchored: true,
|
||||
sourceOffset: 0,
|
||||
page: 0,
|
||||
confidence: 0.9,
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
@@ -145,6 +145,95 @@ describe("runInboundExtraction — stated-status-only mapping", () => {
|
||||
expect(med.statusStated).toBe("ongoing");
|
||||
});
|
||||
|
||||
it("stores the document's real span, not the model's echo of it", async () => {
|
||||
// The OCR text says "Hämoglobin 14.2 g/dL"; the model echoes a tidied
|
||||
// rendering of it. The stored provenance must be the document's characters.
|
||||
const ocrText = "Labor 2026-01-02\nHämoglobin 14.2 g/dL (13.5-17.5)";
|
||||
const provider = fakeProvider(
|
||||
JSON.stringify({
|
||||
facts: [
|
||||
{
|
||||
type: "OBSERVATION",
|
||||
label: "Hämoglobin",
|
||||
value: 14.2,
|
||||
unit: "g/dL",
|
||||
confidence: 0.9,
|
||||
sourceText: "Hämoglobin 14.2 g/dL (13.5-17.5)",
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runInboundExtraction({
|
||||
provider,
|
||||
providerType: "mock",
|
||||
ocrText,
|
||||
});
|
||||
const { provenance } = result.facts[0]!;
|
||||
expect(provenance.anchored).toBe(true);
|
||||
expect(provenance.sourceText).toBe("Hämoglobin 14.2 g/dL (13.5-17.5)");
|
||||
expect(
|
||||
ocrText.slice(
|
||||
provenance.sourceOffset!,
|
||||
provenance.sourceOffset! + provenance.sourceText.length,
|
||||
),
|
||||
).toBe(provenance.sourceText);
|
||||
});
|
||||
|
||||
it("marks a fact unanchored rather than storing a quote it cannot locate", async () => {
|
||||
const ocrText = "Labor 2026-01-02\nHämoglobin 14.2 g/dL";
|
||||
const provider = fakeProvider(
|
||||
JSON.stringify({
|
||||
facts: [
|
||||
{
|
||||
type: "OBSERVATION",
|
||||
label: "Hämoglobin",
|
||||
value: 14.2,
|
||||
unit: "g/dL",
|
||||
confidence: 0.9,
|
||||
// A quote the document never carried.
|
||||
sourceText: "Haemoglobin was measured at 14.2 and is normal",
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runInboundExtraction({
|
||||
provider,
|
||||
providerType: "mock",
|
||||
ocrText,
|
||||
});
|
||||
const { provenance } = result.facts[0]!;
|
||||
expect(provenance.anchored).toBe(false);
|
||||
expect(provenance.sourceText).toBe("");
|
||||
expect(provenance.sourceOffset).toBeNull();
|
||||
// The fact itself still stages — only its provenance claim is withheld.
|
||||
expect(result.facts).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("marks vision-mode facts unanchored — there is no extracted text to verify", async () => {
|
||||
const provider = fakeProvider(
|
||||
JSON.stringify({
|
||||
facts: [
|
||||
{
|
||||
type: "CONDITION",
|
||||
label: "Type 2 diabetes mellitus",
|
||||
confidence: 0.9,
|
||||
sourceText: "Dx: Type 2 diabetes mellitus",
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runInboundExtraction({
|
||||
provider,
|
||||
providerType: "mock",
|
||||
});
|
||||
const { provenance } = result.facts[0]!;
|
||||
expect(provenance.anchored).toBe(false);
|
||||
expect(provenance.sourceText).toBe("");
|
||||
});
|
||||
|
||||
it("throws InboundExtractError on unparseable provider output", async () => {
|
||||
const provider = fakeProvider("not json at all");
|
||||
await expect(
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* Resolve a fact's provenance quote against the REAL extracted document text.
|
||||
*
|
||||
* Why this exists: the extraction prompt asks the model to echo "the exact
|
||||
* verbatim span of the document the fact came from" and the pipeline used to
|
||||
* store that echo as-is. A model can paraphrase it, normalise it, or invent it
|
||||
* outright, and the stored quote is then an anchor that points at nothing — or,
|
||||
* worse, at a span the document never contained, presented to a human reviewer
|
||||
* as if it were a verbatim transcription of their clinical record.
|
||||
*
|
||||
* The model's echo is therefore treated as a LOOKUP KEY, never as content. We
|
||||
* search the extracted text for it and store the span the DOCUMENT actually
|
||||
* carries. When the echo cannot be located, the fact is marked unanchored and
|
||||
* carries no quote at all: "we could not verify this against the source" is an
|
||||
* honest answer, an unverifiable quote is not.
|
||||
*
|
||||
* Two passes, cheapest first:
|
||||
* 1. exact substring — the common case, the model echoed the span verbatim;
|
||||
* 2. whitespace-collapsed + case-folded substring — covers OCR line wrapping,
|
||||
* column merges, and case drift, which are reflow artefacts rather than
|
||||
* paraphrase. The offset map walks the match back to the ORIGINAL text so
|
||||
* the stored span keeps the document's own characters and spacing.
|
||||
*
|
||||
* A paraphrase fails both passes by construction, which is the point.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Minimum normalised length a quote must reach before we try to locate it. A
|
||||
* one- or two-character key matches somewhere in almost any document, so a
|
||||
* "match" that short would be noise dressed up as provenance.
|
||||
*/
|
||||
const MIN_ANCHORABLE_LENGTH = 3;
|
||||
|
||||
/** Longest span we store, mirroring the provenance column's own cap. */
|
||||
const MAX_SPAN_LENGTH = 2000;
|
||||
|
||||
export interface AnchoredSource {
|
||||
/**
|
||||
* The verbatim span as it appears in the extracted text, or "" when the
|
||||
* quote could not be located (or there was no extracted text to search).
|
||||
*/
|
||||
sourceText: string;
|
||||
/** True only when `sourceText` was read back out of the document text. */
|
||||
anchored: boolean;
|
||||
/** Character offset of the span in the extracted text; null when unanchored. */
|
||||
sourceOffset: number | null;
|
||||
}
|
||||
|
||||
/** The unanchored result — no quote, no offset, no claim. */
|
||||
const UNANCHORED: AnchoredSource = {
|
||||
sourceText: "",
|
||||
anchored: false,
|
||||
sourceOffset: null,
|
||||
};
|
||||
|
||||
/**
|
||||
* Whitespace-collapsed, case-folded projection of `text`, plus a map from each
|
||||
* projected character back to its index in the original string. Folding is
|
||||
* skipped for any character whose lowercase form is not exactly one code unit,
|
||||
* so the map stays 1:1 and an offset can never drift.
|
||||
*/
|
||||
function project(text: string): { normalised: string; offsets: number[] } {
|
||||
const chars: string[] = [];
|
||||
const offsets: number[] = [];
|
||||
let pendingSpace = false;
|
||||
for (let i = 0; i < text.length; i += 1) {
|
||||
const ch = text[i]!;
|
||||
if (/\s/.test(ch)) {
|
||||
// Collapse a whitespace run to one space, and never lead with one.
|
||||
if (chars.length > 0) pendingSpace = true;
|
||||
continue;
|
||||
}
|
||||
if (pendingSpace) {
|
||||
chars.push(" ");
|
||||
offsets.push(i);
|
||||
pendingSpace = false;
|
||||
}
|
||||
const lower = ch.toLowerCase();
|
||||
chars.push(lower.length === 1 ? lower : ch);
|
||||
offsets.push(i);
|
||||
}
|
||||
return { normalised: chars.join(""), offsets };
|
||||
}
|
||||
|
||||
/**
|
||||
* Locate `quote` inside `sourceText` and return the span the document actually
|
||||
* carries. `sourceText` is the text the extraction ran against — the OCR output
|
||||
* in text mode. Pass `undefined`/`""` for vision mode, where the model read the
|
||||
* rendered page directly and there is no extracted text to verify against: the
|
||||
* result is unanchored, because nothing in that flow can confirm the quote.
|
||||
*/
|
||||
export function anchorSourceText(
|
||||
quote: string,
|
||||
sourceText: string | undefined,
|
||||
): AnchoredSource {
|
||||
if (!sourceText) return UNANCHORED;
|
||||
const trimmed = quote.trim();
|
||||
if (!trimmed) return UNANCHORED;
|
||||
|
||||
// The length floor gates BOTH passes: a one-character key matches the exact
|
||||
// pass just as readily as the folded one, and a hit that short is noise
|
||||
// either way.
|
||||
const needle = project(trimmed);
|
||||
if (needle.normalised.length < MIN_ANCHORABLE_LENGTH) return UNANCHORED;
|
||||
|
||||
// Pass 1 — the model echoed the span verbatim.
|
||||
const exact = sourceText.indexOf(trimmed);
|
||||
if (exact !== -1) {
|
||||
return {
|
||||
sourceText: sourceText.slice(exact, exact + trimmed.length),
|
||||
anchored: true,
|
||||
sourceOffset: exact,
|
||||
};
|
||||
}
|
||||
|
||||
// Pass 2 — same span, reflowed or case-drifted by OCR.
|
||||
const haystack = project(sourceText);
|
||||
const hit = haystack.normalised.indexOf(needle.normalised);
|
||||
if (hit === -1) return UNANCHORED;
|
||||
|
||||
const start = haystack.offsets[hit]!;
|
||||
const lastIndex = haystack.offsets[hit + needle.normalised.length - 1]!;
|
||||
return {
|
||||
sourceText: sourceText
|
||||
.slice(start, lastIndex + 1)
|
||||
.slice(0, MAX_SPAN_LENGTH),
|
||||
anchored: true,
|
||||
sourceOffset: start,
|
||||
};
|
||||
}
|
||||
@@ -15,8 +15,17 @@
|
||||
* assertion about the patient, never as the app's conclusion. The mandatory
|
||||
* human review screen is the hard backstop — nothing commits without per-fact
|
||||
* confirmation, and a low-confidence fact fails closed.
|
||||
*
|
||||
* Provenance is resolved, not echoed. The model returns a quote of the span it
|
||||
* read a fact from; that quote is used to LOOK UP the real span in the extracted
|
||||
* text (`anchorSourceText`) and the document's own characters are what gets
|
||||
* stored. A quote that matches nothing is dropped and the fact is flagged
|
||||
* unanchored — an anchor that points at nothing, or at the wrong span, is worse
|
||||
* than an honest "unverified" because the review screen presents it to a human
|
||||
* as a verbatim transcription of their record.
|
||||
*/
|
||||
import { AI_BUDGETS } from "@/lib/ai/ai-budgets";
|
||||
import { anchorSourceText } from "@/lib/documents/anchor-source-text";
|
||||
import {
|
||||
DOCUMENT_TEXT_FENCE_START,
|
||||
DOCUMENT_TEXT_FENCE_END,
|
||||
@@ -263,17 +272,28 @@ export async function runInboundExtraction(
|
||||
const reportDate = normaliseDate(envelope.reportDate);
|
||||
|
||||
const facts: StagedFactInput[] = [];
|
||||
let unanchored = 0;
|
||||
for (const raw of envelope.facts.slice(0, INBOUND_MAX_FACTS)) {
|
||||
const data = mapFactData(raw);
|
||||
if (!data) continue;
|
||||
const confidence = raw.confidence;
|
||||
// The model's `sourceText` is a LOOKUP KEY, never content: we search the
|
||||
// extracted text for it and store the span the document actually carries.
|
||||
// A quote we cannot locate is dropped and the fact is marked unanchored —
|
||||
// the reviewer is told "unverified" rather than shown a quote the document
|
||||
// may never have contained. Vision mode has no extracted text at all, so
|
||||
// every fact from it is unanchored by construction.
|
||||
const anchor = anchorSourceText(raw.sourceText, args.ocrText);
|
||||
if (!anchor.anchored) unanchored += 1;
|
||||
facts.push({
|
||||
factType: raw.type,
|
||||
confidence,
|
||||
needsReview: confidence < INBOUND_CONFIDENCE_FLOOR,
|
||||
data,
|
||||
provenance: {
|
||||
sourceText: raw.sourceText.trim().slice(0, 2000),
|
||||
sourceText: anchor.sourceText,
|
||||
anchored: anchor.anchored,
|
||||
sourceOffset: anchor.sourceOffset,
|
||||
page: raw.page,
|
||||
confidence,
|
||||
},
|
||||
@@ -286,6 +306,7 @@ export async function runInboundExtraction(
|
||||
mode: isTextMode ? "text" : "vision",
|
||||
providerType: args.providerType,
|
||||
facts: facts.length,
|
||||
unanchored,
|
||||
conditions: facts.filter((f) => f.factType === "CONDITION").length,
|
||||
observations: facts.filter((f) => f.factType === "OBSERVATION").length,
|
||||
medications: facts.filter((f) => f.factType === "MEDICATION_STATEMENT")
|
||||
|
||||
@@ -158,9 +158,28 @@ export function encryptFactProvenance(
|
||||
return encryptToBytes(JSON.stringify(provenance));
|
||||
}
|
||||
|
||||
/** Decrypt a staged fact's provenance back to its DTO shape. */
|
||||
/**
|
||||
* Decrypt a staged fact's provenance back to its DTO shape.
|
||||
*
|
||||
* Rows written before provenance was anchored carry the model's own echo of the
|
||||
* span with no record of whether it ever matched the document. That is exactly
|
||||
* the unverified case, so they resolve to `anchored: false`. The stored text is
|
||||
* preserved rather than blanked — it is the only provenance those rows have —
|
||||
* but the flag stops a consumer presenting it as a checked transcription.
|
||||
*/
|
||||
export function decryptFactProvenance(buf: Uint8Array): FactProvenance {
|
||||
return JSON.parse(decryptFromBytes(buf)) as FactProvenance;
|
||||
const raw = JSON.parse(decryptFromBytes(buf)) as Partial<FactProvenance>;
|
||||
const anchored = raw.anchored === true;
|
||||
return {
|
||||
sourceText: typeof raw.sourceText === "string" ? raw.sourceText : "",
|
||||
anchored,
|
||||
sourceOffset:
|
||||
anchored && typeof raw.sourceOffset === "number"
|
||||
? raw.sourceOffset
|
||||
: null,
|
||||
page: typeof raw.page === "number" ? raw.page : null,
|
||||
confidence: typeof raw.confidence === "number" ? raw.confidence : 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -97,8 +97,12 @@ export async function linkOcrLabsToVaultDocument(
|
||||
confidence: 1,
|
||||
needsReview: false,
|
||||
dataEncrypted: encryptFactData(observationFromLab(lab)),
|
||||
// No document span to point at: this fact is minted from an already
|
||||
// committed lab row, not transcribed from the document text.
|
||||
provenanceEncrypted: encryptFactProvenance({
|
||||
sourceText: "",
|
||||
anchored: false,
|
||||
sourceOffset: null,
|
||||
page: null,
|
||||
confidence: 1,
|
||||
}),
|
||||
|
||||
@@ -96,6 +96,8 @@ documentBulkSchema.meta({
|
||||
|
||||
const factProvenance = z.object({
|
||||
sourceText: z.string(),
|
||||
anchored: z.boolean(),
|
||||
sourceOffset: z.number().nullable(),
|
||||
page: z.number().nullable(),
|
||||
confidence: z.number(),
|
||||
});
|
||||
|
||||
@@ -179,8 +179,22 @@ export type InboundExtraction = z.infer<typeof inboundExtractionSchema>;
|
||||
|
||||
/** Per-field provenance carried on every staged fact. */
|
||||
export interface FactProvenance {
|
||||
/** The verbatim source span the value was transcribed from. */
|
||||
/**
|
||||
* The verbatim source span the value was transcribed from, read back out of
|
||||
* the extracted document text — NOT the model's echo of it. Empty string when
|
||||
* the span could not be located (see `anchored`).
|
||||
*/
|
||||
sourceText: string;
|
||||
/**
|
||||
* True only when `sourceText` was resolved against the extracted document
|
||||
* text. False means the quote is unverifiable and none is stored: either the
|
||||
* model's echo matched nothing in the source, or the extraction ran in vision
|
||||
* mode, where there is no extracted text to verify against. A reviewer must
|
||||
* read an unanchored fact against the original document itself.
|
||||
*/
|
||||
anchored: boolean;
|
||||
/** Character offset of the span in the extracted text; null when unanchored. */
|
||||
sourceOffset: number | null;
|
||||
/** Optional 0-based page index. */
|
||||
page: number | null;
|
||||
/** The model's self-reported confidence (0..1). */
|
||||
|
||||
Reference in new issue
Block a user