/** * Resolve a fact's provenance quote against the REAL extracted document text. * * Why this exists: the extraction prompt asks the model to echo "the exact * verbatim span of the document the fact came from" and the pipeline used to * store that echo as-is. A model can paraphrase it, normalise it, or invent it * outright, and the stored quote is then an anchor that points at nothing — or, * worse, at a span the document never contained, presented to a human reviewer * as if it were a verbatim transcription of their clinical record. * * The model's echo is therefore treated as a LOOKUP KEY, never as content. We * search the extracted text for it and store the span the DOCUMENT actually * carries. When the echo cannot be located, the fact is marked unanchored and * carries no quote at all: "we could not verify this against the source" is an * honest answer, an unverifiable quote is not. * * Two passes, cheapest first: * 1. exact substring — the common case, the model echoed the span verbatim; * 2. whitespace-collapsed + case-folded substring — covers OCR line wrapping, * column merges, and case drift, which are reflow artefacts rather than * paraphrase. The offset map walks the match back to the ORIGINAL text so * the stored span keeps the document's own characters and spacing. * * A paraphrase fails both passes by construction, which is the point. */ /** * Minimum normalised length a quote must reach before we try to locate it. A * one- or two-character key matches somewhere in almost any document, so a * "match" that short would be noise dressed up as provenance. */ const MIN_ANCHORABLE_LENGTH = 3; /** Longest span we store, mirroring the provenance column's own cap. */ const MAX_SPAN_LENGTH = 2000; export interface AnchoredSource { /** * The verbatim span as it appears in the extracted text, or "" when the * quote could not be located (or there was no extracted text to search). */ sourceText: string; /** True only when `sourceText` was read back out of the document text. */ anchored: boolean; /** Character offset of the span in the extracted text; null when unanchored. */ sourceOffset: number | null; } /** The unanchored result — no quote, no offset, no claim. */ const UNANCHORED: AnchoredSource = { sourceText: "", anchored: false, sourceOffset: null, }; /** * Whitespace-collapsed, case-folded projection of `text`, plus a map from each * projected character back to its index in the original string. Folding is * skipped for any character whose lowercase form is not exactly one code unit, * so the map stays 1:1 and an offset can never drift. */ function project(text: string): { normalised: string; offsets: number[] } { const chars: string[] = []; const offsets: number[] = []; let pendingSpace = false; for (let i = 0; i < text.length; i += 1) { const ch = text[i]!; if (/\s/.test(ch)) { // Collapse a whitespace run to one space, and never lead with one. if (chars.length > 0) pendingSpace = true; continue; } if (pendingSpace) { chars.push(" "); offsets.push(i); pendingSpace = false; } const lower = ch.toLowerCase(); chars.push(lower.length === 1 ? lower : ch); offsets.push(i); } return { normalised: chars.join(""), offsets }; } /** * Locate `quote` inside `sourceText` and return the span the document actually * carries. `sourceText` is the text the extraction ran against — the OCR output * in text mode. Pass `undefined`/`""` for vision mode, where the model read the * rendered page directly and there is no extracted text to verify against: the * result is unanchored, because nothing in that flow can confirm the quote. */ export function anchorSourceText( quote: string, sourceText: string | undefined, ): AnchoredSource { if (!sourceText) return UNANCHORED; const trimmed = quote.trim(); if (!trimmed) return UNANCHORED; // The length floor gates BOTH passes: a one-character key matches the exact // pass just as readily as the folded one, and a hit that short is noise // either way. const needle = project(trimmed); if (needle.normalised.length < MIN_ANCHORABLE_LENGTH) return UNANCHORED; // Pass 1 — the model echoed the span verbatim. const exact = sourceText.indexOf(trimmed); if (exact !== -1) { return { sourceText: sourceText.slice(exact, exact + trimmed.length), anchored: true, sourceOffset: exact, }; } // Pass 2 — same span, reflowed or case-drifted by OCR. const haystack = project(sourceText); const hit = haystack.normalised.indexOf(needle.normalised); if (hit === -1) return UNANCHORED; const start = haystack.offsets[hit]!; const lastIndex = haystack.offsets[hit + needle.normalised.length - 1]!; return { sourceText: sourceText .slice(start, lastIndex + 1) .slice(0, MAX_SPAN_LENGTH), anchored: true, sourceOffset: start, }; }