Merge branch 'main' into fix/wipe-geo-truths

This commit is contained in:
Marc-André Bombeck authored and GitHub committed 2026-08-08 03:29:51 +02:00
commit cbf949d43a
93 files changed
+5417 -236

No files matched your search

@@ -0,0 +1,200 @@
/**
* Structural guard on the measurement ingest boundary.
*
* `VALUE_RANGES` is the band the application declares a metric can occupy.
* Every path a person can drive enforced it from the start; the provider sync
* writers never did, and nothing noticed, because each writer is correct on
* its own terms — it wrote what the remote sent. The gap was only visible from
* above, by listing every writer at once and asking which of them had ever
* been told about the band.
*
* So the list lives here. Every file that writes a `Measurement` row is
* enumerated with the reason it is allowed to, and the guard fails when the
* discovered set stops matching. A provider added tomorrow lands a file that
* is not in the map and has to answer the question before it can ship.
*
* This is a tripwire, not a proof. It cannot show a disposition is the right
* one — only that nobody added a writer without writing down which gate covers
* it. A reviewer who waves through a bogus `derived` label defeats it, and no
* test substitutes for that review.
*/
import { describe, it, expect } from "vitest";
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { walkSourceFiles } from "./helpers/source-files";
const SRC = join(process.cwd(), "src");
/** The gate module every provider-fed writer has to reach. */
const GATE_MODULE = "@/lib/measurements/plausibility-gate";
/** The predicate every person-driven write path has always used. */
const VALIDATOR = "validateMeasurementRange";
/**
* How a writer satisfies the plausibility domain.
*
* - `gate` — provider-fed; must reach {@link GATE_MODULE} itself.
* - `validator` — person-driven; must call {@link VALIDATOR} itself.
* - `upstream` — validated in a named module before the row is built; the
* named module is checked, so the label cannot be a shrug.
* - `derived` — the value is arithmetic over rows already in the table, not a
* number from outside. Gating here would drop a consolidation of data that
* IS present, which is a different failure from admitting one that is not.
* - `literal` — the written value is a constant in the file, so no external
* number can reach the column.
* - `restore` — replays the record's own backup verbatim. Fidelity is the
* contract; a restore that silently dropped rows would be the worse bug,
* and the diagnostic script is how a stored impossible row gets found.
*/
type Disposition =
| { kind: "gate" }
| { kind: "validator" }
| { kind: "upstream"; validatedIn: string }
| { kind: "derived"; why: string }
| { kind: "literal"; why: string }
| { kind: "restore"; why: string };
const WRITERS: Record<string, Disposition> = {
// ── Provider sync writers, gated ──────────────────────────────────────
"lib/withings/sync.ts": { kind: "gate" },
"lib/withings/sync-activity.ts": { kind: "gate" },
"lib/fitbit/sync-core.ts": { kind: "gate" },
"lib/google-health/sync-core.ts": { kind: "gate" },
"lib/nightscout/sync.ts": { kind: "gate" },
// The shared identity seam. Polar, Oura, WHOOP, the Withings sleep leg,
// the Apple export importer and the batch endpoint all write through it,
// so the gate here is what makes a new provider fail closed.
"lib/measurements/reconcile-external-measurement.ts": { kind: "gate" },
// ── Person-driven write paths, validated ──────────────────────────────
"app/api/measurements/route.ts": {
kind: "upstream",
validatedIn: "lib/validations/measurement.ts",
},
"app/api/import/route.ts": { kind: "validator" },
"lib/mcp/writes.ts": { kind: "validator" },
"lib/measurements/create-from-telegram.ts": { kind: "validator" },
"lib/measurements/import-apple-health-export.ts": { kind: "validator" },
"app/api/import/csv/route.ts": {
kind: "upstream",
validatedIn: "lib/import/csv-measurements.ts",
},
// ── Fixed values ──────────────────────────────────────────────────────
"lib/withings/sync-ecg.ts": {
kind: "literal",
why: "an ECG row carries value 1 — the file's own constant, never a provider number",
},
"app/api/mental-health/assessments/route.ts": {
kind: "literal",
why: "the questionnaire score is summed from Zod-bounded item answers",
},
"lib/insights/score-row.ts": {
kind: "literal",
why: "the health score is computed in-app on a closed 0-100 scale",
},
// ── Derived from rows already stored ──────────────────────────────────
"lib/measurements/consolidate-daily-mean.ts": {
kind: "derived",
why: "writes the mean of the day's own stored rows",
},
"lib/measurements/consolidate-legacy-steps.ts": {
kind: "derived",
why: "folds stored per-sample step rows into the day's canonical row",
},
"lib/measurements/dense-intraday-retention.ts": {
kind: "derived",
why: "rebuilds hourly buckets from stored per-sample rows",
},
"lib/measurements/drain-per-sample-cumulative.ts": {
kind: "derived",
why: "reduces stored cumulative per-sample rows to one daily total",
},
// ── Restore ───────────────────────────────────────────────────────────
"app/api/admin/backups/[id]/restore/route.ts": {
kind: "restore",
why: "replays the record's own export; dropping rows would break the restore contract",
},
};
/**
* Any Prisma client handle followed by a measurement write. Whitespace- and
* handle-tolerant on purpose: the earlier generation of this repo's guards
* demanded a literal `prisma.x.findUnique(` and went green because it matched
* nothing, not because nothing was there. The transaction handles in use are
* `tx` and `pc`; a future one is caught by the `[A-Za-z_$][\w$]*` head.
*/
const MEASUREMENT_WRITE =
/[A-Za-z_$][\w$]*\s*\.\s*measurement\s*\.\s*(create|createMany|createManyAndReturn|upsert)\s*\(/;
function sourceFiles(): string[] {
return walkSourceFiles(SRC, { floor: 3000 })
.filter((p) => !p.startsWith("generated/"))
.filter((p) => !p.includes("__tests__"))
.filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx"))
.sort();
}
function read(rel: string): string {
return readFileSync(join(SRC, rel), "utf8");
}
function discoveredWriters(): string[] {
return sourceFiles().filter((rel) => MEASUREMENT_WRITE.test(read(rel)));
}
describe("every measurement writer declares how it meets the plausibility domain", () => {
it("finds the writers at all — an empty sweep is a broken guard, not a clean repo", () => {
// The map is the claim; this asserts the matcher can still see the
// population it describes. A regex that silently stops matching would
// otherwise turn every check below into a tautology.
expect(discoveredWriters().length).toBeGreaterThanOrEqual(15);
});
it("the discovered writer set matches the declared map exactly", () => {
expect(discoveredWriters()).toEqual(Object.keys(WRITERS).sort());
});
it("every provider-fed writer reaches the gate module", () => {
const gated = Object.entries(WRITERS)
.filter(([, d]) => d.kind === "gate")
.map(([rel]) => rel);
expect(gated.length).toBeGreaterThanOrEqual(6);
for (const rel of gated) {
expect(read(rel), `${rel} must import ${GATE_MODULE}`).toContain(
GATE_MODULE,
);
}
});
it("every person-driven writer calls the range validator", () => {
for (const [rel, disposition] of Object.entries(WRITERS)) {
if (disposition.kind === "validator") {
expect(read(rel), `${rel} must call ${VALIDATOR}`).toContain(VALIDATOR);
}
if (disposition.kind === "upstream") {
expect(
read(disposition.validatedIn),
`${disposition.validatedIn} must call ${VALIDATOR} on behalf of ${rel}`,
).toContain(VALIDATOR);
}
}
});
it("the exempt writers carry a written reason", () => {
for (const [rel, disposition] of Object.entries(WRITERS)) {
if (
disposition.kind === "derived" ||
disposition.kind === "literal" ||
disposition.kind === "restore"
) {
expect(disposition.why.length, `${rel} needs a reason`).toBeGreaterThan(
20,
);
}
}
});
});
@@ -0,0 +1,294 @@
/**
* Pair guard binding `prisma/schema.prisma`'s `MoodEntry` and
* `MoodEntryTagLink` models to the disaster-recovery backup selects.
*
* The sibling of `measurement-backup-completeness.test.ts`, written as its own
* file rather than as a second describe inside it, because that guard's own
* note asks for model-by-model growth and one file per model keeps every
* failure message about one model.
*
* The failure it exists to catch, stated by the defect that prompted it: the
* mood payload carried `id, date, mood, score, tags, source, loggedAt,
* externalId, deletedAt, factors` and nothing else. The free-text note, the
* row's timezone and the last-writer-wins counter were selected by the query
* and written to no payload, so a restored entry came back without its text,
* read its day boundaries under the legacy Europe/Berlin assumption, and lost
* the next sync round to a paired device still holding a higher counter. The
* backup validation schema is `.passthrough()`, so nothing complained at parse
* time either. Nothing here was a wrong value; every one of them was silence.
*
* Both ends of the comparison are literals — a Prisma model block and a named
* select object — so the matcher is exact. Every matcher also asserts a
* non-zero match count: an empty match set fails rather than passing quietly,
* which is how the bearer-scope guard stayed green for months while matching
* nothing at all.
*
* Mutation check: comment out `tz: true` in `MOOD_ENTRY_BACKUP_SELECT` and
* this goes red naming that column; add a scalar to the model without adding
* it to the select and it goes red naming the new one; rename the mapper's
* `moodEntry` parameter and the serialization assertion goes red on all of
* them at once.
*/
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, it, expect } from "vitest";
const ROOT = process.cwd();
const SCHEMA = join(ROOT, "prisma", "schema.prisma");
const BACKUP_BUILDER = join(
ROOT,
"src",
"lib",
"export",
"full-backup-payload.ts",
);
/**
* Columns the backup is correct to omit, each with the reason. The reasons are
* length-checked below: an exclusion is a claim, and a claim needs saying.
*/
const EXCLUDED_MOOD_ENTRY_COLUMNS: Record<string, string> = {
userId: "user-scoped backup; owner is re-derived from the restoring account",
};
const EXCLUDED_TAG_LINK_COLUMNS: Record<string, string> = {
moodEntryId: "re-bound to the restored parent row's id, never carried",
moodTagId:
"the link travels by the tag's key; a source-instance id need not exist in the target catalogue",
createdAt:
"the link's own row stamp; the entry carries the moment the person logged it, and a key-only BINARY tag has nowhere to put a second one",
};
function modelBlock(schema: string, model: string): string {
const start = schema.indexOf(`model ${model} {`);
expect(start, `model ${model} not found`).toBeGreaterThan(-1);
const end = schema.indexOf("\n}", start);
expect(end).toBeGreaterThan(start);
return schema.slice(start, end);
}
/** Names of every `model` block, so relation fields can be told apart. */
function declaredModels(schema: string): Set<string> {
const out = new Set<string>();
for (const m of schema.matchAll(/^model\s+([A-Za-z_]\w*)\s*\{/gm)) {
out.add(m[1]);
}
return out;
}
/**
* Scalar (column-backed) fields of a model block: everything that is not a
* relation to another model. Enums and native scalars both map to columns and
* therefore both belong in a backup.
*/
function scalarFields(block: string, models: Set<string>): string[] {
const fields: string[] = [];
for (const rawLine of block.split("\n").slice(1)) {
const line = rawLine.trim();
if (!line || line.startsWith("//") || line.startsWith("@@")) continue;
const m = /^([a-zA-Z_]\w*)\s+([A-Za-z_]\w*)(\[\])?(\?)?/.exec(line);
if (!m) continue;
const [, name, type, isList] = m;
if (isList) continue;
if (models.has(type)) continue;
if (line.includes("@relation")) continue;
fields.push(name);
}
return fields;
}
/** Keys of a named `const <NAME> = {` select literal. */
function selectFields(source: string, constant: string): string[] {
const start = source.indexOf(`const ${constant} = {`);
expect(start, `${constant} not found`).toBeGreaterThan(-1);
const end = source.indexOf("\n}", start);
expect(end).toBeGreaterThan(start);
const body = source.slice(start, end);
return [...body.matchAll(/^ {2}([a-zA-Z_$][\w$]*): true,$/gm)].map(
(m) => m[1],
);
}
/**
* The builder with its comment lines removed.
*
* Every matcher below runs against this rather than against the raw file,
* because prose describing the defect reads exactly like the defect: the first
* draft of this guard went red on its own explanation of the filter it exists
* to forbid. Whole comment lines only — a trailing comment after code is left
* alone, which is safe here because the only risk it carries is a matcher
* accepting a column named in prose, and the select comparison beside it is
* literal either way.
*/
function stripCommentLines(source: string): string {
return source
.split("\n")
.filter((line) => {
const trimmed = line.trim();
return !(
trimmed.startsWith("//") ||
trimmed.startsWith("/*") ||
trimmed.startsWith("*")
);
})
.join("\n");
}
/**
* Columns either serialization arm actually reads off the mapped row.
*
* The capture is the RIGHT-hand side — the column — not the payload key, which
* matters here in a way it did not for `Measurement`: the mood payload renames
* `moodLoggedAt` to `loggedAt` on the wire, so a left-hand matcher would report
* the column as dropped while the value rides perfectly well.
*
* The two arms share one restorable wire format, so this is a union and
* therefore a weaker claim than the select comparison: it catches a column that
* is selected and then written to neither payload, not one carried by only a
* single arm.
*/
function serialisedFields(source: string, variable: string): Set<string> {
const matches = [
...source.matchAll(
new RegExp(`[:(,]\\s*${variable}\\.([a-zA-Z_$][\\w$]*)`, "g"),
),
];
expect(
matches.length,
`no \`${variable}.<column>\` reads found — the mapper parameter was renamed and this matcher now proves nothing`,
).toBeGreaterThan(0);
return new Set(matches.map((m) => m[1]));
}
describe("mood backup completeness", () => {
const schema = readFileSync(SCHEMA, "utf8");
const source = stripCommentLines(readFileSync(BACKUP_BUILDER, "utf8"));
const models = declaredModels(schema);
const entryColumns = scalarFields(modelBlock(schema, "MoodEntry"), models);
const entrySelected = selectFields(source, "MOOD_ENTRY_BACKUP_SELECT");
const linkColumns = scalarFields(
modelBlock(schema, "MoodEntryTagLink"),
models,
);
const linkSelected = selectFields(source, "MOOD_ENTRY_TAG_LINK_SELECT");
it("parses both ends into a plausible shape", () => {
expect(models.size).toBeGreaterThan(50);
expect(entryColumns.length).toBeGreaterThan(10);
expect(entryColumns).toContain("noteEncrypted");
expect(entryColumns).toContain("syncVersion");
expect(entryColumns).not.toContain("user");
expect(entryColumns).not.toContain("tagLinks");
expect(entrySelected.length).toBeGreaterThan(10);
expect(linkColumns).toEqual(
expect.arrayContaining([
"moodEntryId",
"moodTagId",
"createdAt",
"rating",
]),
);
expect(linkColumns).not.toContain("moodEntry");
expect(linkColumns).not.toContain("moodTag");
expect(linkSelected.length).toBeGreaterThan(0);
});
it("the backup select covers every MoodEntry column", () => {
const missing = entryColumns.filter(
(c) => !entrySelected.includes(c) && !(c in EXCLUDED_MOOD_ENTRY_COLUMNS),
);
if (missing.length > 0) {
const report = missing.map((c) => ` ❌ ${c}`).join("\n");
throw new Error(
`MOOD_ENTRY_BACKUP_SELECT omits ${missing.length} column(s) of the MoodEntry model:\n${report}\n\n` +
"A disaster-recovery restore brings these back at their column " +
"default. Add each to the select and to both serialization arms, " +
"extend `moodEntrySchema` and the restore write — or add it to " +
"EXCLUDED_MOOD_ENTRY_COLUMNS with the reason the backup is correct " +
"to drop it.",
);
}
});
it("every selected MoodEntry column reaches a serialized payload", () => {
const emitted = serialisedFields(source, "moodEntry");
const dropped = entrySelected.filter((c) => !emitted.has(c));
if (dropped.length > 0) {
const report = dropped.map((c) => ` ❌ ${c}`).join("\n");
throw new Error(
`${dropped.length} selected MoodEntry column(s) reach neither serialization arm:\n${report}\n\n` +
"The query pays for them and no payload carries them, so a restore " +
"cannot put them back. Emit each from the disaster-recovery arm " +
"(and, where it is readable and not a tombstone, from the portable " +
"arm too).",
);
}
});
it("the tag-link select covers every MoodEntryTagLink column", () => {
const missing = linkColumns.filter(
(c) => !linkSelected.includes(c) && !(c in EXCLUDED_TAG_LINK_COLUMNS),
);
expect(
missing,
`MOOD_ENTRY_TAG_LINK_SELECT omits MoodEntryTagLink column(s): ${missing.join(", ")}`,
).toEqual([]);
});
it("every selected tag-link column reaches a serialized payload", () => {
const emitted = serialisedFields(source, "tagLink");
const dropped = linkSelected.filter((c) => !emitted.has(c));
expect(
dropped,
`selected but serialized by neither arm: ${dropped.join(", ")}`,
).toEqual([]);
});
it("both structured-tag arms are carried, not just the rated one", () => {
// The defect this file was written for: the tag-link read carried a
// filter, and a BINARY link's rating is NULL by definition, so every
// present/absent tag — the kind most people pick — was excluded from the
// file and could not come back from it. The claim is therefore about the
// read itself: it takes the links whole, unfiltered.
//
// Asserted on booleans rather than on the file contents so a failure reads
// as one line instead of printing the whole builder back.
const tagLinkRead = /tagLinks:\s*\{([\s\S]*?)\}/.exec(source);
expect(
tagLinkRead,
"no `tagLinks:` read found in the backup builder — this matcher proves nothing",
).not.toBeNull();
expect(
/where/.test(tagLinkRead![1]),
"the tag-link read filters again; a filter here is what dropped every BINARY link from the backup",
).toBe(false);
expect(
source.includes("structuredTags"),
"no `structuredTags` key in the payload — the BINARY links have no carrier",
).toBe(true);
});
it("the exclusion lists carry no stale entry", () => {
for (const [column, reason] of Object.entries(
EXCLUDED_MOOD_ENTRY_COLUMNS,
)) {
expect(
entryColumns,
`${column} is excluded but is no longer a MoodEntry column`,
).toContain(column);
expect(reason.length).toBeGreaterThan(20);
}
for (const [column, reason] of Object.entries(EXCLUDED_TAG_LINK_COLUMNS)) {
expect(
linkColumns,
`${column} is excluded but is no longer a MoodEntryTagLink column`,
).toContain(column);
expect(reason.length).toBeGreaterThan(20);
}
});
});
@@ -724,6 +724,13 @@ const OCR_EXCEPTIONS: Readonly<Record<string, string>> = {
"inserted.unit": OCR_ROW_EXCEPTION,
"inserted.referenceLow": OCR_ROW_EXCEPTION,
"inserted.referenceHigh": OCR_ROW_EXCEPTION,
"inserted.catalogReferenceLow": OCR_ROW_EXCEPTION,
"inserted.catalogReferenceHigh": OCR_ROW_EXCEPTION,
"inserted.sourceReferenceLow": OCR_ROW_EXCEPTION,
"inserted.sourceReferenceHigh": OCR_ROW_EXCEPTION,
"inserted.sourceReferenceText": OCR_ROW_EXCEPTION,
"inserted.referenceOrigin": OCR_ROW_EXCEPTION,
"inserted.referenceDivergesFromCatalog": OCR_ROW_EXCEPTION,
"inserted.takenAt": OCR_ROW_EXCEPTION,
"inserted.source": OCR_ROW_EXCEPTION,
"inserted.hasNote": OCR_ROW_EXCEPTION,
@@ -684,6 +684,10 @@ const handler = apiHandler(
entry.factors.map((factor) => factor.key),
);
const factorKeys = [...new Set(referencedFactorKeys)];
const referencedTagKeys = payload.moodEntries.flatMap(
(entry) => entry.structuredTags,
);
const tagKeys = [...new Set(referencedTagKeys)];
// Re-create the account's own tag definitions first. The lookup
// below used to ask only for the seeded catalogue (`userId: null`),
// so a single custom rated tag made the whole restore throw
@@ -740,6 +744,30 @@ const handler = apiHandler(
referencedFactorKeys,
);
// The BINARY half of the same taxonomy, looked up the same way and
// reported the same way. It gets its own family because an
// operator reading the report needs to know whether a rated
// factor's score or a ticked tag went missing; they are different
// losses on the same entry.
const tagRows =
tagKeys.length === 0
? []
: await tx.moodTag.findMany({
where: {
key: { in: tagKeys },
kind: "BINARY",
OR: [{ userId: null }, { userId: ownerId }],
},
select: { id: true, key: true },
});
const tagByKey = new Map(tagRows.map((tag) => [tag.key, tag.id]));
recordUnknownKeys(
skips,
"moodTag",
tagKeys.filter((key) => !tagByKey.has(key)),
referencedTagKeys,
);
for (const entry of payload.moodEntries) {
const moodLoggedAt = new Date(entry.loggedAt);
const restoredData = {
@@ -747,10 +775,44 @@ const handler = apiHandler(
mood: entry.mood,
score: entry.score,
tags: entry.tags ?? null,
// The note comes back the way every other dual-column note
// does: ciphertext verbatim when the file carries it, and a
// portable file's plaintext re-encrypted on the way in, with
// the legacy column left null either way.
note: null,
noteEncrypted:
entry.noteEncrypted == null
? encryptNote(entry.note ?? null)
: decodeEncryptedBytes(entry.noteEncrypted),
source: entry.source ?? "MOODLOG",
externalId: entry.externalId ?? null,
moodLoggedAt,
// The zone the `date` string is anchored to. A file written
// before this rode the payload carries nothing, and NULL is
// what such a row meant anyway: the legacy Europe/Berlin
// reading. So an old file keeps its old day boundaries and a
// new one keeps the zone the person actually logged in.
tz: entry.tz ?? null,
// Mood reconciles last-writer-wins by `syncVersion`. Restoring
// at the schema default would hand the next sync round to any
// paired device still holding a higher number, and the device
// would overwrite the recovered row with what it had — a
// restore that loses to the client is worse than one that
// fails. An old file carries nothing here and keeps the
// default, which is the value those rows were written with.
...(entry.syncVersion === undefined
? {}
: { syncVersion: entry.syncVersion }),
deletedAt: entry.deletedAt ? new Date(entry.deletedAt) : null,
...(entry.syncedAt
? { syncedAt: new Date(entry.syncedAt) }
: {}),
...(entry.createdAt
? { createdAt: new Date(entry.createdAt) }
: {}),
...(entry.updatedAt
? { updatedAt: new Date(entry.updatedAt) }
: {}),
};
const createData = { userId: ownerId, ...restoredData };
const restored = entry.id
@@ -798,8 +860,19 @@ const handler = apiHandler(
]
: [];
});
if (factorLinks.length > 0) {
await tx.moodEntryTagLink.createMany({ data: factorLinks });
// The other arm. A BINARY link carries no score — the presence
// of the row is the signal — so `rating` is NULL by definition,
// which is exactly what the backup's old tag-link filter took to
// mean "nothing here".
const tagLinks = entry.structuredTags.flatMap((key) => {
const moodTagId = tagByKey.get(key);
return moodTagId
? [{ moodEntryId: restored.id, moodTagId, rating: null }]
: [];
});
const links = [...factorLinks, ...tagLinks];
if (links.length > 0) {
await tx.moodEntryTagLink.createMany({ data: links });
}
}
}
@@ -894,6 +967,9 @@ const handler = apiHandler(
unit: lab.unit,
referenceLow: lab.referenceLow ?? null,
referenceHigh: lab.referenceHigh ?? null,
sourceReferenceLow: lab.sourceReferenceLow ?? null,
sourceReferenceHigh: lab.sourceReferenceHigh ?? null,
sourceReferenceText: lab.sourceReferenceText ?? null,
takenAt: new Date(lab.takenAt),
source: lab.source,
noteEncrypted:
@@ -57,6 +57,7 @@ function editToFactData(edit: InboundFactEdit): FactData {
unit: edit.unit ?? null,
referenceLow: edit.referenceLow ?? null,
referenceHigh: edit.referenceHigh ?? null,
referenceText: edit.referenceText ?? null,
effectiveDate: edit.effectiveDate ?? null,
};
return data;
@@ -46,6 +46,9 @@ export const GET = apiHandler(async () => {
value: true,
referenceLow: true,
referenceHigh: true,
sourceReferenceLow: true,
sourceReferenceHigh: true,
sourceReferenceText: true,
takenAt: true,
},
orderBy: { takenAt: "desc" },
@@ -58,6 +61,9 @@ export const GET = apiHandler(async () => {
value: r.value as number,
referenceLow: r.referenceLow,
referenceHigh: r.referenceHigh,
sourceReferenceLow: r.sourceReferenceLow,
sourceReferenceHigh: r.sourceReferenceHigh,
sourceReferenceText: r.sourceReferenceText,
takenAt: r.takenAt,
})),
);
+35
View File
@@ -143,6 +143,12 @@ export const PUT = apiHandler(
// historical columns the serialiser ignores. Reject the attempt with a
// 422 rather than silently no-op, so the client never believes an edit
// landed. Edit such fields on the Biomarker itself instead.
//
// `sourceReference*` is deliberately NOT on that list. The catalog band is
// a property of the marker; the source window is a property of THIS
// reading — it is what one report printed beside one value — so it is
// editable on a linked row and there is nothing on the biomarker to edit
// it on instead.
const isLinked = existing.biomarkerId !== null;
if (
isLinked &&
@@ -194,6 +200,15 @@ export const PUT = apiHandler(
if (d.unit !== undefined) data.unit = d.unit;
if (d.referenceLow !== undefined) data.referenceLow = d.referenceLow;
if (d.referenceHigh !== undefined) data.referenceHigh = d.referenceHigh;
if (d.sourceReferenceLow !== undefined) {
data.sourceReferenceLow = d.sourceReferenceLow;
}
if (d.sourceReferenceHigh !== undefined) {
data.sourceReferenceHigh = d.sourceReferenceHigh;
}
if (d.sourceReferenceText !== undefined) {
data.sourceReferenceText = d.sourceReferenceText;
}
if (d.takenAt !== undefined) data.takenAt = d.takenAt;
// Inverted-range guard for a PARTIAL bound update. The schema-level
@@ -212,6 +227,20 @@ export const PUT = apiHandler(
errorCode: "labs.update.referenceRangeInvalid",
});
}
// The same partial-update hole exists on the source window, and closes the
// same way.
if (
isInvertedRange(
effectiveBound(d.sourceReferenceLow, existing.sourceReferenceLow),
effectiveBound(d.sourceReferenceHigh, existing.sourceReferenceHigh),
)
) {
return apiError(
"sourceReferenceLow must not exceed sourceReferenceHigh",
422,
{ errorCode: "labs.update.sourceReferenceRangeInvalid" },
);
}
if (d.note !== undefined) {
data.noteEncrypted = d.note ? encryptNoteToBytes(d.note) : null;
}
@@ -239,6 +268,9 @@ export const PUT = apiHandler(
panel: existing.panel,
referenceLow: existing.referenceLow,
referenceHigh: existing.referenceHigh,
sourceReferenceLow: existing.sourceReferenceLow,
sourceReferenceHigh: existing.sourceReferenceHigh,
sourceReferenceText: existing.sourceReferenceText,
takenAt: existing.takenAt,
},
after: {
@@ -249,6 +281,9 @@ export const PUT = apiHandler(
panel: updated.panel,
referenceLow: updated.referenceLow,
referenceHigh: updated.referenceHigh,
sourceReferenceLow: updated.sourceReferenceLow,
sourceReferenceHigh: updated.sourceReferenceHigh,
sourceReferenceText: updated.sourceReferenceText,
takenAt: updated.takenAt,
},
redacted: d.note !== undefined ? ["note"] : [],
+13 -2
View File
@@ -33,6 +33,7 @@ import { withIdempotency } from "@/lib/idempotency";
import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy";
import { emitDataArrival } from "@/lib/arrivals/emit-shared";
import { resolveOrMintBiomarker } from "@/lib/labs/biomarker-store";
import { parseReferenceRange } from "@/lib/labs/parse-reference-range";
import { serialiseLabResult } from "@/lib/labs/serialise";
import {
linkOcrLabsToVaultDocument,
@@ -151,12 +152,19 @@ async function commitOcrRows(request: NextRequest) {
}
const isQualitative = row.valueText !== undefined;
// The window the scan read off the report. The confirmed numeric bounds
// win over a re-parse of the string (the human may have corrected them on
// the review screen); the string rides along either way.
const printed = parseReferenceRange(row.referenceText, row.unit);
const sourceLow = row.referenceLow ?? printed?.low ?? null;
const sourceHigh = row.referenceHigh ?? printed?.high ?? null;
const sourceText = printed?.text ?? null;
const biomarker = await resolveOrMintBiomarker(user.id, {
analyte: row.analyte,
// A qualitative reading has no numeric unit / range.
unit: isQualitative ? (row.unit ?? "") : (row.unit as string),
referenceLow: isQualitative ? null : (row.referenceLow ?? null),
referenceHigh: isQualitative ? null : (row.referenceHigh ?? null),
referenceLow: isQualitative ? null : sourceLow,
referenceHigh: isQualitative ? null : sourceHigh,
panel: row.panel ?? null,
});
@@ -173,6 +181,9 @@ async function commitOcrRows(request: NextRequest) {
unit: biomarker.unit,
referenceLow: biomarker.lowerBound,
referenceHigh: biomarker.upperBound,
sourceReferenceLow: isQualitative ? null : sourceLow,
sourceReferenceHigh: isQualitative ? null : sourceHigh,
sourceReferenceText: isQualitative ? null : sourceText,
takenAt: row.takenAt,
source: "OCR",
noteEncrypted: null,
+15 -1
View File
@@ -184,6 +184,9 @@ async function postLabResult(request: NextRequest) {
unit,
referenceLow,
referenceHigh,
sourceReferenceLow,
sourceReferenceHigh,
sourceReferenceText,
takenAt,
note,
source,
@@ -248,6 +251,13 @@ async function postLabResult(request: NextRequest) {
unit: biomarker.unit,
referenceLow: biomarker.lowerBound,
referenceHigh: biomarker.upperBound,
// The window this reading's own report printed, kept beside the catalog
// stamp rather than folded into it: it governs THIS reading's verdict,
// and the catalog band stays the net for every reading without one. A
// qualitative reading has no window to record.
sourceReferenceLow: isQualitative ? null : (sourceReferenceLow ?? null),
sourceReferenceHigh: isQualitative ? null : (sourceReferenceHigh ?? null),
sourceReferenceText: isQualitative ? null : (sourceReferenceText ?? null),
takenAt,
// v1.25 (iOS #36) — provenance: the on-device-OCR path posts
// `source: "OCR"`; an omitted field stays "MANUAL" so the legacy
@@ -258,6 +268,7 @@ async function postLabResult(request: NextRequest) {
},
});
invalidateUserHealthScore(user.id);
const dto = serialiseLabResult(created, biomarker);
await auditLog("labResult.create", {
userId: user.id,
@@ -275,6 +286,9 @@ async function postLabResult(request: NextRequest) {
structured: biomarkerId !== undefined,
// v1.25 (iOS #36) — provenance of the reading ("MANUAL" | "OCR").
source: source ?? "MANUAL",
// Which window governs this reading's verdict, so a dashboard can see
// how often a report's own range displaces the catalog band.
referenceOrigin: dto.referenceOrigin,
},
});
@@ -298,5 +312,5 @@ async function postLabResult(request: NextRequest) {
source: "manual",
}).catch(() => {});
return apiSuccess(serialiseLabResult(created, biomarker), 201);
return apiSuccess(dto, 201);
}
+11
View File
@@ -620,6 +620,17 @@ async function postBatch(request: NextRequest): Promise<Response> {
};
failedCount++;
continue;
case "rejected_range":
// The reconciler carries the same plausibility gate this route
// already applied above, so reaching here means the two disagreed —
// report it exactly as the earlier guard would have, and let the
// wide-event tally record which end refused.
results[p.index] = {
index: p.index,
status: "skipped",
reason: "value_out_of_range",
};
continue;
}
if (isMergeableSource(p.row.source as string)) {
+3
View File
@@ -194,6 +194,9 @@ function catalogueLabel(
if (catalogue === "illnessSymptom") {
return t("admin.section.backups.restoreSkippedIllnessSymptom");
}
if (catalogue === "moodTag") {
return t("admin.section.backups.restoreSkippedMoodTag");
}
return t("admin.section.backups.restoreSkippedMoodFactor");
}
@@ -64,6 +64,13 @@ function reading(id: string, value: number, takenAt: string): LabResultDto {
unit: "mg/dL",
referenceLow: 0,
referenceHigh: 100,
catalogReferenceLow: 0,
catalogReferenceHigh: 100,
sourceReferenceLow: null,
sourceReferenceHigh: null,
sourceReferenceText: null,
referenceOrigin: "catalog",
referenceDivergesFromCatalog: false,
takenAt,
source: "MANUAL",
hasNote: false,
+128 -1
View File
@@ -17,6 +17,7 @@ import { Button } from "@/components/ui/button";
import { CHART_HEIGHT_PX } from "@/lib/charts/constants";
import { prefersReducedMotion } from "@/lib/charts/reduced-motion";
import { formatLabValue } from "@/lib/labs/format-value";
import { formatReferenceRange } from "@/lib/labs/reference-range";
import { useTranslations, useFormatters } from "@/lib/i18n/context";
import { RichChartTooltip, type RichTooltipRow } from "../charts/chart-tooltip";
@@ -32,6 +33,15 @@ import type { LabResultDto } from "./types";
* MUTED primary fill — never a danger colour — honouring the no-alarming-
* colour rule. Every reading carries the same y-axis unit because a chart is
* one biomarker only.
*
* A SECOND band draws the window a source report printed, when the readings in
* view carry one. It is visually distinct from the catalog band — an outlined
* band in the chart's secondary hue against the catalog's filled primary — and
* both stay muted; the in/out verdict is the badge's job, never the chart's
* colour. The source window is a per-reading fact, so the band is drawn from
* the NEWEST reading in view that carries one and the legend dates it. When
* the readings in view carry more than one distinct printed window the legend
* says so rather than implying one window covered the whole series.
*/
const RANGE_DAYS = [
@@ -46,6 +56,26 @@ type RangeKey = (typeof RANGE_DAYS)[number]["key"];
interface ChartPoint {
timestamp: number;
value: number;
/** The window this reading was judged against, for the tooltip. */
referenceLow: number | null;
referenceHigh: number | null;
referenceOrigin: LabResultDto["referenceOrigin"];
sourceReferenceText: string | null;
}
/** The source window drawn as the second band, resolved from the readings in view. */
interface SourceBand {
low: number | null;
high: number | null;
/** The reading the band was taken from. */
takenAt: number;
/** True when the readings in view printed more than one distinct window. */
mixed: boolean;
}
/** Identity of a printed window, for counting distinct ones. */
function bandKey(low: number | null, high: number | null): string {
return `${low ?? ""}|${high ?? ""}`;
}
export function LabBiomarkerChart({
@@ -87,9 +117,37 @@ export function LabBiomarkerChart({
return filtered.map((r) => ({
timestamp: new Date(r.takenAt).getTime(),
value: r.value,
referenceLow: r.referenceLow,
referenceHigh: r.referenceHigh,
referenceOrigin: r.referenceOrigin,
sourceReferenceText: r.sourceReferenceText,
}));
}, [readings, range, nowMs]);
// The source band: the newest in-view reading whose report printed a window
// with usable bounds. Readings without one contribute nothing — an absent
// window is absent, not a reason to widen or narrow the band.
const sourceBand = useMemo<SourceBand | null>(() => {
const withWindow = points.filter(
(p) =>
p.referenceOrigin === "source" &&
(p.referenceLow !== null || p.referenceHigh !== null),
);
if (withWindow.length === 0) return null;
const newest = withWindow.reduce((a, b) =>
b.timestamp > a.timestamp ? b : a,
);
const distinct = new Set(
withWindow.map((p) => bandKey(p.referenceLow, p.referenceHigh)),
);
return {
low: newest.referenceLow,
high: newest.referenceHigh,
takenAt: newest.timestamp,
mixed: distinct.size > 1,
};
}, [points]);
// Y domain wraps both the data and the reference window so the band is
// always visible even when every reading sits inside it.
const yDomain = useMemo<[number, number]>(() => {
@@ -97,15 +155,23 @@ export function LabBiomarkerChart({
const candidates = [...values];
if (lowerBound != null) candidates.push(lowerBound);
if (upperBound != null) candidates.push(upperBound);
// The source band has to fit too, or the second band is drawn off-canvas
// and the chart silently shows one window while claiming two.
if (sourceBand?.low != null) candidates.push(sourceBand.low);
if (sourceBand?.high != null) candidates.push(sourceBand.high);
if (candidates.length === 0) return [0, 1];
const min = Math.min(...candidates);
const max = Math.max(...candidates);
const pad = (max - min || Math.abs(max) || 1) * 0.1;
return [min - pad, max + pad];
}, [points, lowerBound, upperBound]);
}, [points, lowerBound, upperBound, sourceBand]);
const animate = !prefersReducedMotion();
const primary = "var(--primary)";
// A second muted hue from the chart palette. Distinct from the primary band
// by hue AND by treatment (outline vs fill), so the two are told apart
// without relying on colour alone.
const secondary = "var(--chart-2)";
return (
<div className="space-y-3">
@@ -185,6 +251,42 @@ export function LabBiomarkerChart({
strokeDasharray="4 4"
/>
) : null}
{/* The source window — outlined, not filled, so it reads as a
second band over the catalog one rather than a second fill
competing with it. Same no-alarm-colour rule. */}
{sourceBand &&
sourceBand.low != null &&
sourceBand.high != null ? (
<ReferenceArea
y1={sourceBand.low}
y2={sourceBand.high}
fill={secondary}
fillOpacity={0.06}
stroke={secondary}
strokeOpacity={0.5}
strokeDasharray="2 3"
/>
) : null}
{sourceBand &&
sourceBand.low != null &&
sourceBand.high == null ? (
<ReferenceLine
y={sourceBand.low}
stroke={secondary}
strokeOpacity={0.6}
strokeDasharray="2 3"
/>
) : null}
{sourceBand &&
sourceBand.high != null &&
sourceBand.low == null ? (
<ReferenceLine
y={sourceBand.high}
stroke={secondary}
strokeOpacity={0.6}
strokeDasharray="2 3"
/>
) : null}
<Tooltip
content={(props) => {
const active = props.active ?? false;
@@ -201,6 +303,22 @@ export function LabBiomarkerChart({
color: primary,
},
];
// Name the window THIS reading was judged against. Two
// readings on one chart can carry different printed windows,
// so the band alone cannot answer it per point.
if (point.referenceOrigin === "source") {
rows.push({
name: t("labs.chart.sourceRangeLabel"),
value:
point.sourceReferenceText ??
`${formatReferenceRange(
point.referenceLow,
point.referenceHigh,
formatLabValue,
)} ${unit}`.trim(),
color: secondary,
});
}
return (
<RichChartTooltip
active
@@ -221,6 +339,15 @@ export function LabBiomarkerChart({
/>
</ComposedChart>
</ResponsiveContainer>
{sourceBand ? (
<p className="text-muted-foreground mt-2 text-xs">
{sourceBand.mixed
? t("labs.chart.sourceBandMixed")
: t("labs.chart.sourceBandLegend", {
date: fmt.dateShortSmart(new Date(sourceBand.takenAt)),
})}
</p>
) : null}
</div>
)}
</div>
+73
View File
@@ -22,6 +22,7 @@ import {
import { Textarea } from "@/components/ui/textarea";
import { apiGet, apiPost } from "@/lib/api/api-fetch";
import { localizedApiError } from "@/lib/api/localized-error";
import { parseReferenceRange } from "@/lib/labs/parse-reference-range";
import { formatReferenceRange } from "@/lib/labs/reference-range";
import { formatLabValue } from "@/lib/labs/format-value";
import { useTranslations } from "@/lib/i18n/context";
@@ -118,6 +119,13 @@ export function LabForm({
const [valueText, setValueText] = useState("");
const [takenAt, setTakenAt] = useState(defaultTakenAtValue);
const [note, setNote] = useState("");
// The window the paper report printed for THIS reading. Optional and folded
// away by default: most entries are typed against a catalog marker and never
// need it, and a field nobody fills is a field in the way. One text input,
// because that is how a report states it — "3,5 - 5,0", "< 5", "bis 5,0".
// The bounds are derived from what is typed; what is typed is kept verbatim.
const [sourceRangeOpen, setSourceRangeOpen] = useState(false);
const [sourceRange, setSourceRange] = useState("");
// v1.30.1 H3 — which submit button is in flight, so the OTHER one doesn't
// also show a spinner. `!== null` replaces the old plain `submitting`
// boolean for every disabled check below.
@@ -193,11 +201,27 @@ export function LabForm({
setPendingAction(keepOpen ? "saveAndAddAnother" : "save");
try {
// Parse client-side so the user sees the derived window before saving,
// and send BOTH: the bounds where they could be read, the text always.
const printed = isQualitative
? null
: parseReferenceRange(sourceRange, selected?.unit);
const created = await apiPost<LabResultDto>("/api/labs", {
biomarkerId,
...(isQualitative
? { valueText: qualitativeValue }
: { value: numericValue }),
...(printed
? {
...(printed.low !== null
? { sourceReferenceLow: printed.low }
: {}),
...(printed.high !== null
? { sourceReferenceHigh: printed.high }
: {}),
sourceReferenceText: printed.text,
}
: {}),
takenAt: new Date(takenAt).toISOString(),
...(note.trim() ? { note: note.trim() } : {}),
});
@@ -221,6 +245,8 @@ export function LabForm({
setValue("");
setValueText("");
setNote("");
// The next analyte off the same report has its own printed window.
setSourceRange("");
setError(null);
onSavedKeepOpen?.(created);
// Return focus to wherever the next entry starts so the flow
@@ -252,6 +278,20 @@ export function LabForm({
)
: "";
// What the typed string resolved to, shown back so the user can see whether
// a window was read out of it before the reading is saved. When none was,
// the hint says the text is kept as written rather than staying silent.
const parsedSourceRange = parseReferenceRange(sourceRange, selected?.unit);
const sourceRangeHint = parsedSourceRange
? parsedSourceRange.low === null && parsedSourceRange.high === null
? t("labs.form.sourceRangeUnparsed")
: `${formatReferenceRange(
parsedSourceRange.low,
parsedSourceRange.high,
formatLabValue,
)} ${selected?.unit ?? ""}`.trim()
: "";
const footerNode = (
<>
{onCancel ? (
@@ -415,6 +455,39 @@ export function LabForm({
</p>
) : null}
{resultType === "numeric" ? (
<div className="space-y-1.5">
{sourceRangeOpen ? (
<>
<Label htmlFor="lab-sourceRange">
{t("labs.form.sourceRange")}
</Label>
<Input
id="lab-sourceRange"
value={sourceRange}
onChange={(e) => setSourceRange(e.target.value)}
placeholder={t("labs.form.sourceRangePlaceholder")}
maxLength={120}
autoFocus
/>
<p className="text-muted-foreground text-xs">
{sourceRangeHint || t("labs.form.sourceRangeHint")}
</p>
</>
) : (
<Button
type="button"
variant="ghost"
size="sm"
className="text-muted-foreground h-auto px-0 text-xs"
onClick={() => setSourceRangeOpen(true)}
>
{t("labs.form.sourceRangeAdd")}
</Button>
)}
</div>
) : null}
<div className="space-y-1.5">
<Label htmlFor="lab-note">{t("labs.form.note")}</Label>
<Textarea
+8 -1
View File
@@ -35,6 +35,7 @@ import { useTranslations } from "@/lib/i18n/context";
import { queryKeys } from "@/lib/query-keys";
import { ReferenceRangeBadge } from "./reference-range-badge";
import { SourceRangeNote } from "./source-range-note";
import type { LabResultDetailDto, LabResultDto } from "./types";
const NOTE_MAX_LENGTH = 2000;
@@ -356,7 +357,13 @@ export function LabHistoryList({ readings }: { readings: LabResultDto[] }) {
{formatDateShort(r.takenAt, true)}
</TableCell>
<TableCell>
<ReferenceRangeBadge status={r.rangeStatus} />
<div className="flex flex-col items-start gap-1">
<ReferenceRangeBadge status={r.rangeStatus} />
<SourceRangeNote
reading={r}
className="text-muted-foreground text-xs"
/>
</div>
</TableCell>
<TableCell className="text-muted-foreground text-sm">
{r.hasNote ? t("labs.hasNote") : ""}
+5
View File
@@ -23,6 +23,7 @@ import { applyOrder, useModuleListPrefs } from "@/lib/module-list-prefs";
import { LabTrendSparkline } from "./lab-trend-sparkline";
import { ReferenceRangeBadge } from "./reference-range-badge";
import { SourceRangeNote } from "./source-range-note";
import type { LabResultDto, LabResultListResponse } from "./types";
interface MarkerGroup {
@@ -316,6 +317,10 @@ export function LabList({ onAddFirst }: { onAddFirst?: () => void } = {}) {
)}
</span>
) : null}
<SourceRangeNote
reading={group.latest}
className="text-xs"
/>
<span className="text-xs">
{formatDate(group.latest.takenAt)}
</span>
@@ -76,6 +76,7 @@ function toCommitRow(row: OcrReviewRow): OcrCommitRowInput | null {
takenAt: takenAt.toISOString(),
...(row.referenceLow !== null ? { referenceLow: row.referenceLow } : {}),
...(row.referenceHigh !== null ? { referenceHigh: row.referenceHigh } : {}),
...(row.referenceText ? { referenceText: row.referenceText } : {}),
};
}
+3
View File
@@ -16,6 +16,8 @@ export interface OcrReviewRow {
unit: string | null;
referenceLow: number | null;
referenceHigh: number | null;
/** The reference range as printed on the scanned report, verbatim. */
referenceText: string | null;
/** ISO calendar day (YYYY-MM-DD) or null. */
takenAt: string | null;
confidence: OcrExtractedRowDto["confidence"];
@@ -39,6 +41,7 @@ export function seedReviewRows(
unit: row.unit,
referenceLow: row.referenceLow,
referenceHigh: row.referenceHigh,
referenceText: row.referenceText,
takenAt: row.takenAt ?? fallbackDate,
confidence: row.confidence,
biomarkerMatch: row.biomarkerMatch,
+26
View File
@@ -247,6 +247,32 @@ export function OcrRowEditor({
</div>
) : null}
{/* The window as the report printed it. Editable, because the human is
reviewing a transcription: the two bounds above are the derived
reading of this string, and correcting the string is how a range the
parser could not read ("bis 5,0", "negativ") still reaches the row
instead of being dropped. */}
{!isQualitative ? (
<div className="space-y-1">
<Label htmlFor={`${fieldId}-refText`} className="text-xs">
{t("labs.ocr.refTextLabel")}
</Label>
<Input
id={`${fieldId}-refText`}
value={row.referenceText ?? ""}
placeholder={t("labs.ocr.refTextPlaceholder")}
maxLength={120}
onChange={(e) => {
const raw = e.target.value;
onChange({
...row,
referenceText: raw.trim() === "" ? null : raw,
});
}}
/>
</div>
) : null}
{valueUnreadable ? (
<p className="text-muted-foreground flex items-center gap-1.5 text-xs">
<AlertCircle aria-hidden className="h-3.5 w-3.5" />
+69
View File
@@ -0,0 +1,69 @@
"use client";
import { formatLabValue } from "@/lib/labs/format-value";
import { formatReferenceRange } from "@/lib/labs/reference-range";
import { useTranslations } from "@/lib/i18n/context";
import type { LabResultDto } from "./types";
/**
* The reference window a reading's own report printed, shown wherever that
* reading is shown.
*
* Renders nothing for a reading with no printed window — most of them — so it
* costs no space on the common row. When the report DID state a window it
* names it, verbatim where the report gave words; and when that window and the
* catalog band disagree it names the catalog band too. A reader told only
* "3,9–5,4" and a reader told only "3,5–5,0" draw different conclusions from
* the same number, so a surface that shows one silently is showing a partial
* answer.
*
* Neutral by construction: no tint, no icon, muted text. The in/out verdict is
* the badge's job — this element states where the window came from, nothing
* about whether the value is good news.
*/
export function SourceRangeNote({
reading,
className,
}: {
reading: Pick<
LabResultDto,
| "unit"
| "referenceLow"
| "referenceHigh"
| "catalogReferenceLow"
| "catalogReferenceHigh"
| "sourceReferenceText"
| "referenceOrigin"
| "referenceDivergesFromCatalog"
>;
className?: string;
}) {
const { t } = useTranslations();
if (reading.referenceOrigin !== "source") return null;
const printed =
reading.sourceReferenceText ??
`${formatReferenceRange(
reading.referenceLow,
reading.referenceHigh,
formatLabValue,
)} ${reading.unit}`.trim();
const catalog = formatReferenceRange(
reading.catalogReferenceLow,
reading.catalogReferenceHigh,
formatLabValue,
);
return (
<span className={className}>
{t("labs.sourceRange.fromReport", { range: printed })}
{reading.referenceDivergesFromCatalog && catalog
? ` · ${t("labs.sourceRange.catalogDiffers", {
range: `${catalog} ${reading.unit}`.trim(),
})}`
: null}
</span>
);
}
+21 -1
View File
@@ -1,4 +1,7 @@
import type { ReferenceRangeStatus } from "@/lib/validations/labs";
import type {
ReferenceRangeOrigin,
ReferenceRangeStatus,
} from "@/lib/validations/labs";
/** A lab-result row as the list / create endpoints serialise it. */
export interface LabResultDto {
@@ -12,8 +15,25 @@ export interface LabResultDto {
/** v1.18.9 — qualitative result text ("negativ" / …); null for a numeric row. */
valueText: string | null;
unit: string;
/**
* The window this reading is judged against and the one `rangeStatus` was
* computed from: the range its own report printed when it carries one, the
* catalog band otherwise. Render these and the range shown always matches
* the verdict shown.
*/
referenceLow: number | null;
referenceHigh: number | null;
/** The catalog marker's band, so both windows can be drawn. */
catalogReferenceLow: number | null;
catalogReferenceHigh: number | null;
/** The window the source report printed; the text verbatim, bounds when derivable. */
sourceReferenceLow: number | null;
sourceReferenceHigh: number | null;
sourceReferenceText: string | null;
/** Which window is in force. */
referenceOrigin: ReferenceRangeOrigin;
/** True when the report's window and the catalog band state different limits. */
referenceDivergesFromCatalog: boolean;
takenAt: string;
source: string;
hasNote: boolean;
+1
View File
@@ -33,6 +33,7 @@ export interface OcrCommitRowInput {
unit?: string;
referenceLow?: number;
referenceHigh?: number;
referenceText?: string;
takenAt: string;
}
+11 -10
View File
@@ -37,7 +37,7 @@
* is always owner-scoped and safe to read), so the block is `null` only when the
* account has no recent readings.
*/
import { classifyReferenceRange } from "@/lib/labs/reference-range";
import { classifyAgainstEffectiveRange } from "@/lib/labs/reference-range";
import { resolveLabFields } from "@/lib/labs/serialise";
import { sanitizeForPrompt } from "@/lib/insights/sanitize";
import { prisma } from "@/lib/db";
@@ -131,6 +131,9 @@ export async function buildLabsSnapshotBlock(
unit: true,
referenceLow: true,
referenceHigh: true,
sourceReferenceLow: true,
sourceReferenceHigh: true,
sourceReferenceText: true,
takenAt: true,
biomarkerId: true,
biomarker: {
@@ -181,15 +184,13 @@ export async function buildLabsSnapshotBlock(
referenceLow: resolved.referenceLow,
referenceHigh: resolved.referenceHigh,
// A qualitative row has nothing to compare against the bounds → neutral
// "unknown", never a fabricated verdict. A numeric row classifies.
rangeStatus:
row.value === null
? "unknown"
: classifyReferenceRange(
row.value,
resolved.referenceLow,
resolved.referenceHigh,
),
// "unknown", never a fabricated verdict. A numeric row classifies against
// the window in force for it — the report's own range when it carries
// one, the catalog band otherwise. Same rule as every other lab surface.
rangeStatus: classifyAgainstEffectiveRange(
row.value,
resolved.effectiveRange,
),
takenAt: row.takenAt.toISOString(),
});
@@ -20,10 +20,18 @@ export function buildClinicalRecordsNotesSection(
// v1.17.1 — structured lab-results section. Populated when the `labs`
// toggle is ON (default) and the user recorded at least one result in the
// window. One row per analyte (latest reading), with the lab's reference
// range and a NEUTRAL in/out-of-range marker — informative, never an
// alarming red. The marker is a quiet glyph (↓ / ↑ / —), not a colour, so
// the clinical PDF stays calm.
// window. One row per analyte (latest reading), with the reference range and
// a NEUTRAL in/out-of-range marker — informative, never an alarming red. The
// marker is a quiet glyph (↓ / ↑ / —), not a colour, so the clinical PDF
// stays calm.
//
// The reference column prints the window the reading was JUDGED against, and
// when that window came off the source report it prints the report's own
// string verbatim — a clinician comparing this table against the original
// report should read the same characters. A reading whose report window
// differs from the catalog band carries a footnote marker, and the catalog
// band is named beneath the table, because "3,9–5,4" and "3,5–5,0" disagree
// about the same number and a table that shows only one is a partial answer.
if (data.labResults && data.labResults.length > 0) {
y = ensureSpace(y, 6 + 18);
doc.setFontSize(14);
@@ -54,6 +62,23 @@ export function buildClinicalRecordsNotesSection(
}
};
// The reference cell: the report's own string when the report stated the
// window, else the formatted catalog band. A divergence gets a "*" the
// note under the table explains.
const referenceCell = (
lr: NonNullable<typeof data.labResults>[number],
): string => {
const printed =
lr.referenceOrigin === "source" && lr.sourceReferenceText
? lr.sourceReferenceText
: rangeText(lr.referenceLow, lr.referenceHigh);
return lr.referenceDivergesFromCatalog ? `${printed} *` : printed;
};
const diverging = data.labResults.filter(
(lr) => lr.referenceDivergesFromCatalog,
);
const labRows = data.labResults.map((lr) => {
// v1.18.9 — a qualitative reading (`value === null`) prints its result
// text in the value column and has no numeric range / status glyph.
@@ -63,7 +88,7 @@ export function buildClinicalRecordsNotesSection(
isQualitative
? (lr.valueText ?? "")
: `${num(lr.value as number)} ${lr.unit}`.trim(),
isQualitative ? "—" : rangeText(lr.referenceLow, lr.referenceHigh),
isQualitative ? "—" : referenceCell(lr),
isQualitative
? ""
: statusGlyph(lr.value as number, lr.referenceLow, lr.referenceHigh),
@@ -107,6 +132,30 @@ export function buildClinicalRecordsNotesSection(
y =
(doc as jsPDF & { lastAutoTable: { finalY: number } }).lastAutoTable
.finalY + 8;
// Name every diverging marker's catalog band, so the reader can see both
// windows rather than being told only that they differ.
if (diverging.length > 0) {
y = ensureSpace(y, 4 + diverging.length * 4);
doc.setFontSize(8);
doc.setFont("helvetica", "normal");
doc.setTextColor(110, 110, 110);
doc.text(t("doctorReport.labsSourceRangeNote"), margin, y);
y += 4;
for (const lr of diverging) {
doc.text(
`* ${lr.analyte}: ${t("doctorReport.labsCatalogRangeLabel")} ${rangeText(
lr.catalogReferenceLow,
lr.catalogReferenceHigh,
)} ${lr.unit}`.trim(),
margin,
y,
);
y += 4;
}
doc.setTextColor(30, 30, 30);
y += 4;
}
}
// v1.18.1 P4 — illness / condition episodes overlapping the window. Present
+15
View File
@@ -241,8 +241,23 @@ export interface DoctorReportData {
*/
valueText: string | null;
unit: string;
/**
* The reference window this reading is judged against: the range printed
* on its own report when it carries one, the catalog band otherwise.
* Resolved server-side through `resolveEffectiveReferenceRange` so the PDF
* marker, the FHIR `referenceRange`, and the API verdict agree.
*/
referenceLow: number | null;
referenceHigh: number | null;
/** The catalog band, kept so the PDF can name a divergence. */
catalogReferenceLow: number | null;
catalogReferenceHigh: number | null;
/** The window the source report printed, verbatim; null when it printed none. */
sourceReferenceText: string | null;
/** Which of the two windows `referenceLow` / `referenceHigh` came from. */
referenceOrigin: "source" | "catalog" | "none";
/** True when the report's window and the catalog band state different limits. */
referenceDivergesFromCatalog: boolean;
/** ISO timestamp of the latest reading. */
takenAt: string;
/** Count of readings for this analyte in the window (≥ 1). */
+20 -2
View File
@@ -11,6 +11,7 @@
* Split out of the aggregator with the selection rework.
*/
import { prisma } from "@/lib/db";
import { resolveEffectiveReferenceRange } from "@/lib/labs/reference-range";
import { getEvent } from "@/lib/logging/context";
import { decryptAllergyReaction } from "@/lib/doctor-report-helpers";
import type { DoctorReportData } from "@/lib/doctor-report-types";
@@ -42,6 +43,9 @@ export async function loadLabResults(
unit: true,
referenceLow: true,
referenceHigh: true,
sourceReferenceLow: true,
sourceReferenceHigh: true,
sourceReferenceText: true,
takenAt: true,
},
});
@@ -52,14 +56,28 @@ export async function loadLabResults(
for (const r of rows) {
const key = r.analyte.toLowerCase();
const prev = byAnalyte.get(key);
// The row already stamps the catalog band it was written under; the
// reading's own report window, when present, is what the physician read
// the value against, so it wins here exactly as it does on every other
// lab surface.
const range = resolveEffectiveReferenceRange(
r.referenceLow,
r.referenceHigh,
r,
);
byAnalyte.set(key, {
panel: r.panel,
analyte: r.analyte,
value: r.value,
valueText: r.valueText,
unit: r.unit,
referenceLow: r.referenceLow,
referenceHigh: r.referenceHigh,
referenceLow: range.low,
referenceHigh: range.high,
catalogReferenceLow: range.catalogLow,
catalogReferenceHigh: range.catalogHigh,
sourceReferenceText: range.sourceText,
referenceOrigin: range.origin,
referenceDivergesFromCatalog: range.divergesFromCatalog,
takenAt: r.takenAt.toISOString(),
count: (prev?.count ?? 0) + 1,
});
+24 -2
View File
@@ -15,6 +15,12 @@
* as-needed medication with notifications off — a record of what the patient
* takes, never a prescription action. An Observation never carries a
* range-flag; the reference bounds ride along only as the document stated them.
*
* The window the report printed beside a value is carried onto the reading, not
* dropped. Until now it only seeded a freshly minted catalog marker, which meant
* that for every analyte the user already had, the range printed on the report
* was read and then thrown away. It now lands on the row's `sourceReference*`
* columns and governs that reading's verdict.
*/
import { encryptToBytes } from "@/lib/ai/coach/bytes-codec";
import { prisma } from "@/lib/db";
@@ -22,6 +28,7 @@ import { invalidateUserHealthScore } from "@/lib/cache/invalidate";
import { emitDataArrival } from "@/lib/arrivals/emit-shared";
import { decryptFactData } from "@/lib/documents/store";
import { resolveOrMintBiomarker } from "@/lib/labs/biomarker-store";
import { parseReferenceRange } from "@/lib/labs/parse-reference-range";
import type { ExtractedFact } from "@/generated/prisma/client";
import type {
ConditionFactData,
@@ -66,11 +73,21 @@ async function commitObservation(
"A numeric value needs a unit before it can be saved",
);
}
// The printed window, read once. `referenceText` is what the report wrote;
// the parser derives bounds from it where it can, and where it cannot the
// string still survives onto the row. A document staged before the field
// existed has no text, and the numeric bounds the model reported are then
// the whole of what it stated.
const printed = parseReferenceRange(data.referenceText, data.unit);
const sourceLow = printed?.low ?? data.referenceLow;
const sourceHigh = printed?.high ?? data.referenceHigh;
const sourceText = printed?.text ?? null;
const biomarker = await resolveOrMintBiomarker(userId, {
analyte: data.label,
unit: isQualitative ? (data.unit ?? "") : (data.unit as string),
referenceLow: isQualitative ? null : data.referenceLow,
referenceHigh: isQualitative ? null : data.referenceHigh,
referenceLow: isQualitative ? null : sourceLow,
referenceHigh: isQualitative ? null : sourceHigh,
panel: null,
});
// Fail closed on a unit mismatch against an EXISTING marker. A freshly
@@ -101,6 +118,11 @@ async function commitObservation(
unit: biomarker.unit,
referenceLow: biomarker.lowerBound,
referenceHigh: biomarker.upperBound,
// The report's own window for THIS reading. A qualitative result has no
// window to record.
sourceReferenceLow: isQualitative ? null : sourceLow,
sourceReferenceHigh: isQualitative ? null : sourceHigh,
sourceReferenceText: isQualitative ? null : sourceText,
takenAt: statedDateOrNow(data.effectiveDate),
source: "DOCUMENT",
noteEncrypted: null,
+12 -2
View File
@@ -74,7 +74,8 @@ Extract these fact types:
- code + codeSystem ("LOINC"): ONLY if a code is printed, else null.
- value: the numeric result OR null. valueText: the qualitative result text (e.g. "negative") OR null. Set EXACTLY ONE.
- unit: the printed unit, or null. NEVER convert or normalise a unit.
- referenceLow / referenceHigh: the printed reference bounds as numbers, or null. Do NOT flag the value against them.
- referenceText: the reference range EXACTLY as printed beside the value, verbatim and complete, including its operator and unit if written (for example "3,5 - 5,0", "< 5", "bis 5,0", "\u2265 3,5", "150.000 - 400.000 /\u00b5l", "negativ"). Copy the characters; do NOT reformat, convert, or normalise them. Null only when no range is printed for that value.
- referenceLow / referenceHigh: the SAME range reduced to two numbers, when it reduces to two numbers, else null. Do NOT flag the value against them.
- effectiveDate: the stated measurement date (YYYY-MM-DD), else null.
3. MEDICATION_STATEMENT — a medication the patient is/was taking, as recorded. Capture:
- label: the drug name, verbatim.
@@ -91,7 +92,7 @@ For EVERY fact also capture:
Do not emit a fact you cannot read — omit it rather than invent one. Also capture reportDate: the document's stated report/collection date (YYYY-MM-DD), or null, and kind: "DOCTOR_REPORT", "DISCHARGE_LETTER", or "OTHER".
Respond ONLY with a JSON object of this exact shape:
{ "reportDate": string|null, "kind": "DOCTOR_REPORT"|"DISCHARGE_LETTER"|"OTHER"|null, "facts": [ { "type": "CONDITION"|"OBSERVATION"|"MEDICATION_STATEMENT", "label": string, "code": string|null, "codeSystem": "SNOMED"|"ICD10"|"LOINC"|"RXNORM"|"ATC"|null, "clinicalStatus": string|null, "verificationStatus": string|null, "value": number|null, "valueText": string|null, "unit": string|null, "referenceLow": number|null, "referenceHigh": number|null, "dose": string|null, "medicationStatus": string|null, "effectiveDate": string|null, "sourceText": string, "page": number|null, "confidence": number } ] }`;
{ "reportDate": string|null, "kind": "DOCTOR_REPORT"|"DISCHARGE_LETTER"|"OTHER"|null, "facts": [ { "type": "CONDITION"|"OBSERVATION"|"MEDICATION_STATEMENT", "label": string, "code": string|null, "codeSystem": "SNOMED"|"ICD10"|"LOINC"|"RXNORM"|"ATC"|null, "clinicalStatus": string|null, "verificationStatus": string|null, "value": number|null, "valueText": string|null, "unit": string|null, "referenceText": string|null, "referenceLow": number|null, "referenceHigh": number|null, "dose": string|null, "medicationStatus": string|null, "effectiveDate": string|null, "sourceText": string, "page": number|null, "confidence": number } ] }`;
const USER_PROMPT = `Transcribe the clinical document in the attached file into the JSON schema described in the system prompt. Reproduce only what is written — do not interpret. Return only the JSON object.`;
@@ -172,6 +173,7 @@ function mapFactData(raw: ExtractedFactRaw): FactData | null {
unit: raw.unit,
referenceLow: raw.referenceLow,
referenceHigh: raw.referenceHigh,
referenceText: raw.referenceText,
effectiveDate,
};
return data;
@@ -309,6 +311,14 @@ export async function runInboundExtraction(
unanchored,
conditions: facts.filter((f) => f.factType === "CONDITION").length,
observations: facts.filter((f) => f.factType === "OBSERVATION").length,
// How many transcribed observations arrived with the window the report
// printed beside them. A collapse here is the extraction quietly going
// back to dropping the range.
observationsWithRange: facts.filter(
(f) =>
f.factType === "OBSERVATION" &&
((f.data as ObservationFactData).referenceText ?? null) !== null,
).length,
medications: facts.filter((f) => f.factType === "MEDICATION_STATEMENT")
.length,
needsReview: facts.filter((f) => f.needsReview).length,
@@ -38,6 +38,8 @@ import { restoreProfileData } from "../profile-backup";
const deletedAt = new Date("2026-07-19T12:00:00.000Z");
const measurementNote = encryptToBytes("canonical measurement note");
const sideEffectNote = encryptToBytes("nausea after the evening dose");
const moodNotePlaintext = "quiet evening, slept badly";
const moodNote = encryptToBytes(moodNotePlaintext);
const appSettings = {
id: "singleton",
@@ -193,11 +195,27 @@ function makePrisma() {
mood: "OKAY",
score: 3,
tags: null,
note: null,
noteEncrypted: moodNote,
source: "MOODLOG",
externalId: "moodlog-42",
moodLoggedAt: new Date("2026-07-19T08:00:00.000Z"),
tz: "America/New_York",
syncedAt: new Date("2026-07-19T08:00:01.000Z"),
syncVersion: 3,
deletedAt,
tagLinks: [{ rating: 4, moodTag: { key: "sleep_quality" } }],
createdAt: new Date("2026-07-19T08:00:00.000Z"),
updatedAt: new Date("2026-07-19T09:00:00.000Z"),
tagLinks: [
{
rating: 4,
moodTag: { key: "sleep_quality", kind: "RATED" },
},
{
rating: null,
moodTag: { key: "headache", kind: "BINARY" },
},
],
},
]),
},
@@ -409,7 +427,13 @@ describe("buildFullBackupPayload disaster-recovery mode", () => {
expect(prisma.moodEntry.findMany).toHaveBeenCalledWith(
expect.objectContaining({
where: { userId: "user-1" },
include: expect.any(Object),
select: expect.objectContaining({
id: true,
noteEncrypted: true,
tz: true,
syncVersion: true,
deletedAt: true,
}),
}),
);
expect(mocks.buildRecordsBackupSection).toHaveBeenCalledWith(
@@ -485,7 +509,16 @@ describe("buildFullBackupPayload disaster-recovery mode", () => {
id: "mood-tombstone",
externalId: "moodlog-42",
deletedAt: deletedAt.toISOString(),
tz: "America/New_York",
syncedAt: "2026-07-19T08:00:01.000Z",
syncVersion: 3,
createdAt: "2026-07-19T08:00:00.000Z",
updatedAt: "2026-07-19T09:00:00.000Z",
note: null,
noteEncrypted: Buffer.from(moodNote).toString("base64"),
factors: [{ key: "sleep_quality", rating: 4 }],
// The BINARY link the old `rating: { not: null }` filter excluded.
structuredTags: ["headache"],
},
],
nutrientDays: [
@@ -1148,3 +1181,79 @@ describe("buildFullBackupPayload — profile and custom metrics", () => {
expect(counts.intradayProfiles).toBe(1);
});
});
describe("buildFullBackupPayload — mood arms", () => {
it("carries the ciphertext and the whole row on the disaster-recovery arm", async () => {
installSectionMocks();
const prisma = makePrisma();
const { payload } = await buildFullBackupPayload(
prisma as never,
"user-1",
{
purpose: "disaster-recovery",
exportedAt: new Date("2026-07-20T00:00:00.000Z"),
},
);
const [entry] = (payload as { moodEntries: Array<Record<string, unknown>> })
.moodEntries;
expect(entry).toEqual(
expect.objectContaining({
id: "mood-tombstone",
note: null,
noteEncrypted: Buffer.from(moodNote).toString("base64"),
tz: "America/New_York",
syncedAt: "2026-07-19T08:00:01.000Z",
syncVersion: 3,
createdAt: "2026-07-19T08:00:00.000Z",
updatedAt: "2026-07-19T09:00:00.000Z",
deletedAt: deletedAt.toISOString(),
factors: [{ key: "sleep_quality", rating: 4 }],
structuredTags: ["headache"],
}),
);
});
it("carries the readable note and no ciphertext on the portable arm", async () => {
installSectionMocks();
const prisma = makePrisma();
const { payload } = await buildFullBackupPayload(
prisma as never,
"user-1",
{
purpose: "portable-export",
exportedAt: new Date("2026-07-20T00:00:00.000Z"),
},
);
const [entry] = (payload as { moodEntries: Array<Record<string, unknown>> })
.moodEntries;
// Same split the measurement arm makes: the readable artefact carries the
// decrypted text and nothing an operator could not read, and the counters
// that only mean something to this instance stay behind.
expect(entry).toEqual(
expect.objectContaining({
note: moodNotePlaintext,
tz: "America/New_York",
structuredTags: ["headache"],
}),
);
expect(entry).not.toHaveProperty("noteEncrypted");
expect(entry).not.toHaveProperty("syncVersion");
expect(entry).not.toHaveProperty("syncedAt");
expect(JSON.stringify(entry)).not.toContain(
Buffer.from(moodNote).toString("base64"),
);
// The tag-link read is unfiltered on both arms; a BINARY link is the one
// most people have and it was the one the file never carried.
expect(prisma.moodEntry.findMany).toHaveBeenCalledWith(
expect.objectContaining({
select: expect.objectContaining({
tagLinks: { select: expect.any(Object) },
}),
}),
);
});
});
+107 -23
View File
@@ -103,6 +103,46 @@ const MEASUREMENT_BACKUP_SELECT = {
updatedAt: true,
} as const satisfies Prisma.MeasurementSelect;
/**
* Every restorable `MoodEntry` scalar. `userId` is absent for the same reason
* it is absent above: the backup is user-scoped and the owner is re-derived
* from the restoring account.
*
* Named rather than inlined so `mood-backup-completeness.test.ts` can compare
* it field-by-field against the model in `prisma/schema.prisma`, exactly as the
* measurement guard does for the select above.
*/
const MOOD_ENTRY_BACKUP_SELECT = {
id: true,
date: true,
mood: true,
score: true,
tags: true,
note: true,
noteEncrypted: true,
source: true,
externalId: true,
moodLoggedAt: true,
tz: true,
syncedAt: true,
syncVersion: true,
deletedAt: true,
createdAt: true,
updatedAt: true,
} as const satisfies Prisma.MoodEntrySelect;
/**
* A structured-tag link, as it rides the wire. The link is carried by the
* tag's KEY, not by its id: a restore resolves the key against the target
* instance's catalogue, where the source instance's row id may not exist at
* all. `kind` comes along so the mapper can split RATED factors from BINARY
* tags without a second query.
*/
const MOOD_ENTRY_TAG_LINK_SELECT = {
rating: true,
moodTag: { select: { key: true, kind: true } },
} as const satisfies Prisma.MoodEntryTagLinkSelect;
/**
* Build the canonical full-backup payload for `userId`. Portable exports omit
* tombstones and document ciphertext; disaster-recovery payloads preserve
@@ -207,14 +247,13 @@ export async function buildFullBackupPayload(
prisma.moodEntry.findMany({
where: disasterRecovery ? { userId } : { userId, deletedAt: null },
orderBy: { moodLoggedAt: "desc" },
include: {
tagLinks: {
where: { rating: { not: null } },
select: {
rating: true,
moodTag: { select: { key: true } },
},
},
select: {
...MOOD_ENTRY_BACKUP_SELECT,
// Every link, not only the rated ones. This read used to filter
// `rating: { not: null }`, and a BINARY link carries a NULL rating by
// definition — so the present/absent tags, the kind most people pick,
// were absent from the file and could not come back from it.
tagLinks: { select: MOOD_ENTRY_TAG_LINK_SELECT },
},
}),
// The account's OWN mood tags. The seeded catalogue is reference data every
@@ -394,21 +433,66 @@ export async function buildFullBackupPayload(
skipped: e.skipped,
source: e.source,
})),
moodEntries: moodEntries.map((e) => ({
id: e.id,
date: e.date,
mood: e.mood,
score: e.score,
tags: e.tags,
source: e.source,
loggedAt: e.moodLoggedAt.toISOString(),
externalId: e.externalId,
deletedAt: e.deletedAt?.toISOString() ?? null,
factors: e.tagLinks.map((link) => ({
key: link.moodTag.key,
rating: link.rating!,
})),
})),
// The parameter is spelled out rather than the `e` every other mapper here
// uses, and deliberately: `mood-backup-completeness.test.ts` proves a
// selected column reaches a payload by matching `: moodEntry.`. A matcher
// on `: e.` would also match the intake-event mapper above it and could
// therefore never fail.
moodEntries: moodEntries.map((moodEntry) => {
// The two link arms partition the entry's links exhaustively: a link is
// a rated factor when its tag is RATED and it actually carries a score,
// and a structured tag otherwise. Total on purpose — a malformed link
// (a RATED tag whose rating went missing) rides as a tag key and comes
// back NAMED as unresolvable on restore, rather than falling between the
// two arms and disappearing the way every BINARY link used to.
const isRatedFactor = (link: (typeof moodEntry.tagLinks)[number]) =>
link.moodTag.kind === "RATED" && link.rating !== null;
const factors = moodEntry.tagLinks
.filter(isRatedFactor)
.map((tagLink) => ({
key: tagLink.moodTag.key,
rating: tagLink.rating as number,
}));
// BINARY link keys. Their own array rather than a nullable rating inside
// `factors`: the restore resolves `factors` against `kind: "RATED"` and
// reports what it cannot resolve, so a rating-less entry in that array
// would collide with both. A separate key leaves every backup file
// written before this parsing and restoring exactly as it did.
const structuredTags = moodEntry.tagLinks
.filter((link) => !isRatedFactor(link))
.map((tagLink) => tagLink.moodTag.key);
return {
...(disasterRecovery
? {
note: moodEntry.note,
noteEncrypted: moodEntry.noteEncrypted
? Buffer.from(moodEntry.noteEncrypted).toString("base64")
: null,
syncedAt: moodEntry.syncedAt.toISOString(),
syncVersion: moodEntry.syncVersion,
createdAt: moodEntry.createdAt.toISOString(),
updatedAt: moodEntry.updatedAt.toISOString(),
}
: { note: readNote(moodEntry.noteEncrypted, moodEntry.note) }),
id: moodEntry.id,
date: moodEntry.date,
mood: moodEntry.mood,
score: moodEntry.score,
tags: moodEntry.tags,
source: moodEntry.source,
loggedAt: moodEntry.moodLoggedAt.toISOString(),
externalId: moodEntry.externalId,
// The zone the `date` string is anchored to. Without it a restored row
// falls back to the legacy Europe/Berlin reading and an account that
// logged elsewhere gets its day boundaries quietly moved.
tz: moodEntry.tz,
deletedAt: moodEntry.deletedAt?.toISOString() ?? null,
factors,
structuredTags,
};
}),
customMoodTags: customMoodTags.map((tag) => ({
...(disasterRecovery ? { id: tag.id } : {}),
key: tag.key,
+7
View File
@@ -54,6 +54,10 @@ export interface LabResultBackupEntry {
unit: string;
referenceLow: number | null;
referenceHigh: number | null;
/** The window the reading's own source report printed, verbatim + derived. */
sourceReferenceLow: number | null;
sourceReferenceHigh: number | null;
sourceReferenceText: string | null;
takenAt: string;
source: string;
/** Human-readable cross-reference into `biomarkers` below, not an id. */
@@ -336,6 +340,9 @@ export async function buildRecordsBackupSection(
unit: r.unit,
referenceLow: r.referenceLow,
referenceHigh: r.referenceHigh,
sourceReferenceLow: r.sourceReferenceLow,
sourceReferenceHigh: r.sourceReferenceHigh,
sourceReferenceText: r.sourceReferenceText,
takenAt: r.takenAt.toISOString(),
source: r.source,
biomarkerName: r.biomarker?.name ?? null,
+5 -3
View File
@@ -1,8 +1,9 @@
/**
* What a restore could not put back, and how much of it there was.
*
* Three of the restore's lookups resolve a key against a SEEDED CATALOGUE —
* cycle symptoms, illness symptoms, mood factors. The catalogue is reference
* Four of the restore's lookups resolve a key against a SEEDED CATALOGUE —
* cycle symptoms, illness symptoms, rated mood factors and the mood tags a
* person ticks present or absent. The catalogue is reference
* data the instance owns, not data the file carries, so it legitimately drifts
* between the day a backup is written and the day it is read: a key gets
* renamed, a symptom is retired, the file is a year old, the instance is three
@@ -28,7 +29,8 @@
*/
/** Which seeded catalogue a key failed to resolve against. */
export type SkippedCatalogue = "cycleSymptom" | "illnessSymptom" | "moodFactor";
export type SkippedCatalogue =
"cycleSymptom" | "illnessSymptom" | "moodFactor" | "moodTag";
/** One key this instance does not know, and the links it cost. */
export interface SkippedCatalogueKey {
@@ -54,6 +54,11 @@ describe("lab-result FHIR Observations", () => {
unit: "mg/dL",
referenceLow: null,
referenceHigh: 116,
catalogReferenceLow: null,
catalogReferenceHigh: null,
sourceReferenceText: null,
referenceOrigin: "catalog" as const,
referenceDivergesFromCatalog: false,
takenAt: "2026-04-20T08:00:00.000Z",
count: 2,
},
@@ -65,6 +70,11 @@ describe("lab-result FHIR Observations", () => {
unit: "%",
referenceLow: 4,
referenceHigh: 5.6,
catalogReferenceLow: null,
catalogReferenceHigh: null,
sourceReferenceText: null,
referenceOrigin: "catalog" as const,
referenceDivergesFromCatalog: false,
takenAt: "2026-04-21T08:00:00.000Z",
count: 1,
},
@@ -137,6 +147,11 @@ describe("lab-result FHIR Observations", () => {
unit: "mg/L",
referenceLow: null,
referenceHigh: null,
catalogReferenceLow: null,
catalogReferenceHigh: null,
sourceReferenceText: null,
referenceOrigin: "catalog" as const,
referenceDivergesFromCatalog: false,
takenAt: "2026-04-22T08:00:00.000Z",
count: 1,
},
@@ -178,6 +193,11 @@ describe("lab-result LOINC + canonical UCUM coding (v1.18.8)", () => {
unit,
referenceLow: null,
referenceHigh: null,
catalogReferenceLow: null,
catalogReferenceHigh: null,
sourceReferenceText: null,
referenceOrigin: "catalog" as const,
referenceDivergesFromCatalog: false,
takenAt: "2026-04-20T08:00:00.000Z",
count: 1,
},
@@ -264,6 +284,11 @@ describe("qualitative lab-result FHIR Observations (v1.18.9)", () => {
unit: "",
referenceLow: null,
referenceHigh: null,
catalogReferenceLow: null,
catalogReferenceHigh: null,
sourceReferenceText: null,
referenceOrigin: "catalog" as const,
referenceDivergesFromCatalog: false,
takenAt: "2026-04-20T08:00:00.000Z",
count: 1,
},
+14 -3
View File
@@ -7,8 +7,15 @@
* `code.text` for the common biomarkers, and stamps the canonical UCUM `code`
* on the value WHEN the recorded unit matches the mapped canonical symbol. An
* analyte that does NOT resolve keeps the honest text-only `code` +
* display-only `unit` (no fabricated terminology). The lab's reference bounds
* map onto R4 `referenceRange`.
* display-only `unit` (no fabricated terminology).
*
* The reference bounds map onto R4 `referenceRange`, and they are the SAME
* bounds the app's own verdict uses: the window the reading's source report
* printed when it carries one, the catalog band otherwise, already resolved
* upstream in `loadLabResults`. When the source report stated its window in
* words the bounds could not be read from ("negativ", "siehe Befund"), that
* string rides on `referenceRange.text` so a receiver still gets what the lab
* wrote rather than a silently empty element.
*/
import type { DoctorReportData } from "@/lib/doctor-report-data";
import { LOINC_SYSTEM, UCUM_SYSTEM } from "@/lib/fhir/loinc-map";
@@ -82,8 +89,11 @@ export function labObservations(
// The bounds are stated in the SAME unit as the value, so they carry the
// SAME coding — a bare unit beside a coded value leaves a receiver unable
// to compare the two.
const sourceText =
lab.referenceOrigin === "source" ? lab.sourceReferenceText : null;
const hasBounds = lab.referenceLow !== null || lab.referenceHigh !== null;
const referenceRange: FhirObservationReferenceRange[] | undefined =
lab.referenceLow !== null || lab.referenceHigh !== null
hasBounds || sourceText
? [
{
...(lab.referenceLow !== null
@@ -92,6 +102,7 @@ export function labObservations(
...(lab.referenceHigh !== null
? { high: quantity(lab.referenceHigh) }
: {}),
...(sourceText ? { text: sourceText } : {}),
},
]
: undefined;
+7
View File
@@ -117,6 +117,13 @@ export interface FhirObservation {
export interface FhirObservationReferenceRange {
low?: FhirQuantity;
high?: FhirQuantity;
/**
* R4 `referenceRange.text`. Carries the window the source report printed,
* verbatim, when the reading has one. A receiver that cannot use the bounds
* (because the report stated a qualitative window like "negativ", or a
* one-sided one) still reads what the lab wrote.
*/
text?: string;
}
export interface FhirDosage {
@@ -0,0 +1,179 @@
/**
* `upsertFitbitMeasurements` is the single write seam for every Fitbit
* resource leg — activity, sleep, daily metrics — and it holds Prisma
* directly, so it does not inherit the shared reconciler's gate. These cases
* pin that the provider's numbers clear the app's own declared plausibility
* band before any of them can become a row, and that a refusal is reported.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
const { findManyMock, createManyMock, updateMock } = vi.hoisted(() => ({
findManyMock: vi.fn(async () => [] as unknown[]),
createManyMock: vi.fn<
() => Promise<Array<{ id: string; type: string; measuredAt: Date }>>
>(async () => []),
updateMock: vi.fn(async () => ({})),
}));
vi.mock("@/lib/db", () => ({
prisma: {
measurement: {
findMany: findManyMock,
createManyAndReturn: createManyMock,
update: updateMock,
},
},
}));
vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() }));
vi.mock("@/lib/integrations/status", () => ({
isReauthRequired: vi.fn(async () => false),
recordSyncFailure: vi.fn(async () => {}),
recordSyncSuccess: vi.fn(async () => {}),
}));
vi.mock("@/lib/rollups/measurement-rollups", () => ({
collapseToTypeDayKeys: vi.fn(() => []),
recomputeBucketsForMeasurement: vi.fn(async () => {}),
recomputeUserRollups: vi.fn(async () => {}),
}));
vi.mock("@/lib/insights/comprehensive-generate", () => ({
invalidateStatusInsightsForTypes: vi.fn(async () => {}),
}));
vi.mock("@/lib/integrations/oauth-refresh", () => ({
persistRotatedToken: vi.fn(async () => {}),
}));
vi.mock("../credentials", () => ({
getUserFitbitCredentials: vi.fn(async () => null),
}));
vi.mock("../client", () => ({ refreshAccessToken: vi.fn() }));
import { eventStorage } from "@/lib/logging/context";
import { WideEventBuilder } from "@/lib/logging/event-builder";
import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate";
import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate";
import { upsertFitbitMeasurements } from "../sync-core";
/** RESTING_HEART_RATE is declared 25–220. */
const REAL_READING = {
type: "RESTING_HEART_RATE",
value: 58,
unit: "bpm",
measuredAt: new Date("2026-07-08T00:00:00.000Z"),
externalId: "rhr:2026-07-08",
};
const IMPOSSIBLE_READING = {
type: "RESTING_HEART_RATE",
value: 4_915_200,
unit: "bpm",
measuredAt: new Date("2026-07-09T00:00:00.000Z"),
externalId: "rhr:2026-07-09",
};
let event: WideEventBuilder;
function tally(): RangeRejectionTally | undefined {
return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as
RangeRejectionTally | undefined;
}
beforeEach(() => {
findManyMock.mockReset().mockResolvedValue([]);
createManyMock.mockReset().mockResolvedValue([]);
updateMock.mockReset().mockResolvedValue({});
event = new WideEventBuilder("background");
});
describe("upsertFitbitMeasurements — plausibility gate", () => {
it("writes a reading inside the metric's declared band", async () => {
createManyMock.mockResolvedValue([
{
id: "row-1",
type: "RESTING_HEART_RATE",
measuredAt: REAL_READING.measuredAt,
},
]);
const result = await eventStorage.run(event, () =>
upsertFitbitMeasurements("user-1", [REAL_READING], { deferRollup: true }),
);
expect(result.imported).toBe(1);
const planned = (createManyMock.mock.calls[0] as unknown[])[0] as {
data: Array<{ value: number }>;
};
expect(planned.data).toHaveLength(1);
expect(planned.data[0]!.value).toBe(58);
expect(tally()).toBeUndefined();
});
it("refuses an out-of-band reading and annotates the drop", async () => {
createManyMock.mockResolvedValue([
{
id: "row-1",
type: "RESTING_HEART_RATE",
measuredAt: REAL_READING.measuredAt,
},
]);
const result = await eventStorage.run(event, () =>
upsertFitbitMeasurements("user-1", [REAL_READING, IMPOSSIBLE_READING], {
deferRollup: true,
}),
);
expect(result.imported).toBe(1);
const planned = (createManyMock.mock.calls[0] as unknown[])[0] as {
data: Array<{ value: number; externalId: string }>;
};
expect(planned.data).toHaveLength(1);
expect(planned.data[0]!.externalId).toBe("rhr:2026-07-08");
// The refused reading never even reached the existence probe.
expect(
(
(findManyMock.mock.calls[0] as unknown[])[0] as {
where: { externalId: { in: string[] } };
}
).where.externalId.in,
).toEqual(["rhr:2026-07-08"]);
expect(tally()).toEqual({
total: 1,
buckets: [
{
source: "fitbit",
type: "RESTING_HEART_RATE",
direction: "above_max",
count: 1,
},
],
});
expect(event.getLevel()).toBe("warn");
});
it("does not overwrite a stored reading with an out-of-band one", async () => {
// The key already exists; ungated, the update branch rewrote its value.
findManyMock.mockResolvedValue([
{
id: "existing",
type: "RESTING_HEART_RATE",
externalId: "rhr:2026-07-09",
value: 58,
unit: "bpm",
measuredAt: IMPOSSIBLE_READING.measuredAt,
sleepStage: null,
deletedAt: null,
},
]);
const result = await eventStorage.run(event, () =>
upsertFitbitMeasurements("user-1", [IMPOSSIBLE_READING], {
deferRollup: true,
}),
);
expect(result.imported).toBe(0);
expect(updateMock).not.toHaveBeenCalled();
expect(createManyMock).not.toHaveBeenCalled();
expect(tally()?.total).toBe(1);
});
});
+11 -1
View File
@@ -15,6 +15,7 @@ import { persistRotatedToken } from "@/lib/integrations/oauth-refresh";
import { recordSyncFailure } from "@/lib/integrations/status";
import type { FailureKind } from "@/lib/integrations/status";
import { getEvent } from "@/lib/logging/context";
import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate";
import {
collapseToTypeDayKeys,
recomputeBucketsForMeasurement,
@@ -530,7 +531,7 @@ export async function replaceStaleFitbitSleep(
*/
export async function upsertFitbitMeasurements(
userId: string,
readings: FitbitMeasurementUpsert[],
incoming: FitbitMeasurementUpsert[],
opts: { deferRollup?: boolean } = {},
): Promise<{
imported: number;
@@ -541,6 +542,15 @@ export async function upsertFitbitMeasurements(
measuredAt: Date;
}>;
}> {
// Every resource leg funnels through here, so this is the one place the
// provider's numbers have to clear the app's declared plausibility band
// before they can become rows. A refused reading is dropped and tallied on
// the ambient event, never clamped and never written for a later reader to
// discover.
const readings = dropImplausibleMeasurements("fitbit", incoming, (r) => ({
type: r.type,
value: r.value,
}));
if (readings.length === 0) return { imported: 0, touched: [], inserted: [] };
// Probe EVERY existing row (live AND tombstoned) for the batch's externalIds
@@ -0,0 +1,180 @@
/**
* `upsertGoogleHealthMeasurements` is the single write seam for every Google
* Health resource leg and it holds Prisma directly, so it does not inherit the
* shared reconciler's gate. These cases pin that the provider's numbers clear
* the app's own declared plausibility band before any of them can become a
* row, and that a refusal is reported rather than swallowed.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
const { findManyMock, createManyMock, updateMock } = vi.hoisted(() => ({
findManyMock: vi.fn(async () => [] as unknown[]),
createManyMock: vi.fn<
() => Promise<Array<{ id: string; type: string; measuredAt: Date }>>
>(async () => []),
updateMock: vi.fn(async () => ({})),
}));
vi.mock("@/lib/db", () => ({
prisma: {
measurement: {
findMany: findManyMock,
createManyAndReturn: createManyMock,
update: updateMock,
},
},
}));
vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() }));
vi.mock("@/lib/integrations/status", () => ({
isReauthRequired: vi.fn(async () => false),
recordSyncFailure: vi.fn(async () => {}),
recordSyncSuccess: vi.fn(async () => {}),
}));
vi.mock("@/lib/rollups/measurement-rollups", () => ({
collapseToTypeDayKeys: vi.fn(() => []),
recomputeBucketsForMeasurement: vi.fn(async () => {}),
recomputeUserRollups: vi.fn(async () => {}),
}));
vi.mock("@/lib/insights/comprehensive-generate", () => ({
invalidateStatusInsightsForTypes: vi.fn(async () => {}),
}));
vi.mock("../credentials", () => ({
getUserGoogleHealthCredentials: vi.fn(async () => null),
}));
vi.mock("../client", async () => {
const actual = await vi.importActual<typeof import("../client")>("../client");
return {
...actual,
refreshAccessToken: vi.fn(),
noteGoogleHealthMapped: vi.fn(),
noteGoogleHealthWritten: vi.fn(),
noteGoogleHealthOutcomeFailure: vi.fn(),
};
});
import { eventStorage } from "@/lib/logging/context";
import { WideEventBuilder } from "@/lib/logging/event-builder";
import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate";
import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate";
import { upsertGoogleHealthMeasurements } from "../sync-core";
/** OXYGEN_SATURATION is declared 50–100. */
const REAL_READING = {
type: "OXYGEN_SATURATION",
value: 97,
unit: "%",
measuredAt: new Date("2026-07-08T00:00:00.000Z"),
externalId: "spo2:2026-07-08",
};
const IMPOSSIBLE_READING = {
type: "OXYGEN_SATURATION",
value: 9_700,
unit: "%",
measuredAt: new Date("2026-07-09T00:00:00.000Z"),
externalId: "spo2:2026-07-09",
};
let event: WideEventBuilder;
function tally(): RangeRejectionTally | undefined {
return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as
RangeRejectionTally | undefined;
}
beforeEach(() => {
findManyMock.mockReset().mockResolvedValue([]);
createManyMock.mockReset().mockResolvedValue([]);
updateMock.mockReset().mockResolvedValue({});
event = new WideEventBuilder("background");
});
describe("upsertGoogleHealthMeasurements — plausibility gate", () => {
it("writes a reading inside the metric's declared band", async () => {
createManyMock.mockResolvedValue([
{
id: "row-1",
type: "OXYGEN_SATURATION",
measuredAt: REAL_READING.measuredAt,
},
]);
const result = await eventStorage.run(event, () =>
upsertGoogleHealthMeasurements("user-1", [REAL_READING], {
deferRollup: true,
}),
);
expect(result.imported).toBe(1);
const planned = (createManyMock.mock.calls[0] as unknown[])[0] as {
data: Array<{ value: number }>;
};
expect(planned.data).toHaveLength(1);
expect(planned.data[0]!.value).toBe(97);
expect(tally()).toBeUndefined();
});
it("refuses an out-of-band reading and annotates the drop", async () => {
createManyMock.mockResolvedValue([
{
id: "row-1",
type: "OXYGEN_SATURATION",
measuredAt: REAL_READING.measuredAt,
},
]);
const result = await eventStorage.run(event, () =>
upsertGoogleHealthMeasurements(
"user-1",
[REAL_READING, IMPOSSIBLE_READING],
{ deferRollup: true },
),
);
expect(result.imported).toBe(1);
const planned = (createManyMock.mock.calls[0] as unknown[])[0] as {
data: Array<{ externalId: string }>;
};
expect(planned.data).toHaveLength(1);
expect(planned.data[0]!.externalId).toBe("spo2:2026-07-08");
expect(tally()).toEqual({
total: 1,
buckets: [
{
source: "googleHealth",
type: "OXYGEN_SATURATION",
direction: "above_max",
count: 1,
},
],
});
expect(event.getLevel()).toBe("warn");
});
it("does not overwrite a stored reading with an out-of-band one", async () => {
findManyMock.mockResolvedValue([
{
id: "existing",
type: "OXYGEN_SATURATION",
externalId: "spo2:2026-07-09",
value: 97,
unit: "%",
measuredAt: IMPOSSIBLE_READING.measuredAt,
sleepStage: null,
deletedAt: null,
},
]);
const result = await eventStorage.run(event, () =>
upsertGoogleHealthMeasurements("user-1", [IMPOSSIBLE_READING], {
deferRollup: true,
}),
);
expect(result.imported).toBe(0);
expect(updateMock).not.toHaveBeenCalled();
expect(createManyMock).not.toHaveBeenCalled();
expect(tally()?.total).toBe(1);
});
});
+13 -2
View File
@@ -20,6 +20,7 @@ import { prisma } from "@/lib/db";
import type { MeasurementType } from "@/generated/prisma/client";
import { encrypt, decrypt } from "@/lib/crypto";
import { getEvent } from "@/lib/logging/context";
import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate";
import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit";
import { recordSyncFailure, type FailureKind } from "@/lib/integrations/status";
import {
@@ -487,7 +488,7 @@ export async function replaceStaleGoogleHealthSleep(
*/
export async function upsertGoogleHealthMeasurements(
userId: string,
readings: GoogleHealthMeasurementUpsert[],
incoming: GoogleHealthMeasurementUpsert[],
opts: { deferRollup?: boolean } = {},
): Promise<{
imported: number;
@@ -498,7 +499,17 @@ export async function upsertGoogleHealthMeasurements(
measuredAt: Date;
}>;
}> {
noteGoogleHealthMapped(readings.length);
noteGoogleHealthMapped(incoming.length);
// Every resource leg funnels through here, so this is the one place the
// provider's numbers have to clear the app's declared plausibility band
// before they can become rows. A refused reading is dropped and tallied on
// the ambient event, never clamped and never written for a later reader to
// discover.
const readings = dropImplausibleMeasurements(
"googleHealth",
incoming,
(r) => ({ type: r.type, value: r.value }),
);
if (readings.length === 0) return { imported: 0, touched: [], inserted: [] };
// Probe EVERY existing row (live AND tombstoned) for the batch's externalIds
@@ -19,7 +19,14 @@ function row(
analyte: string,
value: number,
day: string,
opts: { unit?: string; low?: number | null; high?: number | null } = {},
opts: {
unit?: string;
low?: number | null;
high?: number | null;
sourceLow?: number | null;
sourceHigh?: number | null;
sourceText?: string | null;
} = {},
): LabChangeRow {
return {
analyte,
@@ -27,6 +34,9 @@ function row(
value,
referenceLow: opts.low ?? null,
referenceHigh: opts.high ?? null,
sourceReferenceLow: opts.sourceLow ?? null,
sourceReferenceHigh: opts.sourceHigh ?? null,
sourceReferenceText: opts.sourceText ?? null,
takenAt: new Date(`${day}T08:00:00.000Z`),
};
}
+27 -10
View File
@@ -29,7 +29,10 @@ import {
} from "@/lib/ai/prompts/biomarker";
import { openerArchetypeHint } from "@/lib/ai/prompts/opener-archetype";
import type { Locale } from "@/lib/i18n/config";
import { classifyReferenceRange } from "@/lib/labs/reference-range";
import {
classifyAgainstEffectiveRange,
resolveEffectiveReferenceRange,
} from "@/lib/labs/reference-range";
import { getNoKeyBiomarkerStatusText } from "@/lib/insights/no-key-fallbacks";
import { hashInsightSnapshot } from "@/lib/insights/snapshot-hash";
import { runStatusCompletion } from "@/lib/insights/status-provider";
@@ -158,7 +161,17 @@ export async function generateBiomarkerStatus(args: {
},
orderBy: { takenAt: "desc" },
take: SNAPSHOT_READING_CAP,
select: { id: true, value: true, takenAt: true },
select: {
id: true,
value: true,
takenAt: true,
// The window this reading's own report printed, when it carries one.
// The verdict below is the reading's, not the marker's, so it has to
// read the reading's window — see `resolveEffectiveReferenceRange`.
sourceReferenceLow: true,
sourceReferenceHigh: true,
sourceReferenceText: true,
},
});
if (rows.length === 0) {
annotate({
@@ -174,6 +187,14 @@ export async function generateBiomarkerStatus(args: {
takenAt: r.takenAt,
}));
const latest = readings[0];
// The window the latest reading is judged against: its own report's range
// when that reading carries one, the catalog band otherwise. One rule, the
// same one the API DTO, the doctor report and the coach snapshot apply.
const latestRange = resolveEffectiveReferenceRange(
marker.lowerBound,
marker.upperBound,
rows[0],
);
const totalCount = await prisma.labResult.count({
where: {
userId: args.userId,
@@ -226,7 +247,7 @@ export async function generateBiomarkerStatus(args: {
locale,
promptVersion: PROMPT_VERSION,
count: totalCount,
bounds: { lower: marker.lowerBound, upper: marker.upperBound },
bounds: { lower: latestRange.low, upper: latestRange.high },
latest: {
id: latest.id,
takenAt: latest.takenAt.toISOString(),
@@ -252,11 +273,7 @@ export async function generateBiomarkerStatus(args: {
// Ascending series (oldest → newest) for the trend summary the prompt reads.
const ascending = [...readings].reverse();
const summary = summarizeSeries(ascending.map((r) => ({ value: r.value })));
const latestStatus = classifyReferenceRange(
latest.value,
marker.lowerBound,
marker.upperBound,
);
const latestStatus = classifyAgainstEffectiveRange(latest.value, latestRange);
const snapshot = {
locale,
@@ -266,8 +283,8 @@ export async function generateBiomarkerStatus(args: {
name: marker.name,
unit: marker.unit,
referenceRange:
marker.lowerBound != null || marker.upperBound != null
? { lower: marker.lowerBound, upper: marker.upperBound }
latestRange.low != null || latestRange.high != null
? { lower: latestRange.low, upper: latestRange.high }
: null,
},
dataCoverage: {
@@ -590,6 +590,9 @@ export async function fetchLabDraws(
unit: true,
referenceLow: true,
referenceHigh: true,
sourceReferenceLow: true,
sourceReferenceHigh: true,
sourceReferenceText: true,
panel: true,
value: true,
takenAt: true,
+8 -9
View File
@@ -10,7 +10,7 @@
* point (`extractFeatures`) stays in the hub.
*/
import { prisma } from "@/lib/db";
import { classifyReferenceRange } from "@/lib/labs/reference-range";
import { classifyAgainstEffectiveRange } from "@/lib/labs/reference-range";
import { calendarDaysUntil } from "@/lib/measurement-reminders/due-day";
import { resolveLabFields } from "@/lib/labs/serialise";
import { readRollupBuckets } from "@/lib/rollups/measurement-rollups";
@@ -164,6 +164,9 @@ export async function readLabsBriefingBlock(
valueText: true,
referenceLow: true,
referenceHigh: true,
sourceReferenceLow: true,
sourceReferenceHigh: true,
sourceReferenceText: true,
takenAt: true,
biomarkerId: true,
biomarker: {
@@ -198,14 +201,10 @@ export async function readLabsBriefingBlock(
for (const list of byMarker.values()) {
const latest = list[0];
const resolved = resolveLabFields(latest, latest.biomarker);
const rangeStatus =
latest.value === null
? ("unknown" as const)
: classifyReferenceRange(
latest.value,
resolved.referenceLow,
resolved.referenceHigh,
);
const rangeStatus = classifyAgainstEffectiveRange(
latest.value,
resolved.effectiveRange,
);
// Trend = latest numeric reading vs the immediately prior numeric reading.
let trend: "rising" | "falling" | "flat" | null = null;
+16 -9
View File
@@ -4,9 +4,11 @@
* Groups the user's numeric lab results by panel DATE (the calendar day of
* `takenAt`, UTC), finds the two most-recent dates, and for every analyte that
* appears in BOTH the latest and the previous panel reports the signed delta,
* the direction, and where the latest value sits against its reference band
* (`classifyReferenceRange`). Qualitative (valueText-only) results carry no
* numeric value, so they are skipped — a delta is meaningless for them.
* the direction, and where the latest value sits against the reference window
* in force for it — the range that reading's own report printed when it has
* one, the catalog band otherwise, resolved through the shared
* `resolveEffectiveReferenceRange`. Qualitative (valueText-only) results carry
* no numeric value, so they are skipped — a delta is meaningless for them.
*
* Pure + Prisma-free so the present / absent states are unit-testable: it is
* absent when there are fewer than two panel dates, no analyte is shared, or
@@ -14,12 +16,14 @@
* only — a delta is not a diagnosis.
*/
import {
classifyReferenceRange,
classifyAgainstEffectiveRange,
resolveEffectiveReferenceRange,
type ReferenceRangeStatus,
type SourceReferenceRange,
} from "@/lib/labs/reference-range";
/** A single numeric lab reading. */
export interface LabChangeRow {
/** A single numeric lab reading, with both windows it could be judged against. */
export interface LabChangeRow extends SourceReferenceRange {
analyte: string;
unit: string;
value: number;
@@ -150,10 +154,13 @@ export function summariseLabChanges(
previous: previousRow.value,
delta,
direction,
status: classifyReferenceRange(
status: classifyAgainstEffectiveRange(
latestRow.value,
latestRow.referenceLow,
latestRow.referenceHigh,
resolveEffectiveReferenceRange(
latestRow.referenceLow,
latestRow.referenceHigh,
latestRow,
),
),
});
}
@@ -0,0 +1,128 @@
import { describe, expect, it } from "vitest";
import {
classifyAgainstEffectiveRange,
resolveEffectiveReferenceRange,
} from "@/lib/labs/reference-range";
const NO_SOURCE = {
sourceReferenceLow: null,
sourceReferenceHigh: null,
sourceReferenceText: null,
};
describe("resolveEffectiveReferenceRange", () => {
it("falls back to the catalog band when the reading carries no source window", () => {
const eff = resolveEffectiveReferenceRange(3.5, 5, NO_SOURCE);
expect(eff).toMatchObject({
low: 3.5,
high: 5,
origin: "catalog",
catalogLow: 3.5,
catalogHigh: 5,
sourceText: null,
divergesFromCatalog: false,
});
});
it("reports no window when neither side states one", () => {
expect(resolveEffectiveReferenceRange(null, null, NO_SOURCE)).toMatchObject(
{ low: null, high: null, origin: "none", divergesFromCatalog: false },
);
});
it("lets the source window win over the catalog band for that reading", () => {
const eff = resolveEffectiveReferenceRange(3.5, 5, {
sourceReferenceLow: 3.9,
sourceReferenceHigh: 5.4,
sourceReferenceText: "3,9 - 5,4",
});
expect(eff).toMatchObject({
low: 3.9,
high: 5.4,
origin: "source",
catalogLow: 3.5,
catalogHigh: 5,
sourceText: "3,9 - 5,4",
divergesFromCatalog: true,
});
});
it("does not call an identical window a divergence", () => {
const eff = resolveEffectiveReferenceRange(3.5, 5, {
sourceReferenceLow: 3.5,
sourceReferenceHigh: 5,
sourceReferenceText: "3,5 - 5,0",
});
expect(eff.origin).toBe("source");
expect(eff.divergesFromCatalog).toBe(false);
});
it("takes a one-sided source window whole, not merged with the catalog", () => {
// A report that prints "< 5" states a ceiling and NO floor. Borrowing the
// catalog's floor would invent a window the report never stated.
const eff = resolveEffectiveReferenceRange(3.5, 5, {
sourceReferenceLow: null,
sourceReferenceHigh: 4.2,
sourceReferenceText: "< 4,2",
});
expect(eff).toMatchObject({
low: null,
high: 4.2,
origin: "source",
divergesFromCatalog: true,
});
});
it("does not displace the catalog for a printed window with no derivable bound", () => {
// "negativ" is carried through verbatim, but it is not an argument for
// discarding the catalog band.
const eff = resolveEffectiveReferenceRange(3.5, 5, {
sourceReferenceLow: null,
sourceReferenceHigh: null,
sourceReferenceText: "negativ",
});
expect(eff).toMatchObject({
low: 3.5,
high: 5,
origin: "catalog",
sourceText: "negativ",
divergesFromCatalog: false,
});
});
it("reports a source window as no divergence when the catalog has none", () => {
const eff = resolveEffectiveReferenceRange(null, null, {
sourceReferenceLow: 3.9,
sourceReferenceHigh: 5.4,
sourceReferenceText: "3,9 - 5,4",
});
expect(eff.origin).toBe("source");
expect(eff.divergesFromCatalog).toBe(false);
});
});
describe("classifyAgainstEffectiveRange", () => {
it("judges a reading by the source window, not the catalog band", () => {
// 5.2 sits inside the catalog band (3.5–5.0 would call it high) but the
// report that produced this value states 3.9–5.4, so it is in range.
const eff = resolveEffectiveReferenceRange(3.5, 5, {
sourceReferenceLow: 3.9,
sourceReferenceHigh: 5.4,
sourceReferenceText: "3,9 - 5,4",
});
expect(classifyAgainstEffectiveRange(5.2, eff)).toBe("in-range");
expect(classifyAgainstEffectiveRange(3.7, eff)).toBe("below");
expect(classifyAgainstEffectiveRange(5.5, eff)).toBe("above");
});
it("keeps the catalog verdict for a reading with no source window", () => {
const eff = resolveEffectiveReferenceRange(3.5, 5, NO_SOURCE);
expect(classifyAgainstEffectiveRange(5.2, eff)).toBe("above");
});
it("never classifies a qualitative reading", () => {
const eff = resolveEffectiveReferenceRange(3.5, 5, NO_SOURCE);
expect(classifyAgainstEffectiveRange(null, eff)).toBe("unknown");
});
});
@@ -0,0 +1,219 @@
import { describe, expect, it } from "vitest";
import { parseReferenceRange } from "@/lib/labs/parse-reference-range";
/**
* Real-shaped strings, taken from the notation German and English lab reports
* actually print. The parser's contract is asymmetric on purpose: deriving a
* bound is best-effort, KEEPING the printed string is mandatory. Every case
* below therefore asserts the verbatim text survives, and only the cases that
* can be read without guessing assert bounds.
*/
describe("parseReferenceRange — two-sided windows", () => {
it.each([
["3.5-5.0", 3.5, 5],
["3,5-5,0", 3.5, 5],
["3.5 - 5.0", 3.5, 5],
["3,5 – 5,0", 3.5, 5],
["3,5 — 5,0", 3.5, 5],
["3,5 bis 5,0", 3.5, 5],
["3.5 to 5.0", 3.5, 5],
["3,5 ... 5,0", 3.5, 5],
["3,5 .. 5,0", 3.5, 5],
["von 3,5 bis 5,0", 3.5, 5],
["[3,5 - 5,0]", 3.5, 5],
["(3.5-5.0)", 3.5, 5],
])("reads %s as %s..%s", (raw, low, high) => {
const parsed = parseReferenceRange(raw);
expect(parsed).toEqual({ low, high, text: raw.trim() });
});
it("reads a negative lower bound (base excess prints one)", () => {
expect(parseReferenceRange("-2,0 bis +3,0")).toEqual({
low: -2,
high: 3,
text: "-2,0 bis +3,0",
});
});
it("reads a German thousands separator as one number, not a decimal", () => {
// Platelets print as "150.000 - 400.000 /µl" on a German report. Reading
// the dot as a decimal point would yield a 150–400 window and silently
// flag every normal count as wildly high.
expect(parseReferenceRange("150.000 - 400.000")).toEqual({
low: 150000,
high: 400000,
text: "150.000 - 400.000",
});
});
it("reads an English thousands separator", () => {
expect(parseReferenceRange("150,000 - 400,000")).toEqual({
low: 150000,
high: 400000,
text: "150,000 - 400,000",
});
});
it("refuses a transposed window rather than swapping it", () => {
// Swapping would be a guess about which digit the lab meant to print.
expect(parseReferenceRange("5,0 - 3,5")).toEqual({
low: null,
high: null,
text: "5,0 - 3,5",
});
});
});
describe("parseReferenceRange — one-sided windows", () => {
it.each([
["< 5", 5],
["<5,0", 5],
["≤ 5", 5],
["<= 5", 5],
["bis 5,0", 5],
["bis zu 5,0", 5],
["max. 5,0", 5],
["max 5", 5],
["kleiner 5", 5],
["unter 5", 5],
["up to 5.0", 5],
])("reads %s as an upper bound of %s", (raw, high) => {
expect(parseReferenceRange(raw)).toEqual({
low: null,
high,
text: raw.trim(),
});
});
it.each([
["> 3.5", 3.5],
[">3,5", 3.5],
["≥ 3,5", 3.5],
[">= 3.5", 3.5],
["ab 3,5", 3.5],
["min. 3,5", 3.5],
["mind. 3,5", 3.5],
["mindestens 3,5", 3.5],
["größer 3,5", 3.5],
["groesser 3,5", 3.5],
["über 3,5", 3.5],
])("reads %s as a lower bound of %s", (raw, low) => {
expect(parseReferenceRange(raw)).toEqual({
low,
high: null,
text: raw.trim(),
});
});
});
describe("parseReferenceRange — units", () => {
it("reads a window that carries the reading's own unit", () => {
expect(parseReferenceRange("3,5 - 5,0 mmol/l", "mmol/L")).toEqual({
low: 3.5,
high: 5,
text: "3,5 - 5,0 mmol/l",
});
});
it("reads a trailing unit when the caller states none", () => {
expect(parseReferenceRange("0,5 - 1,2 mg/dl")).toEqual({
low: 0.5,
high: 1.2,
text: "0,5 - 1,2 mg/dl",
});
});
it("refuses bounds when the printed unit contradicts the reading's unit", () => {
// Converting mmol/L into mg/dL is interpretation and needs an analyte-
// specific factor. Keeping the string and no bounds is the honest answer.
const parsed = parseReferenceRange("3,5 - 5,0 mmol/l", "mg/dL");
expect(parsed).toEqual({
low: null,
high: null,
text: "3,5 - 5,0 mmol/l",
});
});
it("matches units case- and space-insensitively", () => {
expect(parseReferenceRange("< 5 MG/DL", "mg/dl")).toEqual({
low: null,
high: 5,
text: "< 5 MG/DL",
});
});
it("tolerates a percent window against a percent reading", () => {
expect(parseReferenceRange("4,0 - 6,0 %", "%")).toEqual({
low: 4,
high: 6,
text: "4,0 - 6,0 %",
});
});
});
describe("parseReferenceRange — text the parser must not invent bounds for", () => {
it.each([
"negativ",
"negative",
"nicht nachweisbar",
"siehe Befund",
"n.n.",
"unauffällig",
"altersabhängig",
"methodenabhängig",
])("keeps %s verbatim with no bounds", (raw) => {
expect(parseReferenceRange(raw)).toEqual({
low: null,
high: null,
text: raw,
});
});
it("keeps a bare number verbatim without treating it as a bound", () => {
// "5,0" alone states neither a ceiling nor a floor; picking one would be
// the parser deciding what the report meant.
expect(parseReferenceRange("5,0")).toEqual({
low: null,
high: null,
text: "5,0",
});
});
it("keeps a three-number string verbatim", () => {
expect(parseReferenceRange("3,5 - 5,0 - 7,0")).toEqual({
low: null,
high: null,
text: "3,5 - 5,0 - 7,0",
});
});
it("caps an over-long string rather than storing a whole paragraph", () => {
const parsed = parseReferenceRange("x".repeat(500));
expect(parsed?.text).toHaveLength(120);
expect(parsed?.low).toBeNull();
expect(parsed?.high).toBeNull();
});
it("collapses internal whitespace so the stored string stays one line", () => {
expect(parseReferenceRange(" 3,5\n\t- 5,0 ")).toEqual({
low: 3.5,
high: 5,
text: "3,5 - 5,0",
});
});
});
describe("parseReferenceRange — absence", () => {
it.each([null, undefined, "", " "])("returns null for %s", (raw) => {
expect(parseReferenceRange(raw)).toBeNull();
});
it("refuses a non-finite bound", () => {
expect(parseReferenceRange("1e999 - 2e999")).toEqual({
low: null,
high: null,
text: "1e999 - 2e999",
});
});
});
+3
View File
@@ -17,6 +17,9 @@ const baseRow: LabRow = {
unit: "mg/dl", // lowercase legacy unit
referenceLow: null,
referenceHigh: 100, // stale per-row bound
sourceReferenceLow: null,
sourceReferenceHigh: null,
sourceReferenceText: null,
takenAt: new Date("2026-01-01T08:00:00.000Z"),
source: "MANUAL",
biomarkerId: "bm_1",
+15 -6
View File
@@ -24,6 +24,7 @@ import {
} from "@/lib/ai/types";
import { AI_BUDGETS } from "@/lib/ai/ai-budgets";
import { prisma } from "@/lib/db";
import { parseReferenceRange } from "@/lib/labs/parse-reference-range";
import { annotate } from "@/lib/logging/context";
import {
extractedLabsSchema,
@@ -42,7 +43,8 @@ Extract every analyte reading you can read. For each reading capture:
- value: the numeric result as a number, OR null if the result is qualitative text.
- valueText: the qualitative result text (e.g. "negativ", "positiv", "nicht nachweisbar"), OR null if the result is numeric.
- unit: the measurement unit (e.g. "mg/dL", "%", "ng/mL"), or null if none is printed or the reading is qualitative.
- referenceLow / referenceHigh: the reference-range bounds as numbers, or null. A one-sided range like "< 116" sets referenceHigh only; "> 40" sets referenceLow only.
- referenceText: the reference range EXACTLY as printed beside the value, verbatim and complete, including its operator and unit if written (for example "3,5 - 5,0", "< 116", "bis 5,0", "150.000 - 400.000 /\u00b5l", "negativ"). Copy the characters; do NOT reformat, convert, or normalise them. Null only when no range is printed for that row.
- referenceLow / referenceHigh: the SAME range reduced to numbers, when it reduces to numbers, else null. A one-sided range like "< 116" sets referenceHigh only; "> 40" sets referenceLow only.
- takenAt: the per-row collection date in ISO 8601 (YYYY-MM-DD) if a date is printed for that row, else null.
- confidence: an object { analyte, value, unit, range } with a 0..1 score for how legible each field was. Use a low score when a field is blurred, ambiguous, or you had to guess.
@@ -51,7 +53,7 @@ Set EXACTLY ONE of value / valueText per reading. Never invent a value you canno
Also capture reportDate: the report's collection/sample date in ISO 8601 (YYYY-MM-DD), or null.
Respond ONLY with a JSON object of this exact shape:
{ "reportDate": string|null, "rows": [ { "analyte": string, "value": number|null, "valueText": string|null, "unit": string|null, "referenceLow": number|null, "referenceHigh": number|null, "takenAt": string|null, "confidence": { "analyte": number, "value": number, "unit": number, "range": number } } ] }`;
{ "reportDate": string|null, "rows": [ { "analyte": string, "value": number|null, "valueText": string|null, "unit": string|null, "referenceText": string|null, "referenceLow": number|null, "referenceHigh": number|null, "takenAt": string|null, "confidence": { "analyte": number, "value": number, "unit": number, "range": number } } ] }`;
const USER_PROMPT = `Transcribe the lab report in the attached image into the JSON schema described in the system prompt. Return only the JSON object.`;
@@ -70,7 +72,8 @@ Extract every analyte reading you can identify. For each reading capture:
- value: the numeric result as a number, OR null if the result is qualitative text. German reports use a decimal comma (e.g. "5,4") — emit it as 5.4.
- valueText: the qualitative result text (e.g. "negativ", "positiv", "nicht nachweisbar"), OR null if the result is numeric.
- unit: the measurement unit (e.g. "mg/dL", "%", "ng/mL"), or null if none is printed or the reading is qualitative.
- referenceLow / referenceHigh: the reference-range bounds as numbers, or null. A one-sided range like "< 116" sets referenceHigh only; "> 40" sets referenceLow only.
- referenceText: the reference range EXACTLY as printed beside the value, verbatim and complete, including its operator and unit if written (for example "3,5 - 5,0", "< 116", "bis 5,0", "150.000 - 400.000 /\u00b5l", "negativ"). Copy the characters; do NOT reformat, convert, or normalise them. Null only when no range is printed for that row.
- referenceLow / referenceHigh: the SAME range reduced to numbers, when it reduces to numbers, else null. A one-sided range like "< 116" sets referenceHigh only; "> 40" sets referenceLow only.
- takenAt: the per-row collection date in ISO 8601 (YYYY-MM-DD) if a date is printed for that row, else null.
- confidence: an object { analyte, value, unit, range } with a 0..1 score for how confident you are in each field. Use a LOW score when the OCR text was garbled, ambiguous, or you had to guess — the text is lossy, so be conservative.
@@ -79,7 +82,7 @@ Set EXACTLY ONE of value / valueText per reading. Never invent a value you canno
Also capture reportDate: the report's collection/sample date in ISO 8601 (YYYY-MM-DD), or null.
Respond ONLY with a JSON object of this exact shape:
{ "reportDate": string|null, "rows": [ { "analyte": string, "value": number|null, "valueText": string|null, "unit": string|null, "referenceLow": number|null, "referenceHigh": number|null, "takenAt": string|null, "confidence": { "analyte": number, "value": number, "unit": number, "range": number } } ] }`;
{ "reportDate": string|null, "rows": [ { "analyte": string, "value": number|null, "valueText": string|null, "unit": string|null, "referenceText": string|null, "referenceLow": number|null, "referenceHigh": number|null, "takenAt": string|null, "confidence": { "analyte": number, "value": number, "unit": number, "range": number } } ] }`;
/** A normalised ISO date string (YYYY-MM-DD) or null. */
function normaliseDate(raw: string | null): string | null {
@@ -130,6 +133,7 @@ async function annotateRow(
): Promise<OcrExtractedRowDto> {
const analyte = row.analyte.trim();
const takenAt = normaliseDate(row.takenAt) ?? reportDate;
const printed = parseReferenceRange(row.referenceText, row.unit);
// Biomarker match — case-insensitive on the catalog identity, presence-only.
const existingBiomarker = await prisma.biomarker.findFirst({
@@ -175,8 +179,13 @@ async function annotateRow(
value: row.value,
valueText: row.valueText,
unit: row.unit,
referenceLow: row.referenceLow,
referenceHigh: row.referenceHigh,
// Both readings of the printed window travel to the review screen: the
// derived bounds where the string yields them, and the string itself
// either way. A window the parser could not read is still the window the
// lab printed, so the user sees it and it reaches the row.
referenceLow: printed?.low ?? row.referenceLow,
referenceHigh: printed?.high ?? row.referenceHigh,
referenceText: printed?.text ?? null,
takenAt,
confidence: row.confidence,
biomarkerMatch: existingBiomarker ? "existing" : "new",
+207
View File
@@ -0,0 +1,207 @@
/**
* Free-text reference-range parser for the range a lab report prints beside a
* value.
*
* A report states its window in prose, not in columns: "3,5 - 5,0", "< 5",
* "bis 5,0", "≥ 3,5", "150.000 - 400.000 /µl", or a word like "negativ" that
* carries no window at all. The document-reading path and the manual entry
* form both hand that string here.
*
* The contract is deliberately asymmetric:
*
* - KEEPING the printed string is mandatory. `text` always carries what the
* report said, whitespace-normalised and length-capped, whether or not a
* bound could be derived from it. A window this parser cannot read is not
* a window that gets thrown away.
* - DERIVING bounds is best-effort and refuses on ambiguity. A string that
* does not match one of the notations below yields `low: null, high: null`
* beside the text. Absence reads as absence — the parser never picks a
* plausible bound out of a string it did not understand, because a fake
* window silently reclassifies a reading as normal or abnormal.
*
* Notations read (German and English, comma- or dot-decimal):
*
* two-sided 3.5-5.0 · 3,5 – 5,0 · 3,5 bis 5,0 · 3.5 to 5.0 · 3,5 ... 5,0
* von 3,5 bis 5,0 · [3,5 - 5,0] · (3.5-5.0)
* upper only < 5 · ≤ 5 · <= 5 · bis 5,0 · bis zu 5,0 · max. 5,0 ·
* kleiner 5 · unter 5 · up to 5.0 · less than 5
* lower only > 3.5 · ≥ 3,5 · >= 3.5 · ab 3,5 · min. 3,5 · mindestens 3,5 ·
* größer 3,5 · über 3,5 · at least 3.5
*
* Units are read, never converted. A window may carry the unit the report
* printed it in; when the caller states the reading's own unit and the two
* disagree, the bounds are refused and only the text survives. Converting
* mmol/L into mg/dL needs a per-analyte factor, so doing it here would be the
* parser deciding what the report meant.
*/
/** What a printed range yielded. `text` is set whenever the input was non-empty. */
export interface ParsedReferenceRange {
/** Lower bound, or null when the notation states none / could not be read. */
low: number | null;
/** Upper bound, or null when the notation states none / could not be read. */
high: number | null;
/** The printed string, whitespace-normalised and capped. Never null here. */
text: string;
}
/**
* Storage cap for the verbatim string. A reference-range cell is a few
* characters; anything longer is a mis-segmented paragraph, and truncating it
* keeps a runaway transcription out of the column.
*/
export const SOURCE_REFERENCE_TEXT_MAX = 120;
/** Magnitude ceiling. Beyond this a "bound" is a transcription artefact. */
const MAX_ABS_BOUND = 1e12;
/** One number token: optional sign, digits, any number of `.`/`,` groups. */
const NUM = String.raw`[+-]?\d+(?:[.,]\d+)*`;
/** Infix separators for a two-sided window. */
const TWO_SIDED_SEP = String.raw`(?:-|–|—|‒|…|\.{2,3}|bis\s+zu|bis|to|until)`;
/** Prefixes that state a ceiling only. */
const UPPER_PREFIX = String.raw`(?:<=|<|≤|bis\s+zu|bis|max\.|maximal|max|kleiner\s+als|kleiner|unter|höchstens|hoechstens|up\s+to|less\s+than|below)`;
/** Prefixes that state a floor only. */
const LOWER_PREFIX = String.raw`(?:>=|>|≥|ab|mind\.|mindestens|minimal|min\.|min|größer\s+als|größer|groesser\s+als|groesser|über|ueber|at\s+least|greater\s+than|above)`;
/**
* Trailing unit tail. Deliberately whitespace-free and comma-free: a real unit
* symbol ("mmol/l", "mg/dL", "%", "/µl", "10^9/L") never contains either, and
* refusing anything that does keeps a third number ("3,5 - 5,0 - 7,0") from
* being swallowed as if it were a unit.
*/
const UNIT_TAIL = String.raw`(?:\s*([^\s,]{1,24}))?`;
const TWO_SIDED_RE = new RegExp(
`^(${NUM})\\s*${TWO_SIDED_SEP}\\s*(${NUM})${UNIT_TAIL}$`,
"iu",
);
const UPPER_RE = new RegExp(`^${UPPER_PREFIX}\\s*(${NUM})${UNIT_TAIL}$`, "iu");
const LOWER_RE = new RegExp(`^${LOWER_PREFIX}\\s*(${NUM})${UNIT_TAIL}$`, "iu");
/** Leading words that only introduce the window ("von 3,5 bis 5,0"). */
const LEADING_FILLER_RE = /^(?:von|from|ref\.?|referenz|reference)\s+/iu;
/**
* Read one number token into a JS number, or null when its separators are
* ambiguous.
*
* Separator rules, in order:
* 1. Both `.` and `,` present → the LAST one is the decimal mark, the other
* groups thousands ("1.234,56" and "1,234.56" both read as 1234.56).
* 2. One separator kind, appearing more than once → all thousands
* ("1.234.567").
* 3. One separator, appearing once, group ≠ 3 digits → decimal ("3,5").
* 4. One separator, appearing once, group = 3 digits → ambiguous in the
* general case. Read as thousands only when the group is "000", which is
* how a German report prints a rounded count ("150.000 - 400.000 /µl");
* reading that as 150 would flag every normal platelet count as wildly
* high. Any other 3-digit group ("1.234" — 1234 or 1.234?) is refused.
*
* The limit of rule 4 is real: a report that prints "5.000" meaning five with
* three decimal places is read as five thousand. The verbatim text is stored
* either way, so the printed window is never lost — only the derived bound is.
*/
function readNumber(token: string): number | null {
const sign = token.startsWith("-") ? -1 : 1;
const body = token.replace(/^[+-]/u, "");
const dots = (body.match(/\./gu) ?? []).length;
const commas = (body.match(/,/gu) ?? []).length;
let normalised: string;
if (dots > 0 && commas > 0) {
const decimalMark =
body.lastIndexOf(".") > body.lastIndexOf(",") ? "." : ",";
const grouping = decimalMark === "." ? "," : ".";
normalised = body.split(grouping).join("").replace(decimalMark, ".");
} else if (dots + commas === 0) {
normalised = body;
} else if (dots > 1 || commas > 1) {
normalised = body.replace(/[.,]/gu, "");
} else {
const idx = body.search(/[.,]/u);
const group = body.slice(idx + 1);
if (group.length !== 3) {
normalised = `${body.slice(0, idx)}.${group}`;
} else if (group === "000") {
normalised = body.slice(0, idx) + group;
} else {
return null;
}
}
const value = sign * Number(normalised);
if (!Number.isFinite(value) || Math.abs(value) > MAX_ABS_BOUND) return null;
return value;
}
/** Compare a printed unit against the reading's unit, ignoring case + spaces. */
function unitsAgree(printed: string | undefined, reading: string | undefined) {
if (!printed) return true;
if (!reading || !reading.trim()) return true;
const norm = (u: string) => u.replace(/\s+/gu, "").toLowerCase();
return norm(printed) === norm(reading);
}
/**
* Parse the reference range a lab report printed for one reading.
*
* @param raw the printed string, exactly as transcribed.
* @param unit the reading's own unit, when known. A printed window in a
* different unit yields text-only — never a converted bound.
* @returns null when there was nothing printed at all.
*/
export function parseReferenceRange(
raw: string | null | undefined,
unit?: string | null,
): ParsedReferenceRange | null {
if (raw === null || raw === undefined) return null;
const text = raw
.replace(/\s+/gu, " ")
.trim()
.slice(0, SOURCE_REFERENCE_TEXT_MAX);
if (text === "") return null;
const noBounds: ParsedReferenceRange = { low: null, high: null, text };
// Strip a bracket wrapper and an introducing word for MATCHING only — the
// stored text stays exactly what the report printed.
let candidate = text;
const bracketed = /^[[(](.*)[\])]$/u.exec(candidate);
if (bracketed) candidate = bracketed[1].trim();
candidate = candidate.replace(LEADING_FILLER_RE, "").trim();
const two = TWO_SIDED_RE.exec(candidate);
if (two) {
if (!unitsAgree(two[3], unit ?? undefined)) return noBounds;
const low = readNumber(two[1]);
const high = readNumber(two[2]);
if (low === null || high === null) return noBounds;
// A transposed window is a transcription error, and swapping it would be a
// guess about which digit the lab meant. Refuse the bounds, keep the text.
if (low > high) return noBounds;
return { low, high, text };
}
const upper = UPPER_RE.exec(candidate);
if (upper) {
if (!unitsAgree(upper[2], unit ?? undefined)) return noBounds;
const high = readNumber(upper[1]);
return high === null ? noBounds : { low: null, high, text };
}
const lower = LOWER_RE.exec(candidate);
if (lower) {
if (!unitsAgree(lower[2], unit ?? undefined)) return noBounds;
const low = readNumber(lower[1]);
return low === null ? noBounds : { low, high: null, text };
}
// Anything else — "negativ", "siehe Befund", a bare number, prose — keeps its
// text and states no window.
return noBounds;
}
+110
View File
@@ -61,3 +61,113 @@ export function formatReferenceRange(
if (hasLow) return `≥ ${formatNumber(low as number)}`;
return opts?.emptyText ?? "";
}
/**
* The reference range a lab report printed beside ONE reading, as stored on
* `LabResult.sourceReference*`.
*/
export interface SourceReferenceRange {
sourceReferenceLow: number | null;
sourceReferenceHigh: number | null;
sourceReferenceText: string | null;
}
/** Where the bounds a reading is judged against came from. */
export type ReferenceRangeOrigin = "source" | "catalog" | "none";
/** The bounds one reading is judged against, plus how they were arrived at. */
export interface EffectiveReferenceRange {
/** The bound the verdict uses. */
low: number | null;
high: number | null;
origin: ReferenceRangeOrigin;
/** The catalog band, kept alongside so a surface can show both. */
catalogLow: number | null;
catalogHigh: number | null;
/** The printed window, kept verbatim. Null when the report stated none. */
sourceText: string | null;
/**
* True when the source window is in force AND the catalog states a window
* that differs from it. This is the case a reader must be told about: the
* same number reads differently against the two, so a surface that shows
* only one of them is showing a partial answer.
*/
divergesFromCatalog: boolean;
}
/** Two bounds are the same window when both ends match (null included). */
function sameWindow(
aLow: number | null,
aHigh: number | null,
bLow: number | null,
bHigh: number | null,
): boolean {
return aLow === bLow && aHigh === bHigh;
}
/**
* Resolve the reference window ONE reading is judged against.
*
* A lab runs its own method on its own device, and the physician reads the
* value against the window printed on that report. So for a reading that
* carries a source window, the source window wins — for that reading alone.
* The catalog band stays the net for every reading that carries none.
*
* A source window counts as stating a window only when it yields at least one
* numeric bound. A printed string with no derivable bound ("negativ", "siehe
* Befund") is carried through as `sourceText` but never displaces the catalog:
* a range that could not be read is not an argument for having no range.
*
* This function is the ONLY place that precedence is expressed. Every surface
* that classifies a reading — the API DTO, the doctor report, insights, the
* coach snapshot, the MCP read, the FHIR export — routes through it, so a
* reading cannot read as in-range on one surface and out on another.
*/
export function resolveEffectiveReferenceRange(
catalogLow: number | null | undefined,
catalogHigh: number | null | undefined,
source: SourceReferenceRange | null | undefined,
): EffectiveReferenceRange {
const cLow = catalogLow ?? null;
const cHigh = catalogHigh ?? null;
const sLow = source?.sourceReferenceLow ?? null;
const sHigh = source?.sourceReferenceHigh ?? null;
const sourceText = source?.sourceReferenceText ?? null;
if (sLow !== null || sHigh !== null) {
return {
low: sLow,
high: sHigh,
origin: "source",
catalogLow: cLow,
catalogHigh: cHigh,
sourceText,
divergesFromCatalog:
(cLow !== null || cHigh !== null) &&
!sameWindow(sLow, sHigh, cLow, cHigh),
};
}
return {
low: cLow,
high: cHigh,
origin: cLow !== null || cHigh !== null ? "catalog" : "none",
catalogLow: cLow,
catalogHigh: cHigh,
sourceText,
divergesFromCatalog: false,
};
}
/**
* Classify one reading against its effective window. Qualitative readings
* (no numeric value) always report `"unknown"` — there is nothing to compare
* against bounds, and a fabricated verdict is worse than none.
*/
export function classifyAgainstEffectiveRange(
value: number | null,
effective: EffectiveReferenceRange,
): ReferenceRangeStatus {
if (value === null) return "unknown";
return classifyReferenceRange(value, effective.low, effective.high);
}
+138 -58
View File
@@ -10,13 +10,40 @@
* - If the row is unlinked (legacy / pre-backfill), it falls back to its own
* free-text `analyte` / `unit` / `reference*` columns.
*
* Either way the verdict (`rangeStatus`) is computed from the RESOLVED bounds
* via `classifyReferenceRange`. The DTO carries the resolved values so the web
* client AND the iOS client render the same numbers — neither recomputes the
* range or guesses the unit. This is the server-authoritative-parity rule for
* Labs.
* On top of that the reading's OWN printed window wins. A lab report states
* the range its method and device produce right beside the value, and that is
* the window a physician evaluates against; the catalog band is the net for
* readings that carry none. `resolveEffectiveReferenceRange` owns that
* precedence and this file is one of its callers, not a second copy of it.
*
* The DTO therefore carries three windows and says which one is in force:
*
* - `referenceLow` / `referenceHigh` — the EFFECTIVE bounds, the ones
* `rangeStatus` was computed from. A client that reads only these fields
* (including an iOS build that predates the source window) renders the
* right numbers and the right verdict with no change of its own.
* - `catalogReferenceLow` / `catalogReferenceHigh` — the catalog band, so a
* surface can paint both and show where they part.
* - `sourceReferenceLow` / `sourceReferenceHigh` / `sourceReferenceText` —
* what the report printed, the last verbatim.
*
* `referenceOrigin` names which window is in force and
* `referenceDivergesFromCatalog` flags the case a reader must be told about.
* Every value is resolved server-side; neither client recomputes a range,
* guesses a unit, or re-derives the verdict.
*
* The two serialisers below spell every field out rather than spreading a
* shared helper: `written-outcome-response-consumer-guard` walks these object
* literals to enumerate the response leaves and pair each with a client
* reader, and a spread of a function call hides them from it. A field added
* here must be visible there.
*/
import { classifyReferenceRange } from "@/lib/labs/reference-range";
import {
classifyAgainstEffectiveRange,
resolveEffectiveReferenceRange,
type EffectiveReferenceRange,
type ReferenceRangeOrigin,
} from "@/lib/labs/reference-range";
/** The minimal biomarker shape the resolver needs (no encrypted context). */
export interface ResolvedBiomarker {
@@ -40,6 +67,10 @@ export interface LabRow {
unit: string;
referenceLow: number | null;
referenceHigh: number | null;
/** The window the source report printed for THIS reading. */
sourceReferenceLow: number | null;
sourceReferenceHigh: number | null;
sourceReferenceText: string | null;
takenAt: Date;
source: string;
biomarkerId: string | null;
@@ -48,60 +79,95 @@ export interface LabRow {
updatedAt: Date;
}
/**
* Resolve the canonical name / unit / bounds / panel for a lab row, preferring
* the linked biomarker over the legacy per-row fields. Returns a plain shape
* the list + detail serialisers both build on.
*/
export function resolveLabFields(
row: Pick<
LabRow,
| "analyte"
| "unit"
| "referenceLow"
| "referenceHigh"
| "panel"
| "biomarkerId"
>,
biomarker: ResolvedBiomarker | null | undefined,
): {
/** The columns `resolveLabFields` needs off a lab row. */
export type LabFieldRow = Pick<
LabRow,
| "analyte"
| "unit"
| "referenceLow"
| "referenceHigh"
| "sourceReferenceLow"
| "sourceReferenceHigh"
| "sourceReferenceText"
| "panel"
| "biomarkerId"
>;
/** What every lab surface reads off a row: identity, unit, and one window. */
export interface ResolvedLabFields {
analyte: string;
unit: string;
panel: string | null;
/** The bounds in force for this reading — source window first, catalog next. */
referenceLow: number | null;
referenceHigh: number | null;
panel: string | null;
} {
if (biomarker) {
return {
analyte: biomarker.name,
unit: biomarker.unit,
referenceLow: biomarker.lowerBound,
referenceHigh: biomarker.upperBound,
panel: biomarker.panel,
};
}
return {
analyte: row.analyte,
unit: row.unit,
referenceLow: row.referenceLow,
referenceHigh: row.referenceHigh,
panel: row.panel,
};
/** The catalog band, so a surface can show both windows. */
catalogReferenceLow: number | null;
catalogReferenceHigh: number | null;
/** What the source report printed for this reading. */
sourceReferenceLow: number | null;
sourceReferenceHigh: number | null;
sourceReferenceText: string | null;
referenceOrigin: ReferenceRangeOrigin;
referenceDivergesFromCatalog: boolean;
/** The resolved window as one value, for callers that pass it on. */
effectiveRange: EffectiveReferenceRange;
}
/**
* v1.18.9 — the reference-range verdict for a row. A qualitative row (no
* numeric `value`) has nothing to compare against the bounds, so it always
* reports `"unknown"` — the neutral, no-verdict state — never a fabricated
* in/out classification. A numeric row classifies against the resolved bounds.
* Resolve the canonical name / unit / window / panel for a lab row.
*
* Two resolutions happen here, in this order:
*
* 1. Identity + catalog band: the linked biomarker wins over the row's legacy
* free-text columns (an unlinked pre-backfill row falls back to its own).
* 2. The window in force: the reading's printed source range wins over the
* catalog band, through `resolveEffectiveReferenceRange`.
*
* Every lab surface calls this, so the window a reading is judged against is
* the same on the API, the doctor report, insights, the coach, MCP and FHIR.
*/
function rowRangeStatus(
value: number | null,
referenceLow: number | null,
referenceHigh: number | null,
) {
if (value === null) return "unknown" as const;
return classifyReferenceRange(value, referenceLow, referenceHigh);
export function resolveLabFields(
row: LabFieldRow,
biomarker: ResolvedBiomarker | null | undefined,
): ResolvedLabFields {
const identity = biomarker
? {
analyte: biomarker.name,
unit: biomarker.unit,
panel: biomarker.panel,
catalogLow: biomarker.lowerBound,
catalogHigh: biomarker.upperBound,
}
: {
analyte: row.analyte,
unit: row.unit,
panel: row.panel,
catalogLow: row.referenceLow,
catalogHigh: row.referenceHigh,
};
const effectiveRange = resolveEffectiveReferenceRange(
identity.catalogLow,
identity.catalogHigh,
row,
);
return {
analyte: identity.analyte,
unit: identity.unit,
panel: identity.panel,
referenceLow: effectiveRange.low,
referenceHigh: effectiveRange.high,
catalogReferenceLow: effectiveRange.catalogLow,
catalogReferenceHigh: effectiveRange.catalogHigh,
sourceReferenceLow: row.sourceReferenceLow,
sourceReferenceHigh: row.sourceReferenceHigh,
sourceReferenceText: effectiveRange.sourceText,
referenceOrigin: effectiveRange.origin,
referenceDivergesFromCatalog: effectiveRange.divergesFromCatalog,
effectiveRange,
};
}
/**
@@ -123,13 +189,21 @@ export function serialiseLabResult(
unit: resolved.unit,
referenceLow: resolved.referenceLow,
referenceHigh: resolved.referenceHigh,
catalogReferenceLow: resolved.catalogReferenceLow,
catalogReferenceHigh: resolved.catalogReferenceHigh,
sourceReferenceLow: resolved.sourceReferenceLow,
sourceReferenceHigh: resolved.sourceReferenceHigh,
sourceReferenceText: resolved.sourceReferenceText,
referenceOrigin: resolved.referenceOrigin,
referenceDivergesFromCatalog: resolved.referenceDivergesFromCatalog,
takenAt: row.takenAt.toISOString(),
source: row.source,
hasNote: row.noteEncrypted !== null,
rangeStatus: rowRangeStatus(
// A qualitative row has no number to place against bounds, so it reports
// the neutral "unknown" rather than a fabricated in/out verdict.
rangeStatus: classifyAgainstEffectiveRange(
row.value,
resolved.referenceLow,
resolved.referenceHigh,
resolved.effectiveRange,
),
createdAt: row.createdAt.toISOString(),
updatedAt: row.updatedAt.toISOString(),
@@ -156,13 +230,19 @@ export function serialiseLabResultDetail(
unit: resolved.unit,
referenceLow: resolved.referenceLow,
referenceHigh: resolved.referenceHigh,
catalogReferenceLow: resolved.catalogReferenceLow,
catalogReferenceHigh: resolved.catalogReferenceHigh,
sourceReferenceLow: resolved.sourceReferenceLow,
sourceReferenceHigh: resolved.sourceReferenceHigh,
sourceReferenceText: resolved.sourceReferenceText,
referenceOrigin: resolved.referenceOrigin,
referenceDivergesFromCatalog: resolved.referenceDivergesFromCatalog,
takenAt: row.takenAt.toISOString(),
source: row.source,
note,
rangeStatus: rowRangeStatus(
rangeStatus: classifyAgainstEffectiveRange(
row.value,
resolved.referenceLow,
resolved.referenceHigh,
resolved.effectiveRange,
),
createdAt: row.createdAt.toISOString(),
updatedAt: row.updatedAt.toISOString(),
+8 -9
View File
@@ -48,7 +48,7 @@ import {
type MetricStatusMetricId,
} from "@/lib/insights/metric-status-registry";
import { allSignals, getSignal } from "@/lib/signals/registry";
import { classifyReferenceRange } from "@/lib/labs/reference-range";
import { classifyAgainstEffectiveRange } from "@/lib/labs/reference-range";
import { resolveLabFields } from "@/lib/labs/serialise";
import { sanitizeValueText } from "@/lib/ai/coach/labs-snapshot";
import { encodeOffsetCursor } from "@/lib/mcp/pagination";
@@ -1316,6 +1316,9 @@ export async function getLabHistory(
unit: true,
referenceLow: true,
referenceHigh: true,
sourceReferenceLow: true,
sourceReferenceHigh: true,
sourceReferenceText: true,
takenAt: true,
biomarkerId: true,
biomarker: {
@@ -1350,14 +1353,10 @@ export async function getLabHistory(
unit: resolved.unit,
referenceLow: resolved.referenceLow,
referenceHigh: resolved.referenceHigh,
rangeStatus:
row.value === null
? "unknown"
: classifyReferenceRange(
row.value,
resolved.referenceLow,
resolved.referenceHigh,
),
rangeStatus: classifyAgainstEffectiveRange(
row.value,
resolved.effectiveRange,
),
takenAt: row.takenAt.toISOString(),
};
});
@@ -0,0 +1,222 @@
/**
* The ingest-side plausibility gate.
*
* `VALUE_RANGES` is the band the application declares a metric can occupy, and
* every path a person can drive has always enforced it. The provider sync
* writers did not, which is how a row the application itself calls impossible
* reached the table and poisoned every mean and personal band derived from it.
* These cases pin the two halves of the fix: the refusal, and the fact that a
* refusal is never silent.
*/
import { beforeEach, describe, expect, it } from "vitest";
import { WideEventBuilder } from "@/lib/logging/event-builder";
import { eventStorage } from "@/lib/logging/context";
import {
RANGE_REJECTED_META_KEY,
classifyRangeRejection,
dropImplausibleMeasurements,
recordRangeRejections,
type RangeRejectionTally,
} from "../plausibility-gate";
let event: WideEventBuilder;
function withEvent<T>(run: () => T): T {
return eventStorage.run(event, run);
}
function tally(): RangeRejectionTally | undefined {
return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as
RangeRejectionTally | undefined;
}
beforeEach(() => {
event = new WideEventBuilder("background");
});
describe("classifyRangeRejection", () => {
it("passes a value inside the metric's declared band", () => {
// PULSE is declared 20–300; both edges are inclusive.
expect(classifyRangeRejection("PULSE", 60)).toBeNull();
expect(classifyRangeRejection("PULSE", 20)).toBeNull();
expect(classifyRangeRejection("PULSE", 300)).toBeNull();
});
it("names the edge an out-of-band value broke", () => {
expect(classifyRangeRejection("PULSE", 55643821.505)).toEqual({
type: "PULSE",
direction: "above_max",
});
expect(classifyRangeRejection("PULSE", 19)).toEqual({
type: "PULSE",
direction: "below_min",
});
});
it("refuses a non-finite value whatever the metric", () => {
expect(classifyRangeRejection("PULSE", Number.NaN)).toEqual({
type: "PULSE",
direction: "not_finite",
});
// No declared band, so the value itself is the only thing to judge.
expect(
classifyRangeRejection("NOT_A_METRIC", Number.POSITIVE_INFINITY),
).toEqual({ type: "NOT_A_METRIC", direction: "not_finite" });
});
it("passes a metric with no declared band — an absent range is an absent fact", () => {
expect(classifyRangeRejection("NOT_A_METRIC", 9_999_999)).toBeNull();
});
});
describe("dropImplausibleMeasurements", () => {
it("keeps the in-band rows and returns them untouched", () => {
const rows = [
{ type: "PULSE", value: 60 },
{ type: "WEIGHT", value: 82.4 },
];
const kept = withEvent(() =>
dropImplausibleMeasurements("withings", rows, (r) => r),
);
expect(kept).toEqual(rows);
expect(kept[0]).toBe(rows[0]);
// Nothing was refused, so the event carries no tally at all — absence
// reads as absence.
expect(tally()).toBeUndefined();
expect(event.toJSON().warnings).toBeUndefined();
});
it("drops the out-of-band rows and tallies them on the wide event", () => {
const kept = withEvent(() =>
dropImplausibleMeasurements(
"withings",
[
{ type: "PULSE", value: 60 },
{ type: "PULSE", value: 55643821.505 },
{ type: "PULSE", value: 900 },
{ type: "WEIGHT", value: 0.2 },
],
(r) => r,
),
);
expect(kept).toEqual([{ type: "PULSE", value: 60 }]);
const recorded = tally();
expect(recorded).toEqual({
total: 3,
buckets: [
{
source: "withings",
type: "PULSE",
direction: "above_max",
count: 2,
},
{
source: "withings",
type: "WEIGHT",
direction: "below_min",
count: 1,
},
],
});
});
it("keeps the refused readings out of the log line", () => {
withEvent(() =>
dropImplausibleMeasurements(
"withings",
[{ type: "PULSE", value: 55643821.505 }],
(r) => r,
),
);
// The value is a health reading. The tally says which metric, which
// provider and which edge — never the number itself.
const serialised = JSON.stringify(event.toJSON());
expect(serialised).not.toContain("55643821");
});
it("lifts the event to warn once per provider", () => {
withEvent(() => {
dropImplausibleMeasurements(
"withings",
[{ type: "PULSE", value: 900 }],
(r) => r,
);
dropImplausibleMeasurements(
"withings",
[{ type: "PULSE", value: 901 }],
(r) => r,
);
dropImplausibleMeasurements(
"fitbit",
[{ type: "PULSE", value: 902 }],
(r) => r,
);
});
expect(event.getLevel()).toBe("warn");
expect(event.toJSON().warnings).toHaveLength(2);
expect(event.toJSON().warnings?.[0]).toContain("withings");
expect(event.toJSON().warnings?.[1]).toContain("fitbit");
});
it("accumulates across calls instead of overwriting the previous tally", () => {
// Fitbit gates once per resource leg, and the reconciler gates once per
// row: a tally that replaced rather than merged would report only the
// last batch, and a run that dropped hundreds would read as dropping one.
withEvent(() => {
dropImplausibleMeasurements(
"fitbit",
[{ type: "PULSE", value: 900 }],
(r) => r,
);
dropImplausibleMeasurements(
"fitbit",
[
{ type: "PULSE", value: 901 },
{ type: "OXYGEN_SATURATION", value: 4 },
],
(r) => r,
);
});
expect(tally()).toEqual({
total: 3,
buckets: [
{
source: "fitbit",
type: "OXYGEN_SATURATION",
direction: "below_min",
count: 1,
},
{ source: "fitbit", type: "PULSE", direction: "above_max", count: 2 },
],
});
});
it("still drops outside a wide-event context", () => {
// A sync running without an ambient event has nowhere to report. The row
// is refused anyway: observability is what is lost, never the guarantee.
const kept = dropImplausibleMeasurements(
"nightscout",
[
{ type: "BLOOD_GLUCOSE", value: 110 },
{ type: "BLOOD_GLUCOSE", value: 9_000 },
],
(r) => r,
);
expect(kept).toEqual([{ type: "BLOOD_GLUCOSE", value: 110 }]);
});
});
describe("recordRangeRejections", () => {
it("writes nothing when there is nothing to report", () => {
withEvent(() => recordRangeRejections("withings", []));
expect(tally()).toBeUndefined();
expect(event.getLevel()).toBe("info");
});
});
@@ -1,6 +1,9 @@
import { describe, expect, it, vi } from "vitest";
import type { Prisma } from "@/generated/prisma/client";
import { eventStorage } from "@/lib/logging/context";
import { WideEventBuilder } from "@/lib/logging/event-builder";
import { RANGE_REJECTED_META_KEY } from "../plausibility-gate";
import {
reconcileExternalMeasurement,
type ExternalMeasurementWrite,
@@ -383,3 +386,104 @@ describe("reconcileExternalMeasurement", () => {
);
});
});
/**
* The reconciler is the shared write seam for every provider that does not
* hold Prisma itself — Polar, Oura, WHOOP, the Withings sleep leg, the Apple
* export importer, the batch endpoint. Gating here rather than at each caller
* is what makes a provider added tomorrow inherit the refusal instead of
* having to remember a check it never knew about.
*/
describe("reconcileExternalMeasurement — plausibility gate", () => {
it("reconciles a value inside the metric's declared band", async () => {
const { tx, measurement } = transaction([]);
measurement.create.mockResolvedValue(row({ id: "fresh" }));
// PULSE is declared 20–300.
const verdict = await reconcileExternalMeasurement(
tx,
desired({ type: "PULSE", value: 112, unit: "bpm", sleepStage: null }),
);
expect(verdict.status).toBe("inserted");
expect(measurement.create).toHaveBeenCalledTimes(1);
});
it("refuses an out-of-band value without touching the transaction", async () => {
const { tx, measurement } = transaction([]);
const verdict = await reconcileExternalMeasurement(
tx,
desired({
type: "PULSE",
value: 55643821.505,
unit: "bpm",
sleepStage: null,
}),
);
expect(verdict).toEqual({
status: "rejected_range",
rejection: { type: "PULSE", direction: "above_max" },
});
// No row written, no row updated, and no savepoint opened — the refusal
// happens before the reconciler does any work at all.
expect(measurement.create).not.toHaveBeenCalled();
expect(measurement.update).not.toHaveBeenCalled();
expect(tx.$executeRawUnsafe).not.toHaveBeenCalled();
});
it("refuses rather than overwriting a good stored row", async () => {
// The dangerous shape: a live row already holds a real reading, and the
// provider re-sends the same identity carrying an impossible value. The
// update branch would have rewritten a good value with a bad one.
const existing = row({
id: "exact-row",
type: "PULSE",
value: 62,
unit: "bpm",
sleepStage: null,
measuredAt: new Date("2026-07-20T06:30:00.000Z"),
});
const { tx, measurement } = transaction([existing]);
const verdict = await reconcileExternalMeasurement(
tx,
desired({
type: "PULSE",
value: 55643821.505,
unit: "bpm",
sleepStage: null,
}),
);
expect(verdict.status).toBe("rejected_range");
expect(measurement.update).not.toHaveBeenCalled();
});
it("tallies the refusal on the ambient wide event under the provider", async () => {
const { tx } = transaction([]);
const event = new WideEventBuilder("background");
await eventStorage.run(event, () =>
reconcileExternalMeasurement(
tx,
desired({
type: "PULSE",
value: 55643821.505,
unit: "bpm",
source: "WHOOP",
sleepStage: null,
}),
),
);
expect(event.toJSON().meta?.[RANGE_REJECTED_META_KEY]).toEqual({
total: 1,
buckets: [
{ source: "whoop", type: "PULSE", direction: "above_max", count: 1 },
],
});
expect(event.getLevel()).toBe("warn");
});
});
@@ -720,6 +720,16 @@ export async function streamParseExportXml(
),
);
// The reconciler carries the same plausibility gate the block above
// already applied, so this arm means the two disagreed. Tally it under
// the same counter the earlier guard uses and count no row.
if (verdict.status === "rejected_range") {
unknown[`${type}::aggregate_out_of_range`] =
(unknown[`${type}::aggregate_out_of_range`] ?? 0) + 1;
stat.durationMs += Date.now() - rowStart;
continue;
}
cumulativeEstimatedDays.add(dayKey);
cumulativeEstimatedRows += 1;
if (verdict.status === "inserted") {
+192
View File
@@ -0,0 +1,192 @@
/**
* The ingest-side half of the app's plausibility domain.
*
* `VALUE_RANGES` in `@/lib/validations/measurement` is the band the
* application declares a metric can occupy — PULSE 20–300, WEIGHT 1–500,
* SLEEP_DURATION 0–1440 minutes. Every path a person can drive enforces it:
* the measurement routes, the batch endpoint, the CSV and Apple-export
* importers, MCP, Telegram. The provider sync writers did not. They took the
* number the remote handed over and wrote it, so a decode slip or a glitched
* device landed a row the application itself calls impossible, and every mean,
* median and personal band downstream digested it as a reading.
*
* `isPlausibleMetricValue` (in `value-domain.ts`) already answers the question
* for the READ side, which is where the damage surfaced first. This module is
* the same question asked one layer earlier, at the point of writing, so the
* row never lands. The two ends share the one predicate on purpose: a band the
* reader hides but the writer accepts is a row that exists and cannot be seen,
* which is the worse of the two failures.
*
* ## Disposition: drop, and say so
*
* An out-of-band incoming value is DROPPED. Not clamped (a clamp invents a
* reading nobody took), not stored-and-flagged (a quarantined row is still a
* row, and every consumer would need to learn the flag), not fatal to the
* batch (one glitched sample must not strand the sync's other readings behind
* a held watermark). Absence reads as absence: the sample is simply not part
* of the record.
*
* A silent drop, though, is indistinguishable from a provider that sent
* nothing — so every drop is tallied onto the ambient wide event under
* {@link RANGE_REJECTED_META_KEY}, with the metric type, the provider, and the
* direction it broke the band in. Counts only. The value itself is a health
* reading and never enters the log line; the direction plus the metric's own
* declared bound is what a diagnosis actually needs.
*
* ## Why meta and not `action`
*
* A wide event holds exactly ONE action, and `setAction` replaces it. A drop
* happens in the middle of a sync whose action is decided at the end, so an
* `action` set here is either clobbered a moment later (the drop vanishes) or,
* on the paths that annotate no terminal action, relabels an otherwise
* successful sync as a rejection. Meta accumulates, survives every later
* `setAction`, and is the shape a dashboard can sum. One warning per provider
* per event rides along so the drop also lifts the event's level to `warn`
* rather than hiding inside a green line.
*/
import { annotate, getEvent } from "@/lib/logging/context";
import { isPlausibleMetricValue } from "@/lib/measurements/value-domain";
import { VALUE_RANGES } from "@/lib/validations/measurement";
/**
* Stable meta key the tally lands under. Pinned so dashboards can count it
* across providers without knowing which one wrote the event.
*/
export const RANGE_REJECTED_META_KEY = "measurement.range_rejected";
/** Which edge of the declared band the value broke, or that it was not a number. */
export type RangeRejectionDirection = "below_min" | "above_max" | "not_finite";
/** One refused reading, reduced to the facts that carry no health value. */
export interface MeasurementRangeRejection {
/** `MeasurementType` of the refused reading. */
type: string;
direction: RangeRejectionDirection;
}
/** One (provider, metric, direction) cell of the event's running tally. */
export interface RangeRejectionBucket {
source: string;
type: string;
direction: RangeRejectionDirection;
count: number;
}
/** The pinned meta shape written under {@link RANGE_REJECTED_META_KEY}. */
export interface RangeRejectionTally {
total: number;
buckets: RangeRejectionBucket[];
}
/**
* Classify a candidate write against the metric's declared band.
*
* Returns `null` when the value may be written. A type with no declared range
* passes — an absent range is an absent fact, never a guessed one, and the
* same asymmetry the read side uses (`isPlausibleMetricValue`). A non-finite
* value never passes, whatever the type: NaN is not an input to anything.
*/
export function classifyRangeRejection(
type: string,
value: number,
): MeasurementRangeRejection | null {
if (isPlausibleMetricValue(type, value)) return null;
if (!Number.isFinite(value)) return { type, direction: "not_finite" };
const range = VALUE_RANGES[type];
// `isPlausibleMetricValue` only answers false for a finite value when the
// type HAS a range, so the lookup cannot miss here.
return {
type,
direction: value < range!.min ? "below_min" : "above_max",
};
}
function readTally(): RangeRejectionTally {
const existing = getEvent()?.toJSON().meta?.[RANGE_REJECTED_META_KEY];
if (
existing !== null &&
typeof existing === "object" &&
Array.isArray((existing as RangeRejectionTally).buckets)
) {
return existing as RangeRejectionTally;
}
return { total: 0, buckets: [] };
}
/**
* Fold refusals onto the ambient wide event.
*
* Merges with whatever the event already carries, so a provider that gates
* several batches (or reconciles row by row) accumulates instead of
* overwriting. Outside a wide-event context this is a no-op — the caller still
* drops the row, it simply has nowhere to report.
*/
export function recordRangeRejections(
source: string,
rejections: readonly MeasurementRangeRejection[],
): void {
if (rejections.length === 0) return;
const evt = getEvent();
if (!evt) return;
const tally = readTally();
const firstForSource = !tally.buckets.some((b) => b.source === source);
for (const rejection of rejections) {
const bucket = tally.buckets.find(
(b) =>
b.source === source &&
b.type === rejection.type &&
b.direction === rejection.direction,
);
if (bucket) bucket.count += 1;
else
tally.buckets.push({
source,
type: rejection.type,
direction: rejection.direction,
count: 1,
});
tally.total += 1;
}
tally.buckets.sort(
(a, b) =>
a.source.localeCompare(b.source) ||
a.type.localeCompare(b.type) ||
a.direction.localeCompare(b.direction),
);
annotate({ meta: { [RANGE_REJECTED_META_KEY]: tally } });
if (firstForSource) {
evt.addWarning(
`${source}: refused reading(s) outside the declared plausibility range — ` +
`dropped, see meta.${RANGE_REJECTED_META_KEY}`,
);
}
}
/**
* Keep the rows whose value sits inside the metric's declared band; tally and
* drop the rest.
*
* `select` exists because the sync writers carry their own row shapes and none
* of them is worth reshaping at the call site: the gate reads the two fields
* it needs and returns the caller's own objects untouched.
*/
export function dropImplausibleMeasurements<T>(
source: string,
rows: readonly T[],
select: (row: T) => { type: string; value: number },
): T[] {
const kept: T[] = [];
const rejections: MeasurementRangeRejection[] = [];
for (const row of rows) {
const { type, value } = select(row);
const rejection = classifyRangeRejection(type, value);
if (rejection) rejections.push(rejection);
else kept.push(row);
}
recordRangeRejections(source, rejections);
return kept;
}
@@ -4,6 +4,11 @@ import {
type MeasurementAggregationProvenance,
type Prisma as PrismaTypes,
} from "@/generated/prisma/client";
import {
classifyRangeRejection,
recordRangeRejections,
type MeasurementRangeRejection,
} from "@/lib/measurements/plausibility-gate";
const SAVEPOINT = "measurement_identity_reconcile";
@@ -50,6 +55,18 @@ export type MeasurementReconciliationVerdict =
| {
status: "failed";
error: { message: string; code?: string };
}
| {
/**
* The incoming value sits outside the metric's declared plausibility
* band, so nothing was written and no existing row was touched. Not a
* failure: the write was refused on purpose and the caller must carry
* on with the rest of its batch. Every caller narrows on this status
* because it carries no `row` — the compiler is what makes a future
* provider handle it rather than dereference a row that does not exist.
*/
status: "rejected_range";
rejection: MeasurementRangeRejection;
};
export type FailedMeasurementReconciliation = Extract<
@@ -219,12 +236,26 @@ async function retireCollision(
* advisory lock serializes cooperative writers sharing either identity, and a
* savepoint turns a non-benign database error into a structured `failed`
* verdict without poisoning the caller's surrounding transaction.
*
* This is also the plausibility gate for every provider that reconciles rather
* than writing Prisma directly. Gating HERE rather than at each caller is the
* point: a provider added tomorrow inherits the refusal by construction, and
* cannot land an impossible reading by forgetting a check it never knew about.
*/
export async function reconcileExternalMeasurement(
tx: PrismaTypes.TransactionClient,
desired: ExternalMeasurementWrite,
options: MeasurementReconciliationOptions = {},
): Promise<MeasurementReconciliationVerdict> {
const rejection = classifyRangeRejection(desired.type, desired.value);
if (rejection) {
recordRangeRejections(
desired.source ? String(desired.source).toLowerCase() : "external",
[rejection],
);
return { status: "rejected_range", rejection };
}
await tx.$executeRawUnsafe(`SAVEPOINT ${SAVEPOINT}`);
try {
@@ -0,0 +1,115 @@
/**
* The Nightscout pull writes BLOOD_GLUCOSE rows directly, so it does not
* inherit the shared reconciler's gate. It is also first-write-wins: a row
* that lands here is immutable, so an impossible sample never corrects itself
* on a later pass. These cases pin the refusal and its annotation.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
const {
upsertMock,
findManyMock,
emitArrivalsMock,
recomputeMock,
invalidateMock,
} = vi.hoisted(() => ({
upsertMock: vi.fn(),
findManyMock: vi.fn(),
emitArrivalsMock: vi.fn(),
recomputeMock: vi.fn(),
invalidateMock: vi.fn(),
}));
vi.mock("@/lib/db", () => ({
prisma: { measurement: { upsert: upsertMock, findMany: findManyMock } },
}));
vi.mock("@/lib/arrivals/measurement-emit", () => ({
emitInsertedMeasurementArrivals: emitArrivalsMock,
}));
vi.mock("@/lib/rollups/measurement-rollups", () => ({
collapseToTypeDayKeys: (rows: Array<{ type: string; measuredAt: Date }>) =>
rows.map((r) => ({ type: r.type, measuredAt: r.measuredAt })),
recomputeBucketsForMeasurement: recomputeMock,
}));
vi.mock("@/lib/insights/comprehensive-generate", () => ({
invalidateStatusInsightsForTypes: invalidateMock,
}));
import { eventStorage } from "@/lib/logging/context";
import { WideEventBuilder } from "@/lib/logging/event-builder";
import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate";
import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate";
import { upsertNightscoutEntries } from "../sync";
/** BLOOD_GLUCOSE is declared 20–800 mg/dL. */
const REAL_ENTRY = { id: "abc", sgv: 112, date: 1718000000000 };
/** A sensor fault, or an mg/dL field carrying something else entirely. */
const IMPOSSIBLE_ENTRY = { id: "def", sgv: 41_600, date: 1718000300000 };
let event: WideEventBuilder;
function tally(): RangeRejectionTally | undefined {
return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as
RangeRejectionTally | undefined;
}
beforeEach(() => {
vi.clearAllMocks();
findManyMock.mockResolvedValue([]);
upsertMock.mockResolvedValue({ id: "row-1" });
emitArrivalsMock.mockResolvedValue(undefined);
recomputeMock.mockResolvedValue(undefined);
invalidateMock.mockResolvedValue(undefined);
event = new WideEventBuilder("background");
});
describe("upsertNightscoutEntries — plausibility gate", () => {
it("writes a reading inside the metric's declared band", async () => {
const result = await eventStorage.run(event, () =>
upsertNightscoutEntries("user-1", [REAL_ENTRY]),
);
expect(result).toEqual({ inserted: 1, failedRows: 0 });
expect(upsertMock).toHaveBeenCalledTimes(1);
expect(tally()).toBeUndefined();
});
it("refuses an out-of-band reading and annotates the drop", async () => {
const result = await eventStorage.run(event, () =>
upsertNightscoutEntries("user-1", [REAL_ENTRY, IMPOSSIBLE_ENTRY]),
);
expect(result).toEqual({ inserted: 1, failedRows: 0 });
expect(upsertMock).toHaveBeenCalledTimes(1);
expect(
(upsertMock.mock.calls[0]![0] as { create: { value: number } }).create
.value,
).toBe(112);
expect(tally()).toEqual({
total: 1,
buckets: [
{
source: "nightscout",
type: "BLOOD_GLUCOSE",
direction: "above_max",
count: 1,
},
],
});
expect(event.getLevel()).toBe("warn");
});
it("writes nothing when the whole page is out of band", async () => {
const result = await eventStorage.run(event, () =>
upsertNightscoutEntries("user-1", [IMPOSSIBLE_ENTRY]),
);
expect(result).toEqual({ inserted: 0, failedRows: 0 });
expect(upsertMock).not.toHaveBeenCalled();
expect(tally()?.total).toBe(1);
});
});
+11 -1
View File
@@ -22,6 +22,7 @@
import { prisma } from "@/lib/db";
import type { MeasurementType } from "@/generated/prisma/client";
import { getEvent } from "@/lib/logging/context";
import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate";
import {
markSyncFailureRecorded,
recordSyncFailure,
@@ -221,7 +222,16 @@ export async function upsertNightscoutEntries(
if (entries.length === 0) return { inserted: 0, failedRows: 0 };
const type = "BLOOD_GLUCOSE" as MeasurementType;
const mapped = entries.map((entry) => mapSgvEntryToMeasurement(entry));
// A CGM that reports a glucose the app's own band calls impossible is
// reporting a sensor fault or a unit slip, not a reading. Drop it: the row
// would otherwise be immutable (first-write-wins), so an impossible sample
// that lands here never corrects itself on a later pass.
const mapped = dropImplausibleMeasurements(
"nightscout",
entries.map((entry) => mapSgvEntryToMeasurement(entry)),
(m) => ({ type, value: m.value }),
);
if (mapped.length === 0) return { inserted: 0, failedRows: 0 };
// Probe which keys already exist so a re-confirmed reading (the upsert's
// no-op update branch) is not miscounted as a fresh import and the arrival
+15 -2
View File
@@ -44,7 +44,13 @@ listLabResultsSchema.meta({
const rangeStatusEnum = z.enum(["in-range", "below", "above", "unknown"]).meta({
id: "LabReferenceRangeStatus",
description:
"Server-computed, NEUTRAL reference-range verdict. `unknown` when the lab reported no usable bounds. Inclusive bounds: a value on the limit reads in-range. The badge that renders this must stay calm and informative — not an alarming red.",
"Server-computed, NEUTRAL reference-range verdict, always against `referenceLow` / `referenceHigh` (the window in force for that reading). `unknown` when no usable bounds are on file. Inclusive bounds: a value on the limit reads in-range. The badge that renders this must stay calm and informative — not an alarming red.",
});
const referenceOriginEnum = z.enum(["source", "catalog", "none"]).meta({
id: "LabReferenceRangeOrigin",
description:
"Which window `referenceLow` / `referenceHigh` came from. `source` = the range printed on this reading's own report, which wins for that reading because it is the window the physician evaluated the value against. `catalog` = the user's Biomarker band, the net for readings with no printed range. `none` = no window on file.",
});
const labResultRow = z
@@ -60,6 +66,13 @@ const labResultRow = z
unit: z.string(),
referenceLow: z.number().nullable(),
referenceHigh: z.number().nullable(),
catalogReferenceLow: z.number().nullable(),
catalogReferenceHigh: z.number().nullable(),
sourceReferenceLow: z.number().nullable(),
sourceReferenceHigh: z.number().nullable(),
sourceReferenceText: z.string().nullable(),
referenceOrigin: referenceOriginEnum,
referenceDivergesFromCatalog: z.boolean(),
takenAt: z.string(),
source: z.string(),
hasNote: z.boolean(),
@@ -70,7 +83,7 @@ const labResultRow = z
.meta({
id: "LabResult",
description:
'A stored lab result. Exactly one of `value` (numeric) or `valueText` (qualitative, e.g. "negativ") is non-null; `rangeStatus` is always `unknown` for a qualitative row (no number to classify). `biomarkerId` links the user-scoped catalog marker (null for legacy free-text rows); when linked, `analyte` / `unit` / `referenceLow` / `referenceHigh` are RESOLVED server-side from the Biomarker — the client renders them and never recomputes the range. The encrypted note is never echoed in list rows; `hasNote` flags its presence and the single-resource GET returns the decrypted `note`.',
"A stored lab result. Exactly one of `value` (numeric) or `valueText` (qualitative, e.g. \"negativ\") is non-null; `rangeStatus` is always `unknown` for a qualitative row (no number to classify). `biomarkerId` links the user-scoped catalog marker (null for legacy free-text rows); when linked, `analyte` / `unit` are RESOLVED server-side from the Biomarker.\n\nThree reference windows ride on the row and the server says which one applies. `referenceLow` / `referenceHigh` are the window IN FORCE and the one `rangeStatus` was computed from — read these to render the range and you are correct without doing any resolution of your own. `sourceReferenceLow` / `sourceReferenceHigh` / `sourceReferenceText` are the range this reading's own report printed (the text verbatim, kept even when no numbers could be read out of it). `catalogReferenceLow` / `catalogReferenceHigh` are the Biomarker band. `referenceOrigin` names the winner and `referenceDivergesFromCatalog` is true when the report's window and the catalog band state different limits — a surface showing only one of the two is showing a partial answer. The client renders these values and never recomputes the range. The encrypted note is never echoed in list rows; `hasNote` flags its presence and the single-resource GET returns the decrypted `note`.",
});
const labResultDetail = labResultRow
+16 -1
View File
@@ -53,6 +53,10 @@ const extractedRow = z
unit: z.string().nullable(),
referenceLow: z.number().nullable(),
referenceHigh: z.number().nullable(),
referenceText: z.string().nullable().meta({
description:
'The reference range EXACTLY as printed on the report, verbatim ("3,5 - 5,0", "< 116", "bis 5,0", "negativ"). `referenceLow` / `referenceHigh` carry the same window reduced to numbers WHEN it reduces to numbers; this field carries it either way, so a window stated in words is not lost between the report and the reading.',
}),
takenAt: z.string().nullable(),
confidence: extractConfidence,
biomarkerMatch: z.enum(["new", "existing"]),
@@ -87,6 +91,13 @@ const committedRow = z
unit: z.string(),
referenceLow: z.number().nullable(),
referenceHigh: z.number().nullable(),
catalogReferenceLow: z.number().nullable(),
catalogReferenceHigh: z.number().nullable(),
sourceReferenceLow: z.number().nullable(),
sourceReferenceHigh: z.number().nullable(),
sourceReferenceText: z.string().nullable(),
referenceOrigin: z.enum(["source", "catalog", "none"]),
referenceDivergesFromCatalog: z.boolean(),
takenAt: z.string(),
source: z.string(),
hasNote: z.boolean(),
@@ -94,7 +105,11 @@ const committedRow = z
createdAt: z.string(),
updatedAt: z.string(),
})
.meta({ id: "OcrCommittedLabResult" });
.meta({
id: "OcrCommittedLabResult",
description:
"A written reading. Its reference window resolves exactly as it does on `/api/labs` — see `LabResult`.",
});
const commitResponse = z
.object({
+4
View File
@@ -575,6 +575,10 @@ export async function upsertOuraMeasurements(
for (let index = 0; index < readings.length; index++) {
const reading = readings[index]!;
const verdict = verdicts[index]!;
// Refused by the plausibility gate: nothing was written and nothing was
// touched, so the reading counts as neither imported nor a rollup dirt
// mark. The drop is already on the wide event.
if (verdict.status === "rejected_range") continue;
for (const dirty of verdict.dirtyIdentities ?? []) {
touched.push(dirty);
}
+4
View File
@@ -364,6 +364,10 @@ export async function upsertPolarMeasurements(
for (let index = 0; index < readings.length; index++) {
const reading = readings[index]!;
const verdict = verdicts[index]!;
// Refused by the plausibility gate: nothing was written and nothing was
// touched, so the reading counts as neither imported nor a rollup dirt
// mark. The drop is already on the wide event.
if (verdict.status === "rejected_range") continue;
for (const dirty of verdict.dirtyIdentities ?? []) {
touched.push(dirty);
}
+23
View File
@@ -255,11 +255,29 @@ const moodEntrySchema = z
mood: z.string().min(1),
score: z.number().int().min(0).max(10),
tags: moodEntryTagsSchema,
// The free text, in both storage shapes. A portable export carries the
// decrypted `note`; a disaster-recovery file carries `noteEncrypted` plus
// whatever legacy plaintext the row still holds. Same pair, and same
// handling on restore, as `Measurement.notes` / `notesEncrypted`.
note: z.string().nullable().optional(),
noteEncrypted: base64BytesSchema.nullable().optional(),
source: z.string().min(1).optional(),
externalId: z.string().nullable().optional(),
loggedAt: isoDateTime,
// The IANA zone the `date` string is anchored to. Absent means the legacy
// Europe/Berlin reading, which is what a row written before v1.4.25 means
// by a NULL here — so an old file keeps its old meaning exactly.
tz: z.string().nullable().optional(),
syncedAt: isoDateTime.optional(),
syncVersion: z.number().int().optional(),
deletedAt: isoDateTime.nullable().optional(),
createdAt: isoDateTime.optional(),
updatedAt: isoDateTime.optional(),
factors: z.array(moodFactorSchema).default([]),
// BINARY structured-tag keys. Separate from `factors`, which keeps its
// RATED-only meaning, so every file written before this one parses
// unchanged and restores identically: absent reads as none.
structuredTags: z.array(z.string().min(1)).default([]),
})
.passthrough();
@@ -690,6 +708,11 @@ const labResultBackupSchema = z
unit: z.string().min(1),
referenceLow: z.number().nullable().optional(),
referenceHigh: z.number().nullable().optional(),
// Optional so a backup written before these columns existed still
// restores; absent reads as "this reading had no source window on file".
sourceReferenceLow: z.number().nullable().optional(),
sourceReferenceHigh: z.number().nullable().optional(),
sourceReferenceText: z.string().nullable().optional(),
takenAt: isoDateTime,
source: z.string().min(1),
biomarkerName: z.string().nullable().optional(),
+18
View File
@@ -167,6 +167,15 @@ export const extractedFactRawSchema = z.object({
unit: nullableText(80),
referenceLow: z.number().finite().nullable().catch(null),
referenceHigh: z.number().finite().nullable().catch(null),
/**
* Observation: the reference range EXACTLY as printed, verbatim. A report
* writes its window in prose ("3,5 - 5,0", "< 5", "bis 5,0", "negativ"), and
* `referenceLow` / `referenceHigh` above can only carry the cases that
* reduce to two numbers. This field is what stops every other case from
* being dropped on the floor — the string is kept whether or not the parser
* gets bounds out of it.
*/
referenceText: nullableText(120),
/** MedicationStatement: dose + stated status (verbatim). */
dose: nullableText(120),
medicationStatus: nullableText(64),
@@ -237,6 +246,14 @@ export interface ObservationFactData {
unit: string | null;
referenceLow: number | null;
referenceHigh: number | null;
/**
* The reference range as printed on the report, verbatim.
*
* Optional because it genuinely is: a fact staged before this field existed
* carries no key for it, and that is an honest absence rather than a shim
* for a caller we do not ship. Read it as `?? null`.
*/
referenceText?: string | null;
effectiveDate: string | null;
}
@@ -379,6 +396,7 @@ const observationEditSchema = z
unit: optText(80),
referenceLow: z.number().finite().nullable().optional(),
referenceHigh: z.number().finite().nullable().optional(),
referenceText: optText(120),
effectiveDate: isoDate,
})
// Numeric XOR qualitative, mirroring the lab-result discipline. A null/absent
+20
View File
@@ -78,6 +78,13 @@ export const extractedRowSchema = z.object({
unit: z.string().trim().max(80).nullable().catch(null),
referenceLow: z.number().finite().nullable().catch(null),
referenceHigh: z.number().finite().nullable().catch(null),
/**
* The reference range EXACTLY as printed, verbatim. The two numeric bounds
* above can only carry a window that reduces to two numbers; a report that
* prints "bis 5,0" or "negativ" states a window all the same, and this is
* what keeps it instead of dropping it.
*/
referenceText: z.string().trim().max(120).nullable().catch(null),
takenAt: z.string().nullable().catch(null),
confidence: extractedConfidenceSchema,
});
@@ -107,6 +114,8 @@ export interface OcrExtractedRowDto {
unit: string | null;
referenceLow: number | null;
referenceHigh: number | null;
/** The reference range as printed on the report, verbatim. */
referenceText: string | null;
takenAt: string | null;
confidence: {
analyte: number;
@@ -150,6 +159,17 @@ export const ocrCommitRowSchema = z
unit: z.string().trim().min(1).max(40).optional(),
referenceLow,
referenceHigh,
/**
* The window the scanned report printed for this row. Confirmed by the
* human on the review screen alongside everything else, and written onto
* the reading rather than only used to seed a new catalog marker.
*/
referenceText: z
.string()
.trim()
.max(120)
.optional()
.or(z.literal("").transform(() => undefined)),
takenAt: takenAtField,
})
// Numeric XOR qualitative: exactly one of value / valueText.
+60
View File
@@ -14,6 +14,7 @@
*/
import { z } from "zod/v4";
import { SOURCE_REFERENCE_TEXT_MAX } from "@/lib/labs/parse-reference-range";
import { validateEntryInstant } from "@/lib/validations/entry-instant";
/**
@@ -63,6 +64,29 @@ const optionalNote = z
const referenceLow = z.number().finite().optional();
const referenceHigh = z.number().finite().optional();
/**
* The reference range the SOURCE REPORT printed for this one reading.
*
* Distinct from `referenceLow` / `referenceHigh` above, which seed (and are
* stamped from) the catalog marker's band. A lab runs its own method on its
* own device, so the window a physician evaluates against is the one on that
* report; these fields carry it, and `resolveEffectiveReferenceRange` lets it
* win for that reading alone.
*
* `sourceReferenceText` is what was printed, verbatim. It is accepted with or
* without derivable bounds: a window stated as "negativ" or "siehe Befund"
* still belongs on the record, and dropping it because no number could be read
* out of it is exactly the loss this field exists to stop.
*/
const sourceReferenceLow = z.number().finite().optional();
const sourceReferenceHigh = z.number().finite().optional();
const sourceReferenceText = z
.string()
.trim()
.max(SOURCE_REFERENCE_TEXT_MAX)
.optional()
.or(z.literal("").transform(() => undefined));
/**
* v1.18.1 — optional link to a user-scoped `Biomarker` (the catalog row).
* When set, the structured-entry path resolves the unit + reference range
@@ -101,6 +125,9 @@ export const createLabResultSchema = z
unit: requiredText(40).optional(),
referenceLow,
referenceHigh,
sourceReferenceLow,
sourceReferenceHigh,
sourceReferenceText,
takenAt: takenAtField,
note: optionalNote,
source,
@@ -142,6 +169,19 @@ export const createLabResultSchema = z
message: "referenceLow must not exceed referenceHigh",
path: ["referenceLow"],
},
)
// Same guard on the source window. A transposed window is a client error
// here for the same reason: nothing downstream can tell it from a real one.
.refine(
(d) =>
d.valueText !== undefined ||
d.sourceReferenceLow === undefined ||
d.sourceReferenceHigh === undefined ||
d.sourceReferenceLow <= d.sourceReferenceHigh,
{
message: "sourceReferenceLow must not exceed sourceReferenceHigh",
path: ["sourceReferenceLow"],
},
);
export type CreateLabResultInput = z.infer<typeof createLabResultSchema>;
@@ -164,6 +204,9 @@ export const updateLabResultSchema = z
unit: requiredText(40).optional(),
referenceLow: referenceLow.or(z.null()),
referenceHigh: referenceHigh.or(z.null()),
sourceReferenceLow: sourceReferenceLow.or(z.null()),
sourceReferenceHigh: sourceReferenceHigh.or(z.null()),
sourceReferenceText: sourceReferenceText.or(z.null()),
takenAt: takenAtField.optional(),
note: optionalNote.or(z.null()),
})
@@ -183,6 +226,18 @@ export const updateLabResultSchema = z
message: "referenceLow must not exceed referenceHigh",
path: ["referenceLow"],
},
)
.refine(
(d) =>
d.sourceReferenceLow === undefined ||
d.sourceReferenceLow === null ||
d.sourceReferenceHigh === undefined ||
d.sourceReferenceHigh === null ||
d.sourceReferenceLow <= d.sourceReferenceHigh,
{
message: "sourceReferenceLow must not exceed sourceReferenceHigh",
path: ["sourceReferenceLow"],
},
);
export type UpdateLabResultInput = z.infer<typeof updateLabResultSchema>;
@@ -246,7 +301,12 @@ export type ListLabResultsInput = z.infer<typeof listLabResultsSchema>;
* implementation. Re-exported here for the existing import sites.
*/
export {
classifyAgainstEffectiveRange,
classifyReferenceRange,
formatReferenceRange,
resolveEffectiveReferenceRange,
type EffectiveReferenceRange,
type ReferenceRangeOrigin,
type ReferenceRangeStatus,
type SourceReferenceRange,
} from "@/lib/labs/reference-range";
+4
View File
@@ -480,6 +480,10 @@ export async function upsertWhoopMeasurements(
for (let index = chunkStart; index < chunkEnd; index++) {
const reading = readings[index]!;
const verdict = verdicts[index - chunkStart]!;
// Refused by the plausibility gate: nothing was written and nothing was
// touched, so the reading counts as neither imported nor a rollup dirt
// mark. The drop is already on the wide event.
if (verdict.status === "rejected_range") continue;
for (const dirty of verdict.dirtyIdentities ?? []) {
touched.push(dirty);
touchedTypes.add(dirty.type);
@@ -0,0 +1,261 @@
/**
* Withings writes the measurement table directly — the measures leg row by
* row, the activity leg as a planned batch — so neither inherits the shared
* reconciler's gate. These cases pin that both legs now clear the app's own
* declared plausibility band before anything reaches Prisma, and that a
* refusal lands on the wide event rather than vanishing.
*
* The measures leg matters twice over: it does not only insert, it OVERWRITES
* the stored value of an existing slot, so an ungated impossible reading could
* replace a real one.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("@/lib/db", () => ({
prisma: {
measurement: {
findFirst: vi.fn(),
findMany: vi.fn(),
create: vi.fn(),
createMany: vi.fn(),
update: vi.fn(),
},
withingsConnection: {
findUnique: vi.fn(),
update: vi.fn(),
},
$queryRaw: vi.fn(),
$transaction: vi.fn((ops: unknown) =>
Array.isArray(ops) ? Promise.all(ops) : Promise.resolve(),
),
},
}));
vi.mock("@/lib/crypto", () => ({
decrypt: vi.fn((v: string) => v),
encrypt: vi.fn((v: string) => v),
}));
vi.mock("../client", async () => {
const actual = await vi.importActual<typeof import("../client")>("../client");
return {
...actual,
fetchMeasurements: vi.fn(),
refreshAccessToken: vi.fn(),
subscribeWebhook: vi.fn(),
};
});
vi.mock("../credentials", () => ({
getUserWithingsCredentials: vi.fn(async () => ({
clientId: "cid",
clientSecret: "secret",
})),
}));
vi.mock("@/lib/integrations/status", () => ({
isReauthRequired: vi.fn().mockResolvedValue(false),
parkIntegrationAtReauth: vi.fn(),
recordSyncFailure: vi.fn().mockResolvedValue(undefined),
recordSyncSuccess: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/rollups/measurement-rollups", async () => {
const actual = await vi.importActual<
typeof import("@/lib/rollups/measurement-rollups")
>("@/lib/rollups/measurement-rollups");
return {
...actual,
recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined),
};
});
vi.mock("@/lib/insights/comprehensive-generate", () => ({
invalidateStatusInsightsForTypes: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/cache/invalidate", () => ({
invalidateUserDashboardSnapshot: vi.fn(),
}));
import { prisma } from "@/lib/db";
import { eventStorage } from "@/lib/logging/context";
import { WideEventBuilder } from "@/lib/logging/event-builder";
import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate";
import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate";
import { fetchMeasurements } from "../client";
import { syncUserActivity } from "../sync-activity";
import { syncUserMeasurements } from "../sync";
let event: WideEventBuilder;
function tally(): RangeRejectionTally | undefined {
return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as
RangeRejectionTally | undefined;
}
function installActivityFetch(
entries: Array<{ date: string; steps?: number; distance?: number }>,
) {
vi.stubGlobal(
"fetch",
vi.fn(async () => ({
status: 200,
headers: new Headers(),
json: async () => ({
status: 0,
body: { activities: entries, more: false, offset: 0 },
}),
})),
);
}
beforeEach(() => {
vi.clearAllMocks();
event = new WideEventBuilder("background");
vi.mocked(prisma.withingsConnection.findUnique).mockResolvedValue({
id: "conn-1",
withingsUserId: "wu-1",
accessToken: "enc-access",
refreshToken: "enc-refresh",
scope: "user.metrics,user.activity",
tokenExpiresAt: new Date(Date.now() + 60 * 60 * 1000),
lastSyncedAt: new Date(Date.now() - 60 * 60 * 1000),
} as never);
vi.mocked(prisma.withingsConnection.update).mockResolvedValue({} as never);
vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null);
vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never);
vi.mocked(prisma.measurement.create).mockResolvedValue({
id: "m-1",
type: "PULSE",
measuredAt: new Date(),
} as never);
vi.mocked(prisma.measurement.createMany).mockResolvedValue({
count: 1,
} as never);
vi.mocked(prisma.measurement.update).mockResolvedValue({} as never);
});
afterEach(() => {
vi.unstubAllGlobals();
});
describe("syncUserMeasurements — plausibility gate", () => {
it("writes a reading inside the metric's declared band", async () => {
vi.mocked(fetchMeasurements).mockResolvedValue([
{
type: "PULSE",
value: 112,
measuredAt: new Date("2026-05-16T08:00:00Z"),
},
] as never);
const result = await eventStorage.run(event, () =>
syncUserMeasurements("user-1"),
);
expect(result).toEqual({ imported: 1, failed: false });
expect(prisma.measurement.create).toHaveBeenCalledTimes(1);
expect(tally()).toBeUndefined();
});
it("refuses an out-of-band reading and annotates the drop", async () => {
vi.mocked(fetchMeasurements).mockResolvedValue([
{
type: "PULSE",
value: 112,
measuredAt: new Date("2026-05-16T08:00:00Z"),
},
{
type: "PULSE",
value: 55643821.505,
measuredAt: new Date("2026-05-16T08:05:00Z"),
},
] as never);
const result = await eventStorage.run(event, () =>
syncUserMeasurements("user-1"),
);
// Only the real reading landed, and the run is still a clean success: one
// impossible sample must not strand the rest behind a held watermark.
expect(result).toEqual({ imported: 1, failed: false });
expect(prisma.measurement.create).toHaveBeenCalledTimes(1);
expect(tally()).toEqual({
total: 1,
buckets: [
{ source: "withings", type: "PULSE", direction: "above_max", count: 1 },
],
});
expect(event.getLevel()).toBe("warn");
});
it("does not overwrite a stored reading with an out-of-band one", async () => {
// The slot already holds a real pulse; the provider re-sends the same
// instant carrying a decode slip. Ungated, the update branch rewrote the
// good value with the bad one.
vi.mocked(prisma.measurement.findFirst).mockResolvedValue({
id: "existing-row",
} as never);
vi.mocked(fetchMeasurements).mockResolvedValue([
{
type: "PULSE",
value: 55643821.505,
measuredAt: new Date("2026-05-16T08:00:00Z"),
},
] as never);
const result = await eventStorage.run(event, () =>
syncUserMeasurements("user-1"),
);
expect(result).toEqual({ imported: 0, failed: false });
expect(prisma.measurement.update).not.toHaveBeenCalled();
expect(tally()?.total).toBe(1);
});
});
describe("syncUserActivity — plausibility gate", () => {
it("plans a daily total inside the metric's declared band", async () => {
installActivityFetch([{ date: "2026-05-16", steps: 8_400 }]);
const imported = await eventStorage.run(event, () =>
syncUserActivity("user-1"),
);
expect(imported).toBe(1);
expect(prisma.measurement.createMany).toHaveBeenCalledTimes(1);
expect(tally()).toBeUndefined();
});
it("refuses an out-of-band daily total and annotates the drop", async () => {
// ACTIVITY_STEPS is declared 0–200 000.
installActivityFetch([
{ date: "2026-05-16", steps: 8_400 },
{ date: "2026-05-17", steps: 9_900_000 },
]);
const imported = await eventStorage.run(event, () =>
syncUserActivity("user-1"),
);
expect(imported).toBe(1);
const created = vi.mocked(prisma.measurement.createMany).mock.calls[0]?.[0]
?.data as Array<{ value: number }>;
expect(created).toHaveLength(1);
expect(created[0]!.value).toBe(8_400);
expect(tally()).toEqual({
total: 1,
buckets: [
{
source: "withings",
type: "ACTIVITY_STEPS",
direction: "above_max",
count: 1,
},
],
});
expect(event.getLevel()).toBe("warn");
});
});
+12 -2
View File
@@ -39,6 +39,7 @@ import { prisma } from "@/lib/db";
import { getEvent } from "@/lib/logging/context";
import { safeFetch } from "@/lib/safe-fetch";
import { getUnitForType } from "@/lib/validations/measurement";
import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate";
import {
collapseToTypeDayKeys,
recomputeBucketsForMeasurement,
@@ -342,7 +343,7 @@ export async function syncUserActivity(
value: number;
externalId: string;
}
const planned: PlannedRow[] = [];
const collected: PlannedRow[] = [];
for (const entry of entries) {
if (!entry.date) continue;
const measuredAt = activityMeasuredAt(entry.date);
@@ -352,7 +353,7 @@ export async function syncUserActivity(
// valid (a day of rest still records 0 steps). Only skip
// undefined / null / NaN.
if (raw == null || !Number.isFinite(raw)) continue;
planned.push({
collected.push({
type,
measuredAt,
value: raw,
@@ -361,6 +362,15 @@ export async function syncUserActivity(
}
}
// A daily-activity total the app's own band calls impossible (a step count
// past 200 000, a negative distance) is a provider artefact, not a day. Drop
// it before the plan is read against the table, so it can neither insert a
// fresh impossible slot nor overwrite a good stored one.
const planned = dropImplausibleMeasurements("withings", collected, (p) => ({
type: p.type,
value: p.value,
}));
if (planned.length > 0) {
const slotKey = (type: MeasurementType, measuredAt: Date) =>
`${type}|${measuredAt.getTime()}`;
+6
View File
@@ -474,6 +474,9 @@ export async function syncUserSleep(
if (verdict.status === "failed") {
throw new MeasurementReconciliationError(verdict);
}
// Refused by the plausibility gate: nothing was written and nothing
// was touched. The drop is already on the wide event.
if (verdict.status === "rejected_range") continue;
for (const dirty of verdict.dirtyIdentities ?? []) {
touched.push(dirty);
}
@@ -542,6 +545,9 @@ export async function syncUserSleep(
if (verdict.status === "failed") {
throw new MeasurementReconciliationError(verdict);
}
// Refused by the plausibility gate: nothing was written and
// nothing was touched. The drop is already on the wide event.
if (verdict.status === "rejected_range") continue;
for (const dirty of verdict.dirtyIdentities ?? []) {
touched.push(dirty);
}
+13 -2
View File
@@ -15,6 +15,7 @@ import { getUserWithingsCredentials } from "./credentials";
import { annotate, getEvent } from "@/lib/logging/context";
import { isP2002 } from "@/lib/prisma-errors";
import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit";
import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate";
import {
isReauthRequired,
recordSyncFailure,
@@ -229,14 +230,24 @@ export async function syncUserMeasurements(
)
: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); // 30 days back
let measures: Awaited<ReturnType<typeof fetchMeasurements>>;
let fetched: Awaited<ReturnType<typeof fetchMeasurements>>;
try {
measures = await fetchMeasurements(tokenInfo.accessToken, startDate);
fetched = await fetchMeasurements(tokenInfo.accessToken, startDate);
} catch (err) {
await recordWithingsSyncFailure(userId, err, WITHINGS_LEG_MEASURES);
throw err;
}
// The app's own plausibility band applies to a provider reading exactly as
// it applies to a typed one. A remote that reports a pulse in the millions
// is reporting a decode slip, not a heartbeat, and the row would poison
// every mean and personal band derived from it. Refuse it here, before the
// update branch below can overwrite a good stored value with it.
const measures = dropImplausibleMeasurements("withings", fetched, (m) => ({
type: m.type,
value: m.value,
}));
let imported = 0;
// v1.4.39.1 — track every (type, measuredAt) we touched so the
// persistent rollup tier can be re-folded at the end of the sync.