mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-14 14:40:52 +00:00
Merge branch 'main' into fix/wipe-geo-truths
This commit is contained in:
93 files changed
+5417
-236
No files matched your search
@@ -37,7 +37,7 @@
|
||||
* is always owner-scoped and safe to read), so the block is `null` only when the
|
||||
* account has no recent readings.
|
||||
*/
|
||||
import { classifyReferenceRange } from "@/lib/labs/reference-range";
|
||||
import { classifyAgainstEffectiveRange } from "@/lib/labs/reference-range";
|
||||
import { resolveLabFields } from "@/lib/labs/serialise";
|
||||
import { sanitizeForPrompt } from "@/lib/insights/sanitize";
|
||||
import { prisma } from "@/lib/db";
|
||||
@@ -131,6 +131,9 @@ export async function buildLabsSnapshotBlock(
|
||||
unit: true,
|
||||
referenceLow: true,
|
||||
referenceHigh: true,
|
||||
sourceReferenceLow: true,
|
||||
sourceReferenceHigh: true,
|
||||
sourceReferenceText: true,
|
||||
takenAt: true,
|
||||
biomarkerId: true,
|
||||
biomarker: {
|
||||
@@ -181,15 +184,13 @@ export async function buildLabsSnapshotBlock(
|
||||
referenceLow: resolved.referenceLow,
|
||||
referenceHigh: resolved.referenceHigh,
|
||||
// A qualitative row has nothing to compare against the bounds → neutral
|
||||
// "unknown", never a fabricated verdict. A numeric row classifies.
|
||||
rangeStatus:
|
||||
row.value === null
|
||||
? "unknown"
|
||||
: classifyReferenceRange(
|
||||
row.value,
|
||||
resolved.referenceLow,
|
||||
resolved.referenceHigh,
|
||||
),
|
||||
// "unknown", never a fabricated verdict. A numeric row classifies against
|
||||
// the window in force for it — the report's own range when it carries
|
||||
// one, the catalog band otherwise. Same rule as every other lab surface.
|
||||
rangeStatus: classifyAgainstEffectiveRange(
|
||||
row.value,
|
||||
resolved.effectiveRange,
|
||||
),
|
||||
takenAt: row.takenAt.toISOString(),
|
||||
});
|
||||
|
||||
|
||||
@@ -20,10 +20,18 @@ export function buildClinicalRecordsNotesSection(
|
||||
|
||||
// v1.17.1 — structured lab-results section. Populated when the `labs`
|
||||
// toggle is ON (default) and the user recorded at least one result in the
|
||||
// window. One row per analyte (latest reading), with the lab's reference
|
||||
// range and a NEUTRAL in/out-of-range marker — informative, never an
|
||||
// alarming red. The marker is a quiet glyph (↓ / ↑ / —), not a colour, so
|
||||
// the clinical PDF stays calm.
|
||||
// window. One row per analyte (latest reading), with the reference range and
|
||||
// a NEUTRAL in/out-of-range marker — informative, never an alarming red. The
|
||||
// marker is a quiet glyph (↓ / ↑ / —), not a colour, so the clinical PDF
|
||||
// stays calm.
|
||||
//
|
||||
// The reference column prints the window the reading was JUDGED against, and
|
||||
// when that window came off the source report it prints the report's own
|
||||
// string verbatim — a clinician comparing this table against the original
|
||||
// report should read the same characters. A reading whose report window
|
||||
// differs from the catalog band carries a footnote marker, and the catalog
|
||||
// band is named beneath the table, because "3,9–5,4" and "3,5–5,0" disagree
|
||||
// about the same number and a table that shows only one is a partial answer.
|
||||
if (data.labResults && data.labResults.length > 0) {
|
||||
y = ensureSpace(y, 6 + 18);
|
||||
doc.setFontSize(14);
|
||||
@@ -54,6 +62,23 @@ export function buildClinicalRecordsNotesSection(
|
||||
}
|
||||
};
|
||||
|
||||
// The reference cell: the report's own string when the report stated the
|
||||
// window, else the formatted catalog band. A divergence gets a "*" the
|
||||
// note under the table explains.
|
||||
const referenceCell = (
|
||||
lr: NonNullable<typeof data.labResults>[number],
|
||||
): string => {
|
||||
const printed =
|
||||
lr.referenceOrigin === "source" && lr.sourceReferenceText
|
||||
? lr.sourceReferenceText
|
||||
: rangeText(lr.referenceLow, lr.referenceHigh);
|
||||
return lr.referenceDivergesFromCatalog ? `${printed} *` : printed;
|
||||
};
|
||||
|
||||
const diverging = data.labResults.filter(
|
||||
(lr) => lr.referenceDivergesFromCatalog,
|
||||
);
|
||||
|
||||
const labRows = data.labResults.map((lr) => {
|
||||
// v1.18.9 — a qualitative reading (`value === null`) prints its result
|
||||
// text in the value column and has no numeric range / status glyph.
|
||||
@@ -63,7 +88,7 @@ export function buildClinicalRecordsNotesSection(
|
||||
isQualitative
|
||||
? (lr.valueText ?? "")
|
||||
: `${num(lr.value as number)} ${lr.unit}`.trim(),
|
||||
isQualitative ? "—" : rangeText(lr.referenceLow, lr.referenceHigh),
|
||||
isQualitative ? "—" : referenceCell(lr),
|
||||
isQualitative
|
||||
? ""
|
||||
: statusGlyph(lr.value as number, lr.referenceLow, lr.referenceHigh),
|
||||
@@ -107,6 +132,30 @@ export function buildClinicalRecordsNotesSection(
|
||||
y =
|
||||
(doc as jsPDF & { lastAutoTable: { finalY: number } }).lastAutoTable
|
||||
.finalY + 8;
|
||||
|
||||
// Name every diverging marker's catalog band, so the reader can see both
|
||||
// windows rather than being told only that they differ.
|
||||
if (diverging.length > 0) {
|
||||
y = ensureSpace(y, 4 + diverging.length * 4);
|
||||
doc.setFontSize(8);
|
||||
doc.setFont("helvetica", "normal");
|
||||
doc.setTextColor(110, 110, 110);
|
||||
doc.text(t("doctorReport.labsSourceRangeNote"), margin, y);
|
||||
y += 4;
|
||||
for (const lr of diverging) {
|
||||
doc.text(
|
||||
`* ${lr.analyte}: ${t("doctorReport.labsCatalogRangeLabel")} ${rangeText(
|
||||
lr.catalogReferenceLow,
|
||||
lr.catalogReferenceHigh,
|
||||
)} ${lr.unit}`.trim(),
|
||||
margin,
|
||||
y,
|
||||
);
|
||||
y += 4;
|
||||
}
|
||||
doc.setTextColor(30, 30, 30);
|
||||
y += 4;
|
||||
}
|
||||
}
|
||||
|
||||
// v1.18.1 P4 — illness / condition episodes overlapping the window. Present
|
||||
|
||||
@@ -241,8 +241,23 @@ export interface DoctorReportData {
|
||||
*/
|
||||
valueText: string | null;
|
||||
unit: string;
|
||||
/**
|
||||
* The reference window this reading is judged against: the range printed
|
||||
* on its own report when it carries one, the catalog band otherwise.
|
||||
* Resolved server-side through `resolveEffectiveReferenceRange` so the PDF
|
||||
* marker, the FHIR `referenceRange`, and the API verdict agree.
|
||||
*/
|
||||
referenceLow: number | null;
|
||||
referenceHigh: number | null;
|
||||
/** The catalog band, kept so the PDF can name a divergence. */
|
||||
catalogReferenceLow: number | null;
|
||||
catalogReferenceHigh: number | null;
|
||||
/** The window the source report printed, verbatim; null when it printed none. */
|
||||
sourceReferenceText: string | null;
|
||||
/** Which of the two windows `referenceLow` / `referenceHigh` came from. */
|
||||
referenceOrigin: "source" | "catalog" | "none";
|
||||
/** True when the report's window and the catalog band state different limits. */
|
||||
referenceDivergesFromCatalog: boolean;
|
||||
/** ISO timestamp of the latest reading. */
|
||||
takenAt: string;
|
||||
/** Count of readings for this analyte in the window (≥ 1). */
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
* Split out of the aggregator with the selection rework.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { resolveEffectiveReferenceRange } from "@/lib/labs/reference-range";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
import { decryptAllergyReaction } from "@/lib/doctor-report-helpers";
|
||||
import type { DoctorReportData } from "@/lib/doctor-report-types";
|
||||
@@ -42,6 +43,9 @@ export async function loadLabResults(
|
||||
unit: true,
|
||||
referenceLow: true,
|
||||
referenceHigh: true,
|
||||
sourceReferenceLow: true,
|
||||
sourceReferenceHigh: true,
|
||||
sourceReferenceText: true,
|
||||
takenAt: true,
|
||||
},
|
||||
});
|
||||
@@ -52,14 +56,28 @@ export async function loadLabResults(
|
||||
for (const r of rows) {
|
||||
const key = r.analyte.toLowerCase();
|
||||
const prev = byAnalyte.get(key);
|
||||
// The row already stamps the catalog band it was written under; the
|
||||
// reading's own report window, when present, is what the physician read
|
||||
// the value against, so it wins here exactly as it does on every other
|
||||
// lab surface.
|
||||
const range = resolveEffectiveReferenceRange(
|
||||
r.referenceLow,
|
||||
r.referenceHigh,
|
||||
r,
|
||||
);
|
||||
byAnalyte.set(key, {
|
||||
panel: r.panel,
|
||||
analyte: r.analyte,
|
||||
value: r.value,
|
||||
valueText: r.valueText,
|
||||
unit: r.unit,
|
||||
referenceLow: r.referenceLow,
|
||||
referenceHigh: r.referenceHigh,
|
||||
referenceLow: range.low,
|
||||
referenceHigh: range.high,
|
||||
catalogReferenceLow: range.catalogLow,
|
||||
catalogReferenceHigh: range.catalogHigh,
|
||||
sourceReferenceText: range.sourceText,
|
||||
referenceOrigin: range.origin,
|
||||
referenceDivergesFromCatalog: range.divergesFromCatalog,
|
||||
takenAt: r.takenAt.toISOString(),
|
||||
count: (prev?.count ?? 0) + 1,
|
||||
});
|
||||
|
||||
@@ -15,6 +15,12 @@
|
||||
* as-needed medication with notifications off — a record of what the patient
|
||||
* takes, never a prescription action. An Observation never carries a
|
||||
* range-flag; the reference bounds ride along only as the document stated them.
|
||||
*
|
||||
* The window the report printed beside a value is carried onto the reading, not
|
||||
* dropped. Until now it only seeded a freshly minted catalog marker, which meant
|
||||
* that for every analyte the user already had, the range printed on the report
|
||||
* was read and then thrown away. It now lands on the row's `sourceReference*`
|
||||
* columns and governs that reading's verdict.
|
||||
*/
|
||||
import { encryptToBytes } from "@/lib/ai/coach/bytes-codec";
|
||||
import { prisma } from "@/lib/db";
|
||||
@@ -22,6 +28,7 @@ import { invalidateUserHealthScore } from "@/lib/cache/invalidate";
|
||||
import { emitDataArrival } from "@/lib/arrivals/emit-shared";
|
||||
import { decryptFactData } from "@/lib/documents/store";
|
||||
import { resolveOrMintBiomarker } from "@/lib/labs/biomarker-store";
|
||||
import { parseReferenceRange } from "@/lib/labs/parse-reference-range";
|
||||
import type { ExtractedFact } from "@/generated/prisma/client";
|
||||
import type {
|
||||
ConditionFactData,
|
||||
@@ -66,11 +73,21 @@ async function commitObservation(
|
||||
"A numeric value needs a unit before it can be saved",
|
||||
);
|
||||
}
|
||||
// The printed window, read once. `referenceText` is what the report wrote;
|
||||
// the parser derives bounds from it where it can, and where it cannot the
|
||||
// string still survives onto the row. A document staged before the field
|
||||
// existed has no text, and the numeric bounds the model reported are then
|
||||
// the whole of what it stated.
|
||||
const printed = parseReferenceRange(data.referenceText, data.unit);
|
||||
const sourceLow = printed?.low ?? data.referenceLow;
|
||||
const sourceHigh = printed?.high ?? data.referenceHigh;
|
||||
const sourceText = printed?.text ?? null;
|
||||
|
||||
const biomarker = await resolveOrMintBiomarker(userId, {
|
||||
analyte: data.label,
|
||||
unit: isQualitative ? (data.unit ?? "") : (data.unit as string),
|
||||
referenceLow: isQualitative ? null : data.referenceLow,
|
||||
referenceHigh: isQualitative ? null : data.referenceHigh,
|
||||
referenceLow: isQualitative ? null : sourceLow,
|
||||
referenceHigh: isQualitative ? null : sourceHigh,
|
||||
panel: null,
|
||||
});
|
||||
// Fail closed on a unit mismatch against an EXISTING marker. A freshly
|
||||
@@ -101,6 +118,11 @@ async function commitObservation(
|
||||
unit: biomarker.unit,
|
||||
referenceLow: biomarker.lowerBound,
|
||||
referenceHigh: biomarker.upperBound,
|
||||
// The report's own window for THIS reading. A qualitative result has no
|
||||
// window to record.
|
||||
sourceReferenceLow: isQualitative ? null : sourceLow,
|
||||
sourceReferenceHigh: isQualitative ? null : sourceHigh,
|
||||
sourceReferenceText: isQualitative ? null : sourceText,
|
||||
takenAt: statedDateOrNow(data.effectiveDate),
|
||||
source: "DOCUMENT",
|
||||
noteEncrypted: null,
|
||||
|
||||
@@ -74,7 +74,8 @@ Extract these fact types:
|
||||
- code + codeSystem ("LOINC"): ONLY if a code is printed, else null.
|
||||
- value: the numeric result OR null. valueText: the qualitative result text (e.g. "negative") OR null. Set EXACTLY ONE.
|
||||
- unit: the printed unit, or null. NEVER convert or normalise a unit.
|
||||
- referenceLow / referenceHigh: the printed reference bounds as numbers, or null. Do NOT flag the value against them.
|
||||
- referenceText: the reference range EXACTLY as printed beside the value, verbatim and complete, including its operator and unit if written (for example "3,5 - 5,0", "< 5", "bis 5,0", "\u2265 3,5", "150.000 - 400.000 /\u00b5l", "negativ"). Copy the characters; do NOT reformat, convert, or normalise them. Null only when no range is printed for that value.
|
||||
- referenceLow / referenceHigh: the SAME range reduced to two numbers, when it reduces to two numbers, else null. Do NOT flag the value against them.
|
||||
- effectiveDate: the stated measurement date (YYYY-MM-DD), else null.
|
||||
3. MEDICATION_STATEMENT — a medication the patient is/was taking, as recorded. Capture:
|
||||
- label: the drug name, verbatim.
|
||||
@@ -91,7 +92,7 @@ For EVERY fact also capture:
|
||||
Do not emit a fact you cannot read — omit it rather than invent one. Also capture reportDate: the document's stated report/collection date (YYYY-MM-DD), or null, and kind: "DOCTOR_REPORT", "DISCHARGE_LETTER", or "OTHER".
|
||||
|
||||
Respond ONLY with a JSON object of this exact shape:
|
||||
{ "reportDate": string|null, "kind": "DOCTOR_REPORT"|"DISCHARGE_LETTER"|"OTHER"|null, "facts": [ { "type": "CONDITION"|"OBSERVATION"|"MEDICATION_STATEMENT", "label": string, "code": string|null, "codeSystem": "SNOMED"|"ICD10"|"LOINC"|"RXNORM"|"ATC"|null, "clinicalStatus": string|null, "verificationStatus": string|null, "value": number|null, "valueText": string|null, "unit": string|null, "referenceLow": number|null, "referenceHigh": number|null, "dose": string|null, "medicationStatus": string|null, "effectiveDate": string|null, "sourceText": string, "page": number|null, "confidence": number } ] }`;
|
||||
{ "reportDate": string|null, "kind": "DOCTOR_REPORT"|"DISCHARGE_LETTER"|"OTHER"|null, "facts": [ { "type": "CONDITION"|"OBSERVATION"|"MEDICATION_STATEMENT", "label": string, "code": string|null, "codeSystem": "SNOMED"|"ICD10"|"LOINC"|"RXNORM"|"ATC"|null, "clinicalStatus": string|null, "verificationStatus": string|null, "value": number|null, "valueText": string|null, "unit": string|null, "referenceText": string|null, "referenceLow": number|null, "referenceHigh": number|null, "dose": string|null, "medicationStatus": string|null, "effectiveDate": string|null, "sourceText": string, "page": number|null, "confidence": number } ] }`;
|
||||
|
||||
const USER_PROMPT = `Transcribe the clinical document in the attached file into the JSON schema described in the system prompt. Reproduce only what is written — do not interpret. Return only the JSON object.`;
|
||||
|
||||
@@ -172,6 +173,7 @@ function mapFactData(raw: ExtractedFactRaw): FactData | null {
|
||||
unit: raw.unit,
|
||||
referenceLow: raw.referenceLow,
|
||||
referenceHigh: raw.referenceHigh,
|
||||
referenceText: raw.referenceText,
|
||||
effectiveDate,
|
||||
};
|
||||
return data;
|
||||
@@ -309,6 +311,14 @@ export async function runInboundExtraction(
|
||||
unanchored,
|
||||
conditions: facts.filter((f) => f.factType === "CONDITION").length,
|
||||
observations: facts.filter((f) => f.factType === "OBSERVATION").length,
|
||||
// How many transcribed observations arrived with the window the report
|
||||
// printed beside them. A collapse here is the extraction quietly going
|
||||
// back to dropping the range.
|
||||
observationsWithRange: facts.filter(
|
||||
(f) =>
|
||||
f.factType === "OBSERVATION" &&
|
||||
((f.data as ObservationFactData).referenceText ?? null) !== null,
|
||||
).length,
|
||||
medications: facts.filter((f) => f.factType === "MEDICATION_STATEMENT")
|
||||
.length,
|
||||
needsReview: facts.filter((f) => f.needsReview).length,
|
||||
|
||||
@@ -38,6 +38,8 @@ import { restoreProfileData } from "../profile-backup";
|
||||
const deletedAt = new Date("2026-07-19T12:00:00.000Z");
|
||||
const measurementNote = encryptToBytes("canonical measurement note");
|
||||
const sideEffectNote = encryptToBytes("nausea after the evening dose");
|
||||
const moodNotePlaintext = "quiet evening, slept badly";
|
||||
const moodNote = encryptToBytes(moodNotePlaintext);
|
||||
|
||||
const appSettings = {
|
||||
id: "singleton",
|
||||
@@ -193,11 +195,27 @@ function makePrisma() {
|
||||
mood: "OKAY",
|
||||
score: 3,
|
||||
tags: null,
|
||||
note: null,
|
||||
noteEncrypted: moodNote,
|
||||
source: "MOODLOG",
|
||||
externalId: "moodlog-42",
|
||||
moodLoggedAt: new Date("2026-07-19T08:00:00.000Z"),
|
||||
tz: "America/New_York",
|
||||
syncedAt: new Date("2026-07-19T08:00:01.000Z"),
|
||||
syncVersion: 3,
|
||||
deletedAt,
|
||||
tagLinks: [{ rating: 4, moodTag: { key: "sleep_quality" } }],
|
||||
createdAt: new Date("2026-07-19T08:00:00.000Z"),
|
||||
updatedAt: new Date("2026-07-19T09:00:00.000Z"),
|
||||
tagLinks: [
|
||||
{
|
||||
rating: 4,
|
||||
moodTag: { key: "sleep_quality", kind: "RATED" },
|
||||
},
|
||||
{
|
||||
rating: null,
|
||||
moodTag: { key: "headache", kind: "BINARY" },
|
||||
},
|
||||
],
|
||||
},
|
||||
]),
|
||||
},
|
||||
@@ -409,7 +427,13 @@ describe("buildFullBackupPayload disaster-recovery mode", () => {
|
||||
expect(prisma.moodEntry.findMany).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
where: { userId: "user-1" },
|
||||
include: expect.any(Object),
|
||||
select: expect.objectContaining({
|
||||
id: true,
|
||||
noteEncrypted: true,
|
||||
tz: true,
|
||||
syncVersion: true,
|
||||
deletedAt: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(mocks.buildRecordsBackupSection).toHaveBeenCalledWith(
|
||||
@@ -485,7 +509,16 @@ describe("buildFullBackupPayload disaster-recovery mode", () => {
|
||||
id: "mood-tombstone",
|
||||
externalId: "moodlog-42",
|
||||
deletedAt: deletedAt.toISOString(),
|
||||
tz: "America/New_York",
|
||||
syncedAt: "2026-07-19T08:00:01.000Z",
|
||||
syncVersion: 3,
|
||||
createdAt: "2026-07-19T08:00:00.000Z",
|
||||
updatedAt: "2026-07-19T09:00:00.000Z",
|
||||
note: null,
|
||||
noteEncrypted: Buffer.from(moodNote).toString("base64"),
|
||||
factors: [{ key: "sleep_quality", rating: 4 }],
|
||||
// The BINARY link the old `rating: { not: null }` filter excluded.
|
||||
structuredTags: ["headache"],
|
||||
},
|
||||
],
|
||||
nutrientDays: [
|
||||
@@ -1148,3 +1181,79 @@ describe("buildFullBackupPayload — profile and custom metrics", () => {
|
||||
expect(counts.intradayProfiles).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildFullBackupPayload — mood arms", () => {
|
||||
it("carries the ciphertext and the whole row on the disaster-recovery arm", async () => {
|
||||
installSectionMocks();
|
||||
const prisma = makePrisma();
|
||||
|
||||
const { payload } = await buildFullBackupPayload(
|
||||
prisma as never,
|
||||
"user-1",
|
||||
{
|
||||
purpose: "disaster-recovery",
|
||||
exportedAt: new Date("2026-07-20T00:00:00.000Z"),
|
||||
},
|
||||
);
|
||||
|
||||
const [entry] = (payload as { moodEntries: Array<Record<string, unknown>> })
|
||||
.moodEntries;
|
||||
expect(entry).toEqual(
|
||||
expect.objectContaining({
|
||||
id: "mood-tombstone",
|
||||
note: null,
|
||||
noteEncrypted: Buffer.from(moodNote).toString("base64"),
|
||||
tz: "America/New_York",
|
||||
syncedAt: "2026-07-19T08:00:01.000Z",
|
||||
syncVersion: 3,
|
||||
createdAt: "2026-07-19T08:00:00.000Z",
|
||||
updatedAt: "2026-07-19T09:00:00.000Z",
|
||||
deletedAt: deletedAt.toISOString(),
|
||||
factors: [{ key: "sleep_quality", rating: 4 }],
|
||||
structuredTags: ["headache"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("carries the readable note and no ciphertext on the portable arm", async () => {
|
||||
installSectionMocks();
|
||||
const prisma = makePrisma();
|
||||
|
||||
const { payload } = await buildFullBackupPayload(
|
||||
prisma as never,
|
||||
"user-1",
|
||||
{
|
||||
purpose: "portable-export",
|
||||
exportedAt: new Date("2026-07-20T00:00:00.000Z"),
|
||||
},
|
||||
);
|
||||
|
||||
const [entry] = (payload as { moodEntries: Array<Record<string, unknown>> })
|
||||
.moodEntries;
|
||||
// Same split the measurement arm makes: the readable artefact carries the
|
||||
// decrypted text and nothing an operator could not read, and the counters
|
||||
// that only mean something to this instance stay behind.
|
||||
expect(entry).toEqual(
|
||||
expect.objectContaining({
|
||||
note: moodNotePlaintext,
|
||||
tz: "America/New_York",
|
||||
structuredTags: ["headache"],
|
||||
}),
|
||||
);
|
||||
expect(entry).not.toHaveProperty("noteEncrypted");
|
||||
expect(entry).not.toHaveProperty("syncVersion");
|
||||
expect(entry).not.toHaveProperty("syncedAt");
|
||||
expect(JSON.stringify(entry)).not.toContain(
|
||||
Buffer.from(moodNote).toString("base64"),
|
||||
);
|
||||
// The tag-link read is unfiltered on both arms; a BINARY link is the one
|
||||
// most people have and it was the one the file never carried.
|
||||
expect(prisma.moodEntry.findMany).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
select: expect.objectContaining({
|
||||
tagLinks: { select: expect.any(Object) },
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -103,6 +103,46 @@ const MEASUREMENT_BACKUP_SELECT = {
|
||||
updatedAt: true,
|
||||
} as const satisfies Prisma.MeasurementSelect;
|
||||
|
||||
/**
|
||||
* Every restorable `MoodEntry` scalar. `userId` is absent for the same reason
|
||||
* it is absent above: the backup is user-scoped and the owner is re-derived
|
||||
* from the restoring account.
|
||||
*
|
||||
* Named rather than inlined so `mood-backup-completeness.test.ts` can compare
|
||||
* it field-by-field against the model in `prisma/schema.prisma`, exactly as the
|
||||
* measurement guard does for the select above.
|
||||
*/
|
||||
const MOOD_ENTRY_BACKUP_SELECT = {
|
||||
id: true,
|
||||
date: true,
|
||||
mood: true,
|
||||
score: true,
|
||||
tags: true,
|
||||
note: true,
|
||||
noteEncrypted: true,
|
||||
source: true,
|
||||
externalId: true,
|
||||
moodLoggedAt: true,
|
||||
tz: true,
|
||||
syncedAt: true,
|
||||
syncVersion: true,
|
||||
deletedAt: true,
|
||||
createdAt: true,
|
||||
updatedAt: true,
|
||||
} as const satisfies Prisma.MoodEntrySelect;
|
||||
|
||||
/**
|
||||
* A structured-tag link, as it rides the wire. The link is carried by the
|
||||
* tag's KEY, not by its id: a restore resolves the key against the target
|
||||
* instance's catalogue, where the source instance's row id may not exist at
|
||||
* all. `kind` comes along so the mapper can split RATED factors from BINARY
|
||||
* tags without a second query.
|
||||
*/
|
||||
const MOOD_ENTRY_TAG_LINK_SELECT = {
|
||||
rating: true,
|
||||
moodTag: { select: { key: true, kind: true } },
|
||||
} as const satisfies Prisma.MoodEntryTagLinkSelect;
|
||||
|
||||
/**
|
||||
* Build the canonical full-backup payload for `userId`. Portable exports omit
|
||||
* tombstones and document ciphertext; disaster-recovery payloads preserve
|
||||
@@ -207,14 +247,13 @@ export async function buildFullBackupPayload(
|
||||
prisma.moodEntry.findMany({
|
||||
where: disasterRecovery ? { userId } : { userId, deletedAt: null },
|
||||
orderBy: { moodLoggedAt: "desc" },
|
||||
include: {
|
||||
tagLinks: {
|
||||
where: { rating: { not: null } },
|
||||
select: {
|
||||
rating: true,
|
||||
moodTag: { select: { key: true } },
|
||||
},
|
||||
},
|
||||
select: {
|
||||
...MOOD_ENTRY_BACKUP_SELECT,
|
||||
// Every link, not only the rated ones. This read used to filter
|
||||
// `rating: { not: null }`, and a BINARY link carries a NULL rating by
|
||||
// definition — so the present/absent tags, the kind most people pick,
|
||||
// were absent from the file and could not come back from it.
|
||||
tagLinks: { select: MOOD_ENTRY_TAG_LINK_SELECT },
|
||||
},
|
||||
}),
|
||||
// The account's OWN mood tags. The seeded catalogue is reference data every
|
||||
@@ -394,21 +433,66 @@ export async function buildFullBackupPayload(
|
||||
skipped: e.skipped,
|
||||
source: e.source,
|
||||
})),
|
||||
moodEntries: moodEntries.map((e) => ({
|
||||
id: e.id,
|
||||
date: e.date,
|
||||
mood: e.mood,
|
||||
score: e.score,
|
||||
tags: e.tags,
|
||||
source: e.source,
|
||||
loggedAt: e.moodLoggedAt.toISOString(),
|
||||
externalId: e.externalId,
|
||||
deletedAt: e.deletedAt?.toISOString() ?? null,
|
||||
factors: e.tagLinks.map((link) => ({
|
||||
key: link.moodTag.key,
|
||||
rating: link.rating!,
|
||||
})),
|
||||
})),
|
||||
// The parameter is spelled out rather than the `e` every other mapper here
|
||||
// uses, and deliberately: `mood-backup-completeness.test.ts` proves a
|
||||
// selected column reaches a payload by matching `: moodEntry.`. A matcher
|
||||
// on `: e.` would also match the intake-event mapper above it and could
|
||||
// therefore never fail.
|
||||
moodEntries: moodEntries.map((moodEntry) => {
|
||||
// The two link arms partition the entry's links exhaustively: a link is
|
||||
// a rated factor when its tag is RATED and it actually carries a score,
|
||||
// and a structured tag otherwise. Total on purpose — a malformed link
|
||||
// (a RATED tag whose rating went missing) rides as a tag key and comes
|
||||
// back NAMED as unresolvable on restore, rather than falling between the
|
||||
// two arms and disappearing the way every BINARY link used to.
|
||||
const isRatedFactor = (link: (typeof moodEntry.tagLinks)[number]) =>
|
||||
link.moodTag.kind === "RATED" && link.rating !== null;
|
||||
|
||||
const factors = moodEntry.tagLinks
|
||||
.filter(isRatedFactor)
|
||||
.map((tagLink) => ({
|
||||
key: tagLink.moodTag.key,
|
||||
rating: tagLink.rating as number,
|
||||
}));
|
||||
// BINARY link keys. Their own array rather than a nullable rating inside
|
||||
// `factors`: the restore resolves `factors` against `kind: "RATED"` and
|
||||
// reports what it cannot resolve, so a rating-less entry in that array
|
||||
// would collide with both. A separate key leaves every backup file
|
||||
// written before this parsing and restoring exactly as it did.
|
||||
const structuredTags = moodEntry.tagLinks
|
||||
.filter((link) => !isRatedFactor(link))
|
||||
.map((tagLink) => tagLink.moodTag.key);
|
||||
|
||||
return {
|
||||
...(disasterRecovery
|
||||
? {
|
||||
note: moodEntry.note,
|
||||
noteEncrypted: moodEntry.noteEncrypted
|
||||
? Buffer.from(moodEntry.noteEncrypted).toString("base64")
|
||||
: null,
|
||||
syncedAt: moodEntry.syncedAt.toISOString(),
|
||||
syncVersion: moodEntry.syncVersion,
|
||||
createdAt: moodEntry.createdAt.toISOString(),
|
||||
updatedAt: moodEntry.updatedAt.toISOString(),
|
||||
}
|
||||
: { note: readNote(moodEntry.noteEncrypted, moodEntry.note) }),
|
||||
id: moodEntry.id,
|
||||
date: moodEntry.date,
|
||||
mood: moodEntry.mood,
|
||||
score: moodEntry.score,
|
||||
tags: moodEntry.tags,
|
||||
source: moodEntry.source,
|
||||
loggedAt: moodEntry.moodLoggedAt.toISOString(),
|
||||
externalId: moodEntry.externalId,
|
||||
// The zone the `date` string is anchored to. Without it a restored row
|
||||
// falls back to the legacy Europe/Berlin reading and an account that
|
||||
// logged elsewhere gets its day boundaries quietly moved.
|
||||
tz: moodEntry.tz,
|
||||
deletedAt: moodEntry.deletedAt?.toISOString() ?? null,
|
||||
factors,
|
||||
structuredTags,
|
||||
};
|
||||
}),
|
||||
customMoodTags: customMoodTags.map((tag) => ({
|
||||
...(disasterRecovery ? { id: tag.id } : {}),
|
||||
key: tag.key,
|
||||
|
||||
@@ -54,6 +54,10 @@ export interface LabResultBackupEntry {
|
||||
unit: string;
|
||||
referenceLow: number | null;
|
||||
referenceHigh: number | null;
|
||||
/** The window the reading's own source report printed, verbatim + derived. */
|
||||
sourceReferenceLow: number | null;
|
||||
sourceReferenceHigh: number | null;
|
||||
sourceReferenceText: string | null;
|
||||
takenAt: string;
|
||||
source: string;
|
||||
/** Human-readable cross-reference into `biomarkers` below, not an id. */
|
||||
@@ -336,6 +340,9 @@ export async function buildRecordsBackupSection(
|
||||
unit: r.unit,
|
||||
referenceLow: r.referenceLow,
|
||||
referenceHigh: r.referenceHigh,
|
||||
sourceReferenceLow: r.sourceReferenceLow,
|
||||
sourceReferenceHigh: r.sourceReferenceHigh,
|
||||
sourceReferenceText: r.sourceReferenceText,
|
||||
takenAt: r.takenAt.toISOString(),
|
||||
source: r.source,
|
||||
biomarkerName: r.biomarker?.name ?? null,
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
/**
|
||||
* What a restore could not put back, and how much of it there was.
|
||||
*
|
||||
* Three of the restore's lookups resolve a key against a SEEDED CATALOGUE —
|
||||
* cycle symptoms, illness symptoms, mood factors. The catalogue is reference
|
||||
* Four of the restore's lookups resolve a key against a SEEDED CATALOGUE —
|
||||
* cycle symptoms, illness symptoms, rated mood factors and the mood tags a
|
||||
* person ticks present or absent. The catalogue is reference
|
||||
* data the instance owns, not data the file carries, so it legitimately drifts
|
||||
* between the day a backup is written and the day it is read: a key gets
|
||||
* renamed, a symptom is retired, the file is a year old, the instance is three
|
||||
@@ -28,7 +29,8 @@
|
||||
*/
|
||||
|
||||
/** Which seeded catalogue a key failed to resolve against. */
|
||||
export type SkippedCatalogue = "cycleSymptom" | "illnessSymptom" | "moodFactor";
|
||||
export type SkippedCatalogue =
|
||||
"cycleSymptom" | "illnessSymptom" | "moodFactor" | "moodTag";
|
||||
|
||||
/** One key this instance does not know, and the links it cost. */
|
||||
export interface SkippedCatalogueKey {
|
||||
|
||||
@@ -54,6 +54,11 @@ describe("lab-result FHIR Observations", () => {
|
||||
unit: "mg/dL",
|
||||
referenceLow: null,
|
||||
referenceHigh: 116,
|
||||
catalogReferenceLow: null,
|
||||
catalogReferenceHigh: null,
|
||||
sourceReferenceText: null,
|
||||
referenceOrigin: "catalog" as const,
|
||||
referenceDivergesFromCatalog: false,
|
||||
takenAt: "2026-04-20T08:00:00.000Z",
|
||||
count: 2,
|
||||
},
|
||||
@@ -65,6 +70,11 @@ describe("lab-result FHIR Observations", () => {
|
||||
unit: "%",
|
||||
referenceLow: 4,
|
||||
referenceHigh: 5.6,
|
||||
catalogReferenceLow: null,
|
||||
catalogReferenceHigh: null,
|
||||
sourceReferenceText: null,
|
||||
referenceOrigin: "catalog" as const,
|
||||
referenceDivergesFromCatalog: false,
|
||||
takenAt: "2026-04-21T08:00:00.000Z",
|
||||
count: 1,
|
||||
},
|
||||
@@ -137,6 +147,11 @@ describe("lab-result FHIR Observations", () => {
|
||||
unit: "mg/L",
|
||||
referenceLow: null,
|
||||
referenceHigh: null,
|
||||
catalogReferenceLow: null,
|
||||
catalogReferenceHigh: null,
|
||||
sourceReferenceText: null,
|
||||
referenceOrigin: "catalog" as const,
|
||||
referenceDivergesFromCatalog: false,
|
||||
takenAt: "2026-04-22T08:00:00.000Z",
|
||||
count: 1,
|
||||
},
|
||||
@@ -178,6 +193,11 @@ describe("lab-result LOINC + canonical UCUM coding (v1.18.8)", () => {
|
||||
unit,
|
||||
referenceLow: null,
|
||||
referenceHigh: null,
|
||||
catalogReferenceLow: null,
|
||||
catalogReferenceHigh: null,
|
||||
sourceReferenceText: null,
|
||||
referenceOrigin: "catalog" as const,
|
||||
referenceDivergesFromCatalog: false,
|
||||
takenAt: "2026-04-20T08:00:00.000Z",
|
||||
count: 1,
|
||||
},
|
||||
@@ -264,6 +284,11 @@ describe("qualitative lab-result FHIR Observations (v1.18.9)", () => {
|
||||
unit: "",
|
||||
referenceLow: null,
|
||||
referenceHigh: null,
|
||||
catalogReferenceLow: null,
|
||||
catalogReferenceHigh: null,
|
||||
sourceReferenceText: null,
|
||||
referenceOrigin: "catalog" as const,
|
||||
referenceDivergesFromCatalog: false,
|
||||
takenAt: "2026-04-20T08:00:00.000Z",
|
||||
count: 1,
|
||||
},
|
||||
|
||||
@@ -7,8 +7,15 @@
|
||||
* `code.text` for the common biomarkers, and stamps the canonical UCUM `code`
|
||||
* on the value WHEN the recorded unit matches the mapped canonical symbol. An
|
||||
* analyte that does NOT resolve keeps the honest text-only `code` +
|
||||
* display-only `unit` (no fabricated terminology). The lab's reference bounds
|
||||
* map onto R4 `referenceRange`.
|
||||
* display-only `unit` (no fabricated terminology).
|
||||
*
|
||||
* The reference bounds map onto R4 `referenceRange`, and they are the SAME
|
||||
* bounds the app's own verdict uses: the window the reading's source report
|
||||
* printed when it carries one, the catalog band otherwise, already resolved
|
||||
* upstream in `loadLabResults`. When the source report stated its window in
|
||||
* words the bounds could not be read from ("negativ", "siehe Befund"), that
|
||||
* string rides on `referenceRange.text` so a receiver still gets what the lab
|
||||
* wrote rather than a silently empty element.
|
||||
*/
|
||||
import type { DoctorReportData } from "@/lib/doctor-report-data";
|
||||
import { LOINC_SYSTEM, UCUM_SYSTEM } from "@/lib/fhir/loinc-map";
|
||||
@@ -82,8 +89,11 @@ export function labObservations(
|
||||
// The bounds are stated in the SAME unit as the value, so they carry the
|
||||
// SAME coding — a bare unit beside a coded value leaves a receiver unable
|
||||
// to compare the two.
|
||||
const sourceText =
|
||||
lab.referenceOrigin === "source" ? lab.sourceReferenceText : null;
|
||||
const hasBounds = lab.referenceLow !== null || lab.referenceHigh !== null;
|
||||
const referenceRange: FhirObservationReferenceRange[] | undefined =
|
||||
lab.referenceLow !== null || lab.referenceHigh !== null
|
||||
hasBounds || sourceText
|
||||
? [
|
||||
{
|
||||
...(lab.referenceLow !== null
|
||||
@@ -92,6 +102,7 @@ export function labObservations(
|
||||
...(lab.referenceHigh !== null
|
||||
? { high: quantity(lab.referenceHigh) }
|
||||
: {}),
|
||||
...(sourceText ? { text: sourceText } : {}),
|
||||
},
|
||||
]
|
||||
: undefined;
|
||||
|
||||
@@ -117,6 +117,13 @@ export interface FhirObservation {
|
||||
export interface FhirObservationReferenceRange {
|
||||
low?: FhirQuantity;
|
||||
high?: FhirQuantity;
|
||||
/**
|
||||
* R4 `referenceRange.text`. Carries the window the source report printed,
|
||||
* verbatim, when the reading has one. A receiver that cannot use the bounds
|
||||
* (because the report stated a qualitative window like "negativ", or a
|
||||
* one-sided one) still reads what the lab wrote.
|
||||
*/
|
||||
text?: string;
|
||||
}
|
||||
|
||||
export interface FhirDosage {
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* `upsertFitbitMeasurements` is the single write seam for every Fitbit
|
||||
* resource leg — activity, sleep, daily metrics — and it holds Prisma
|
||||
* directly, so it does not inherit the shared reconciler's gate. These cases
|
||||
* pin that the provider's numbers clear the app's own declared plausibility
|
||||
* band before any of them can become a row, and that a refusal is reported.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { findManyMock, createManyMock, updateMock } = vi.hoisted(() => ({
|
||||
findManyMock: vi.fn(async () => [] as unknown[]),
|
||||
createManyMock: vi.fn<
|
||||
() => Promise<Array<{ id: string; type: string; measuredAt: Date }>>
|
||||
>(async () => []),
|
||||
updateMock: vi.fn(async () => ({})),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
measurement: {
|
||||
findMany: findManyMock,
|
||||
createManyAndReturn: createManyMock,
|
||||
update: updateMock,
|
||||
},
|
||||
},
|
||||
}));
|
||||
vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() }));
|
||||
vi.mock("@/lib/integrations/status", () => ({
|
||||
isReauthRequired: vi.fn(async () => false),
|
||||
recordSyncFailure: vi.fn(async () => {}),
|
||||
recordSyncSuccess: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("@/lib/rollups/measurement-rollups", () => ({
|
||||
collapseToTypeDayKeys: vi.fn(() => []),
|
||||
recomputeBucketsForMeasurement: vi.fn(async () => {}),
|
||||
recomputeUserRollups: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("@/lib/insights/comprehensive-generate", () => ({
|
||||
invalidateStatusInsightsForTypes: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("@/lib/integrations/oauth-refresh", () => ({
|
||||
persistRotatedToken: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("../credentials", () => ({
|
||||
getUserFitbitCredentials: vi.fn(async () => null),
|
||||
}));
|
||||
vi.mock("../client", () => ({ refreshAccessToken: vi.fn() }));
|
||||
|
||||
import { eventStorage } from "@/lib/logging/context";
|
||||
import { WideEventBuilder } from "@/lib/logging/event-builder";
|
||||
import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate";
|
||||
import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate";
|
||||
|
||||
import { upsertFitbitMeasurements } from "../sync-core";
|
||||
|
||||
/** RESTING_HEART_RATE is declared 25–220. */
|
||||
const REAL_READING = {
|
||||
type: "RESTING_HEART_RATE",
|
||||
value: 58,
|
||||
unit: "bpm",
|
||||
measuredAt: new Date("2026-07-08T00:00:00.000Z"),
|
||||
externalId: "rhr:2026-07-08",
|
||||
};
|
||||
|
||||
const IMPOSSIBLE_READING = {
|
||||
type: "RESTING_HEART_RATE",
|
||||
value: 4_915_200,
|
||||
unit: "bpm",
|
||||
measuredAt: new Date("2026-07-09T00:00:00.000Z"),
|
||||
externalId: "rhr:2026-07-09",
|
||||
};
|
||||
|
||||
let event: WideEventBuilder;
|
||||
|
||||
function tally(): RangeRejectionTally | undefined {
|
||||
return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as
|
||||
RangeRejectionTally | undefined;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
findManyMock.mockReset().mockResolvedValue([]);
|
||||
createManyMock.mockReset().mockResolvedValue([]);
|
||||
updateMock.mockReset().mockResolvedValue({});
|
||||
event = new WideEventBuilder("background");
|
||||
});
|
||||
|
||||
describe("upsertFitbitMeasurements — plausibility gate", () => {
|
||||
it("writes a reading inside the metric's declared band", async () => {
|
||||
createManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "row-1",
|
||||
type: "RESTING_HEART_RATE",
|
||||
measuredAt: REAL_READING.measuredAt,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertFitbitMeasurements("user-1", [REAL_READING], { deferRollup: true }),
|
||||
);
|
||||
|
||||
expect(result.imported).toBe(1);
|
||||
const planned = (createManyMock.mock.calls[0] as unknown[])[0] as {
|
||||
data: Array<{ value: number }>;
|
||||
};
|
||||
expect(planned.data).toHaveLength(1);
|
||||
expect(planned.data[0]!.value).toBe(58);
|
||||
expect(tally()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refuses an out-of-band reading and annotates the drop", async () => {
|
||||
createManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "row-1",
|
||||
type: "RESTING_HEART_RATE",
|
||||
measuredAt: REAL_READING.measuredAt,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertFitbitMeasurements("user-1", [REAL_READING, IMPOSSIBLE_READING], {
|
||||
deferRollup: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.imported).toBe(1);
|
||||
const planned = (createManyMock.mock.calls[0] as unknown[])[0] as {
|
||||
data: Array<{ value: number; externalId: string }>;
|
||||
};
|
||||
expect(planned.data).toHaveLength(1);
|
||||
expect(planned.data[0]!.externalId).toBe("rhr:2026-07-08");
|
||||
// The refused reading never even reached the existence probe.
|
||||
expect(
|
||||
(
|
||||
(findManyMock.mock.calls[0] as unknown[])[0] as {
|
||||
where: { externalId: { in: string[] } };
|
||||
}
|
||||
).where.externalId.in,
|
||||
).toEqual(["rhr:2026-07-08"]);
|
||||
expect(tally()).toEqual({
|
||||
total: 1,
|
||||
buckets: [
|
||||
{
|
||||
source: "fitbit",
|
||||
type: "RESTING_HEART_RATE",
|
||||
direction: "above_max",
|
||||
count: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(event.getLevel()).toBe("warn");
|
||||
});
|
||||
|
||||
it("does not overwrite a stored reading with an out-of-band one", async () => {
|
||||
// The key already exists; ungated, the update branch rewrote its value.
|
||||
findManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "existing",
|
||||
type: "RESTING_HEART_RATE",
|
||||
externalId: "rhr:2026-07-09",
|
||||
value: 58,
|
||||
unit: "bpm",
|
||||
measuredAt: IMPOSSIBLE_READING.measuredAt,
|
||||
sleepStage: null,
|
||||
deletedAt: null,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertFitbitMeasurements("user-1", [IMPOSSIBLE_READING], {
|
||||
deferRollup: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.imported).toBe(0);
|
||||
expect(updateMock).not.toHaveBeenCalled();
|
||||
expect(createManyMock).not.toHaveBeenCalled();
|
||||
expect(tally()?.total).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -15,6 +15,7 @@ import { persistRotatedToken } from "@/lib/integrations/oauth-refresh";
|
||||
import { recordSyncFailure } from "@/lib/integrations/status";
|
||||
import type { FailureKind } from "@/lib/integrations/status";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate";
|
||||
import {
|
||||
collapseToTypeDayKeys,
|
||||
recomputeBucketsForMeasurement,
|
||||
@@ -530,7 +531,7 @@ export async function replaceStaleFitbitSleep(
|
||||
*/
|
||||
export async function upsertFitbitMeasurements(
|
||||
userId: string,
|
||||
readings: FitbitMeasurementUpsert[],
|
||||
incoming: FitbitMeasurementUpsert[],
|
||||
opts: { deferRollup?: boolean } = {},
|
||||
): Promise<{
|
||||
imported: number;
|
||||
@@ -541,6 +542,15 @@ export async function upsertFitbitMeasurements(
|
||||
measuredAt: Date;
|
||||
}>;
|
||||
}> {
|
||||
// Every resource leg funnels through here, so this is the one place the
|
||||
// provider's numbers have to clear the app's declared plausibility band
|
||||
// before they can become rows. A refused reading is dropped and tallied on
|
||||
// the ambient event, never clamped and never written for a later reader to
|
||||
// discover.
|
||||
const readings = dropImplausibleMeasurements("fitbit", incoming, (r) => ({
|
||||
type: r.type,
|
||||
value: r.value,
|
||||
}));
|
||||
if (readings.length === 0) return { imported: 0, touched: [], inserted: [] };
|
||||
|
||||
// Probe EVERY existing row (live AND tombstoned) for the batch's externalIds
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* `upsertGoogleHealthMeasurements` is the single write seam for every Google
|
||||
* Health resource leg and it holds Prisma directly, so it does not inherit the
|
||||
* shared reconciler's gate. These cases pin that the provider's numbers clear
|
||||
* the app's own declared plausibility band before any of them can become a
|
||||
* row, and that a refusal is reported rather than swallowed.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { findManyMock, createManyMock, updateMock } = vi.hoisted(() => ({
|
||||
findManyMock: vi.fn(async () => [] as unknown[]),
|
||||
createManyMock: vi.fn<
|
||||
() => Promise<Array<{ id: string; type: string; measuredAt: Date }>>
|
||||
>(async () => []),
|
||||
updateMock: vi.fn(async () => ({})),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
measurement: {
|
||||
findMany: findManyMock,
|
||||
createManyAndReturn: createManyMock,
|
||||
update: updateMock,
|
||||
},
|
||||
},
|
||||
}));
|
||||
vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() }));
|
||||
vi.mock("@/lib/integrations/status", () => ({
|
||||
isReauthRequired: vi.fn(async () => false),
|
||||
recordSyncFailure: vi.fn(async () => {}),
|
||||
recordSyncSuccess: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("@/lib/rollups/measurement-rollups", () => ({
|
||||
collapseToTypeDayKeys: vi.fn(() => []),
|
||||
recomputeBucketsForMeasurement: vi.fn(async () => {}),
|
||||
recomputeUserRollups: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("@/lib/insights/comprehensive-generate", () => ({
|
||||
invalidateStatusInsightsForTypes: vi.fn(async () => {}),
|
||||
}));
|
||||
vi.mock("../credentials", () => ({
|
||||
getUserGoogleHealthCredentials: vi.fn(async () => null),
|
||||
}));
|
||||
vi.mock("../client", async () => {
|
||||
const actual = await vi.importActual<typeof import("../client")>("../client");
|
||||
return {
|
||||
...actual,
|
||||
refreshAccessToken: vi.fn(),
|
||||
noteGoogleHealthMapped: vi.fn(),
|
||||
noteGoogleHealthWritten: vi.fn(),
|
||||
noteGoogleHealthOutcomeFailure: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
import { eventStorage } from "@/lib/logging/context";
|
||||
import { WideEventBuilder } from "@/lib/logging/event-builder";
|
||||
import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate";
|
||||
import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate";
|
||||
|
||||
import { upsertGoogleHealthMeasurements } from "../sync-core";
|
||||
|
||||
/** OXYGEN_SATURATION is declared 50–100. */
|
||||
const REAL_READING = {
|
||||
type: "OXYGEN_SATURATION",
|
||||
value: 97,
|
||||
unit: "%",
|
||||
measuredAt: new Date("2026-07-08T00:00:00.000Z"),
|
||||
externalId: "spo2:2026-07-08",
|
||||
};
|
||||
|
||||
const IMPOSSIBLE_READING = {
|
||||
type: "OXYGEN_SATURATION",
|
||||
value: 9_700,
|
||||
unit: "%",
|
||||
measuredAt: new Date("2026-07-09T00:00:00.000Z"),
|
||||
externalId: "spo2:2026-07-09",
|
||||
};
|
||||
|
||||
let event: WideEventBuilder;
|
||||
|
||||
function tally(): RangeRejectionTally | undefined {
|
||||
return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as
|
||||
RangeRejectionTally | undefined;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
findManyMock.mockReset().mockResolvedValue([]);
|
||||
createManyMock.mockReset().mockResolvedValue([]);
|
||||
updateMock.mockReset().mockResolvedValue({});
|
||||
event = new WideEventBuilder("background");
|
||||
});
|
||||
|
||||
describe("upsertGoogleHealthMeasurements — plausibility gate", () => {
|
||||
it("writes a reading inside the metric's declared band", async () => {
|
||||
createManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "row-1",
|
||||
type: "OXYGEN_SATURATION",
|
||||
measuredAt: REAL_READING.measuredAt,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertGoogleHealthMeasurements("user-1", [REAL_READING], {
|
||||
deferRollup: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.imported).toBe(1);
|
||||
const planned = (createManyMock.mock.calls[0] as unknown[])[0] as {
|
||||
data: Array<{ value: number }>;
|
||||
};
|
||||
expect(planned.data).toHaveLength(1);
|
||||
expect(planned.data[0]!.value).toBe(97);
|
||||
expect(tally()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refuses an out-of-band reading and annotates the drop", async () => {
|
||||
createManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "row-1",
|
||||
type: "OXYGEN_SATURATION",
|
||||
measuredAt: REAL_READING.measuredAt,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertGoogleHealthMeasurements(
|
||||
"user-1",
|
||||
[REAL_READING, IMPOSSIBLE_READING],
|
||||
{ deferRollup: true },
|
||||
),
|
||||
);
|
||||
|
||||
expect(result.imported).toBe(1);
|
||||
const planned = (createManyMock.mock.calls[0] as unknown[])[0] as {
|
||||
data: Array<{ externalId: string }>;
|
||||
};
|
||||
expect(planned.data).toHaveLength(1);
|
||||
expect(planned.data[0]!.externalId).toBe("spo2:2026-07-08");
|
||||
expect(tally()).toEqual({
|
||||
total: 1,
|
||||
buckets: [
|
||||
{
|
||||
source: "googleHealth",
|
||||
type: "OXYGEN_SATURATION",
|
||||
direction: "above_max",
|
||||
count: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(event.getLevel()).toBe("warn");
|
||||
});
|
||||
|
||||
it("does not overwrite a stored reading with an out-of-band one", async () => {
|
||||
findManyMock.mockResolvedValue([
|
||||
{
|
||||
id: "existing",
|
||||
type: "OXYGEN_SATURATION",
|
||||
externalId: "spo2:2026-07-09",
|
||||
value: 97,
|
||||
unit: "%",
|
||||
measuredAt: IMPOSSIBLE_READING.measuredAt,
|
||||
sleepStage: null,
|
||||
deletedAt: null,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertGoogleHealthMeasurements("user-1", [IMPOSSIBLE_READING], {
|
||||
deferRollup: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.imported).toBe(0);
|
||||
expect(updateMock).not.toHaveBeenCalled();
|
||||
expect(createManyMock).not.toHaveBeenCalled();
|
||||
expect(tally()?.total).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -20,6 +20,7 @@ import { prisma } from "@/lib/db";
|
||||
import type { MeasurementType } from "@/generated/prisma/client";
|
||||
import { encrypt, decrypt } from "@/lib/crypto";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate";
|
||||
import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit";
|
||||
import { recordSyncFailure, type FailureKind } from "@/lib/integrations/status";
|
||||
import {
|
||||
@@ -487,7 +488,7 @@ export async function replaceStaleGoogleHealthSleep(
|
||||
*/
|
||||
export async function upsertGoogleHealthMeasurements(
|
||||
userId: string,
|
||||
readings: GoogleHealthMeasurementUpsert[],
|
||||
incoming: GoogleHealthMeasurementUpsert[],
|
||||
opts: { deferRollup?: boolean } = {},
|
||||
): Promise<{
|
||||
imported: number;
|
||||
@@ -498,7 +499,17 @@ export async function upsertGoogleHealthMeasurements(
|
||||
measuredAt: Date;
|
||||
}>;
|
||||
}> {
|
||||
noteGoogleHealthMapped(readings.length);
|
||||
noteGoogleHealthMapped(incoming.length);
|
||||
// Every resource leg funnels through here, so this is the one place the
|
||||
// provider's numbers have to clear the app's declared plausibility band
|
||||
// before they can become rows. A refused reading is dropped and tallied on
|
||||
// the ambient event, never clamped and never written for a later reader to
|
||||
// discover.
|
||||
const readings = dropImplausibleMeasurements(
|
||||
"googleHealth",
|
||||
incoming,
|
||||
(r) => ({ type: r.type, value: r.value }),
|
||||
);
|
||||
if (readings.length === 0) return { imported: 0, touched: [], inserted: [] };
|
||||
|
||||
// Probe EVERY existing row (live AND tombstoned) for the batch's externalIds
|
||||
|
||||
@@ -19,7 +19,14 @@ function row(
|
||||
analyte: string,
|
||||
value: number,
|
||||
day: string,
|
||||
opts: { unit?: string; low?: number | null; high?: number | null } = {},
|
||||
opts: {
|
||||
unit?: string;
|
||||
low?: number | null;
|
||||
high?: number | null;
|
||||
sourceLow?: number | null;
|
||||
sourceHigh?: number | null;
|
||||
sourceText?: string | null;
|
||||
} = {},
|
||||
): LabChangeRow {
|
||||
return {
|
||||
analyte,
|
||||
@@ -27,6 +34,9 @@ function row(
|
||||
value,
|
||||
referenceLow: opts.low ?? null,
|
||||
referenceHigh: opts.high ?? null,
|
||||
sourceReferenceLow: opts.sourceLow ?? null,
|
||||
sourceReferenceHigh: opts.sourceHigh ?? null,
|
||||
sourceReferenceText: opts.sourceText ?? null,
|
||||
takenAt: new Date(`${day}T08:00:00.000Z`),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -29,7 +29,10 @@ import {
|
||||
} from "@/lib/ai/prompts/biomarker";
|
||||
import { openerArchetypeHint } from "@/lib/ai/prompts/opener-archetype";
|
||||
import type { Locale } from "@/lib/i18n/config";
|
||||
import { classifyReferenceRange } from "@/lib/labs/reference-range";
|
||||
import {
|
||||
classifyAgainstEffectiveRange,
|
||||
resolveEffectiveReferenceRange,
|
||||
} from "@/lib/labs/reference-range";
|
||||
import { getNoKeyBiomarkerStatusText } from "@/lib/insights/no-key-fallbacks";
|
||||
import { hashInsightSnapshot } from "@/lib/insights/snapshot-hash";
|
||||
import { runStatusCompletion } from "@/lib/insights/status-provider";
|
||||
@@ -158,7 +161,17 @@ export async function generateBiomarkerStatus(args: {
|
||||
},
|
||||
orderBy: { takenAt: "desc" },
|
||||
take: SNAPSHOT_READING_CAP,
|
||||
select: { id: true, value: true, takenAt: true },
|
||||
select: {
|
||||
id: true,
|
||||
value: true,
|
||||
takenAt: true,
|
||||
// The window this reading's own report printed, when it carries one.
|
||||
// The verdict below is the reading's, not the marker's, so it has to
|
||||
// read the reading's window — see `resolveEffectiveReferenceRange`.
|
||||
sourceReferenceLow: true,
|
||||
sourceReferenceHigh: true,
|
||||
sourceReferenceText: true,
|
||||
},
|
||||
});
|
||||
if (rows.length === 0) {
|
||||
annotate({
|
||||
@@ -174,6 +187,14 @@ export async function generateBiomarkerStatus(args: {
|
||||
takenAt: r.takenAt,
|
||||
}));
|
||||
const latest = readings[0];
|
||||
// The window the latest reading is judged against: its own report's range
|
||||
// when that reading carries one, the catalog band otherwise. One rule, the
|
||||
// same one the API DTO, the doctor report and the coach snapshot apply.
|
||||
const latestRange = resolveEffectiveReferenceRange(
|
||||
marker.lowerBound,
|
||||
marker.upperBound,
|
||||
rows[0],
|
||||
);
|
||||
const totalCount = await prisma.labResult.count({
|
||||
where: {
|
||||
userId: args.userId,
|
||||
@@ -226,7 +247,7 @@ export async function generateBiomarkerStatus(args: {
|
||||
locale,
|
||||
promptVersion: PROMPT_VERSION,
|
||||
count: totalCount,
|
||||
bounds: { lower: marker.lowerBound, upper: marker.upperBound },
|
||||
bounds: { lower: latestRange.low, upper: latestRange.high },
|
||||
latest: {
|
||||
id: latest.id,
|
||||
takenAt: latest.takenAt.toISOString(),
|
||||
@@ -252,11 +273,7 @@ export async function generateBiomarkerStatus(args: {
|
||||
// Ascending series (oldest → newest) for the trend summary the prompt reads.
|
||||
const ascending = [...readings].reverse();
|
||||
const summary = summarizeSeries(ascending.map((r) => ({ value: r.value })));
|
||||
const latestStatus = classifyReferenceRange(
|
||||
latest.value,
|
||||
marker.lowerBound,
|
||||
marker.upperBound,
|
||||
);
|
||||
const latestStatus = classifyAgainstEffectiveRange(latest.value, latestRange);
|
||||
|
||||
const snapshot = {
|
||||
locale,
|
||||
@@ -266,8 +283,8 @@ export async function generateBiomarkerStatus(args: {
|
||||
name: marker.name,
|
||||
unit: marker.unit,
|
||||
referenceRange:
|
||||
marker.lowerBound != null || marker.upperBound != null
|
||||
? { lower: marker.lowerBound, upper: marker.upperBound }
|
||||
latestRange.low != null || latestRange.high != null
|
||||
? { lower: latestRange.low, upper: latestRange.high }
|
||||
: null,
|
||||
},
|
||||
dataCoverage: {
|
||||
|
||||
@@ -590,6 +590,9 @@ export async function fetchLabDraws(
|
||||
unit: true,
|
||||
referenceLow: true,
|
||||
referenceHigh: true,
|
||||
sourceReferenceLow: true,
|
||||
sourceReferenceHigh: true,
|
||||
sourceReferenceText: true,
|
||||
panel: true,
|
||||
value: true,
|
||||
takenAt: true,
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
* point (`extractFeatures`) stays in the hub.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { classifyReferenceRange } from "@/lib/labs/reference-range";
|
||||
import { classifyAgainstEffectiveRange } from "@/lib/labs/reference-range";
|
||||
import { calendarDaysUntil } from "@/lib/measurement-reminders/due-day";
|
||||
import { resolveLabFields } from "@/lib/labs/serialise";
|
||||
import { readRollupBuckets } from "@/lib/rollups/measurement-rollups";
|
||||
@@ -164,6 +164,9 @@ export async function readLabsBriefingBlock(
|
||||
valueText: true,
|
||||
referenceLow: true,
|
||||
referenceHigh: true,
|
||||
sourceReferenceLow: true,
|
||||
sourceReferenceHigh: true,
|
||||
sourceReferenceText: true,
|
||||
takenAt: true,
|
||||
biomarkerId: true,
|
||||
biomarker: {
|
||||
@@ -198,14 +201,10 @@ export async function readLabsBriefingBlock(
|
||||
for (const list of byMarker.values()) {
|
||||
const latest = list[0];
|
||||
const resolved = resolveLabFields(latest, latest.biomarker);
|
||||
const rangeStatus =
|
||||
latest.value === null
|
||||
? ("unknown" as const)
|
||||
: classifyReferenceRange(
|
||||
latest.value,
|
||||
resolved.referenceLow,
|
||||
resolved.referenceHigh,
|
||||
);
|
||||
const rangeStatus = classifyAgainstEffectiveRange(
|
||||
latest.value,
|
||||
resolved.effectiveRange,
|
||||
);
|
||||
|
||||
// Trend = latest numeric reading vs the immediately prior numeric reading.
|
||||
let trend: "rising" | "falling" | "flat" | null = null;
|
||||
|
||||
@@ -4,9 +4,11 @@
|
||||
* Groups the user's numeric lab results by panel DATE (the calendar day of
|
||||
* `takenAt`, UTC), finds the two most-recent dates, and for every analyte that
|
||||
* appears in BOTH the latest and the previous panel reports the signed delta,
|
||||
* the direction, and where the latest value sits against its reference band
|
||||
* (`classifyReferenceRange`). Qualitative (valueText-only) results carry no
|
||||
* numeric value, so they are skipped — a delta is meaningless for them.
|
||||
* the direction, and where the latest value sits against the reference window
|
||||
* in force for it — the range that reading's own report printed when it has
|
||||
* one, the catalog band otherwise, resolved through the shared
|
||||
* `resolveEffectiveReferenceRange`. Qualitative (valueText-only) results carry
|
||||
* no numeric value, so they are skipped — a delta is meaningless for them.
|
||||
*
|
||||
* Pure + Prisma-free so the present / absent states are unit-testable: it is
|
||||
* absent when there are fewer than two panel dates, no analyte is shared, or
|
||||
@@ -14,12 +16,14 @@
|
||||
* only — a delta is not a diagnosis.
|
||||
*/
|
||||
import {
|
||||
classifyReferenceRange,
|
||||
classifyAgainstEffectiveRange,
|
||||
resolveEffectiveReferenceRange,
|
||||
type ReferenceRangeStatus,
|
||||
type SourceReferenceRange,
|
||||
} from "@/lib/labs/reference-range";
|
||||
|
||||
/** A single numeric lab reading. */
|
||||
export interface LabChangeRow {
|
||||
/** A single numeric lab reading, with both windows it could be judged against. */
|
||||
export interface LabChangeRow extends SourceReferenceRange {
|
||||
analyte: string;
|
||||
unit: string;
|
||||
value: number;
|
||||
@@ -150,10 +154,13 @@ export function summariseLabChanges(
|
||||
previous: previousRow.value,
|
||||
delta,
|
||||
direction,
|
||||
status: classifyReferenceRange(
|
||||
status: classifyAgainstEffectiveRange(
|
||||
latestRow.value,
|
||||
latestRow.referenceLow,
|
||||
latestRow.referenceHigh,
|
||||
resolveEffectiveReferenceRange(
|
||||
latestRow.referenceLow,
|
||||
latestRow.referenceHigh,
|
||||
latestRow,
|
||||
),
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
classifyAgainstEffectiveRange,
|
||||
resolveEffectiveReferenceRange,
|
||||
} from "@/lib/labs/reference-range";
|
||||
|
||||
const NO_SOURCE = {
|
||||
sourceReferenceLow: null,
|
||||
sourceReferenceHigh: null,
|
||||
sourceReferenceText: null,
|
||||
};
|
||||
|
||||
describe("resolveEffectiveReferenceRange", () => {
|
||||
it("falls back to the catalog band when the reading carries no source window", () => {
|
||||
const eff = resolveEffectiveReferenceRange(3.5, 5, NO_SOURCE);
|
||||
expect(eff).toMatchObject({
|
||||
low: 3.5,
|
||||
high: 5,
|
||||
origin: "catalog",
|
||||
catalogLow: 3.5,
|
||||
catalogHigh: 5,
|
||||
sourceText: null,
|
||||
divergesFromCatalog: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("reports no window when neither side states one", () => {
|
||||
expect(resolveEffectiveReferenceRange(null, null, NO_SOURCE)).toMatchObject(
|
||||
{ low: null, high: null, origin: "none", divergesFromCatalog: false },
|
||||
);
|
||||
});
|
||||
|
||||
it("lets the source window win over the catalog band for that reading", () => {
|
||||
const eff = resolveEffectiveReferenceRange(3.5, 5, {
|
||||
sourceReferenceLow: 3.9,
|
||||
sourceReferenceHigh: 5.4,
|
||||
sourceReferenceText: "3,9 - 5,4",
|
||||
});
|
||||
expect(eff).toMatchObject({
|
||||
low: 3.9,
|
||||
high: 5.4,
|
||||
origin: "source",
|
||||
catalogLow: 3.5,
|
||||
catalogHigh: 5,
|
||||
sourceText: "3,9 - 5,4",
|
||||
divergesFromCatalog: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not call an identical window a divergence", () => {
|
||||
const eff = resolveEffectiveReferenceRange(3.5, 5, {
|
||||
sourceReferenceLow: 3.5,
|
||||
sourceReferenceHigh: 5,
|
||||
sourceReferenceText: "3,5 - 5,0",
|
||||
});
|
||||
expect(eff.origin).toBe("source");
|
||||
expect(eff.divergesFromCatalog).toBe(false);
|
||||
});
|
||||
|
||||
it("takes a one-sided source window whole, not merged with the catalog", () => {
|
||||
// A report that prints "< 5" states a ceiling and NO floor. Borrowing the
|
||||
// catalog's floor would invent a window the report never stated.
|
||||
const eff = resolveEffectiveReferenceRange(3.5, 5, {
|
||||
sourceReferenceLow: null,
|
||||
sourceReferenceHigh: 4.2,
|
||||
sourceReferenceText: "< 4,2",
|
||||
});
|
||||
expect(eff).toMatchObject({
|
||||
low: null,
|
||||
high: 4.2,
|
||||
origin: "source",
|
||||
divergesFromCatalog: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not displace the catalog for a printed window with no derivable bound", () => {
|
||||
// "negativ" is carried through verbatim, but it is not an argument for
|
||||
// discarding the catalog band.
|
||||
const eff = resolveEffectiveReferenceRange(3.5, 5, {
|
||||
sourceReferenceLow: null,
|
||||
sourceReferenceHigh: null,
|
||||
sourceReferenceText: "negativ",
|
||||
});
|
||||
expect(eff).toMatchObject({
|
||||
low: 3.5,
|
||||
high: 5,
|
||||
origin: "catalog",
|
||||
sourceText: "negativ",
|
||||
divergesFromCatalog: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("reports a source window as no divergence when the catalog has none", () => {
|
||||
const eff = resolveEffectiveReferenceRange(null, null, {
|
||||
sourceReferenceLow: 3.9,
|
||||
sourceReferenceHigh: 5.4,
|
||||
sourceReferenceText: "3,9 - 5,4",
|
||||
});
|
||||
expect(eff.origin).toBe("source");
|
||||
expect(eff.divergesFromCatalog).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("classifyAgainstEffectiveRange", () => {
|
||||
it("judges a reading by the source window, not the catalog band", () => {
|
||||
// 5.2 sits inside the catalog band (3.5–5.0 would call it high) but the
|
||||
// report that produced this value states 3.9–5.4, so it is in range.
|
||||
const eff = resolveEffectiveReferenceRange(3.5, 5, {
|
||||
sourceReferenceLow: 3.9,
|
||||
sourceReferenceHigh: 5.4,
|
||||
sourceReferenceText: "3,9 - 5,4",
|
||||
});
|
||||
expect(classifyAgainstEffectiveRange(5.2, eff)).toBe("in-range");
|
||||
expect(classifyAgainstEffectiveRange(3.7, eff)).toBe("below");
|
||||
expect(classifyAgainstEffectiveRange(5.5, eff)).toBe("above");
|
||||
});
|
||||
|
||||
it("keeps the catalog verdict for a reading with no source window", () => {
|
||||
const eff = resolveEffectiveReferenceRange(3.5, 5, NO_SOURCE);
|
||||
expect(classifyAgainstEffectiveRange(5.2, eff)).toBe("above");
|
||||
});
|
||||
|
||||
it("never classifies a qualitative reading", () => {
|
||||
const eff = resolveEffectiveReferenceRange(3.5, 5, NO_SOURCE);
|
||||
expect(classifyAgainstEffectiveRange(null, eff)).toBe("unknown");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,219 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { parseReferenceRange } from "@/lib/labs/parse-reference-range";
|
||||
|
||||
/**
|
||||
* Real-shaped strings, taken from the notation German and English lab reports
|
||||
* actually print. The parser's contract is asymmetric on purpose: deriving a
|
||||
* bound is best-effort, KEEPING the printed string is mandatory. Every case
|
||||
* below therefore asserts the verbatim text survives, and only the cases that
|
||||
* can be read without guessing assert bounds.
|
||||
*/
|
||||
describe("parseReferenceRange — two-sided windows", () => {
|
||||
it.each([
|
||||
["3.5-5.0", 3.5, 5],
|
||||
["3,5-5,0", 3.5, 5],
|
||||
["3.5 - 5.0", 3.5, 5],
|
||||
["3,5 – 5,0", 3.5, 5],
|
||||
["3,5 — 5,0", 3.5, 5],
|
||||
["3,5 bis 5,0", 3.5, 5],
|
||||
["3.5 to 5.0", 3.5, 5],
|
||||
["3,5 ... 5,0", 3.5, 5],
|
||||
["3,5 .. 5,0", 3.5, 5],
|
||||
["von 3,5 bis 5,0", 3.5, 5],
|
||||
["[3,5 - 5,0]", 3.5, 5],
|
||||
["(3.5-5.0)", 3.5, 5],
|
||||
])("reads %s as %s..%s", (raw, low, high) => {
|
||||
const parsed = parseReferenceRange(raw);
|
||||
expect(parsed).toEqual({ low, high, text: raw.trim() });
|
||||
});
|
||||
|
||||
it("reads a negative lower bound (base excess prints one)", () => {
|
||||
expect(parseReferenceRange("-2,0 bis +3,0")).toEqual({
|
||||
low: -2,
|
||||
high: 3,
|
||||
text: "-2,0 bis +3,0",
|
||||
});
|
||||
});
|
||||
|
||||
it("reads a German thousands separator as one number, not a decimal", () => {
|
||||
// Platelets print as "150.000 - 400.000 /µl" on a German report. Reading
|
||||
// the dot as a decimal point would yield a 150–400 window and silently
|
||||
// flag every normal count as wildly high.
|
||||
expect(parseReferenceRange("150.000 - 400.000")).toEqual({
|
||||
low: 150000,
|
||||
high: 400000,
|
||||
text: "150.000 - 400.000",
|
||||
});
|
||||
});
|
||||
|
||||
it("reads an English thousands separator", () => {
|
||||
expect(parseReferenceRange("150,000 - 400,000")).toEqual({
|
||||
low: 150000,
|
||||
high: 400000,
|
||||
text: "150,000 - 400,000",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses a transposed window rather than swapping it", () => {
|
||||
// Swapping would be a guess about which digit the lab meant to print.
|
||||
expect(parseReferenceRange("5,0 - 3,5")).toEqual({
|
||||
low: null,
|
||||
high: null,
|
||||
text: "5,0 - 3,5",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseReferenceRange — one-sided windows", () => {
|
||||
it.each([
|
||||
["< 5", 5],
|
||||
["<5,0", 5],
|
||||
["≤ 5", 5],
|
||||
["<= 5", 5],
|
||||
["bis 5,0", 5],
|
||||
["bis zu 5,0", 5],
|
||||
["max. 5,0", 5],
|
||||
["max 5", 5],
|
||||
["kleiner 5", 5],
|
||||
["unter 5", 5],
|
||||
["up to 5.0", 5],
|
||||
])("reads %s as an upper bound of %s", (raw, high) => {
|
||||
expect(parseReferenceRange(raw)).toEqual({
|
||||
low: null,
|
||||
high,
|
||||
text: raw.trim(),
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["> 3.5", 3.5],
|
||||
[">3,5", 3.5],
|
||||
["≥ 3,5", 3.5],
|
||||
[">= 3.5", 3.5],
|
||||
["ab 3,5", 3.5],
|
||||
["min. 3,5", 3.5],
|
||||
["mind. 3,5", 3.5],
|
||||
["mindestens 3,5", 3.5],
|
||||
["größer 3,5", 3.5],
|
||||
["groesser 3,5", 3.5],
|
||||
["über 3,5", 3.5],
|
||||
])("reads %s as a lower bound of %s", (raw, low) => {
|
||||
expect(parseReferenceRange(raw)).toEqual({
|
||||
low,
|
||||
high: null,
|
||||
text: raw.trim(),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseReferenceRange — units", () => {
|
||||
it("reads a window that carries the reading's own unit", () => {
|
||||
expect(parseReferenceRange("3,5 - 5,0 mmol/l", "mmol/L")).toEqual({
|
||||
low: 3.5,
|
||||
high: 5,
|
||||
text: "3,5 - 5,0 mmol/l",
|
||||
});
|
||||
});
|
||||
|
||||
it("reads a trailing unit when the caller states none", () => {
|
||||
expect(parseReferenceRange("0,5 - 1,2 mg/dl")).toEqual({
|
||||
low: 0.5,
|
||||
high: 1.2,
|
||||
text: "0,5 - 1,2 mg/dl",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses bounds when the printed unit contradicts the reading's unit", () => {
|
||||
// Converting mmol/L into mg/dL is interpretation and needs an analyte-
|
||||
// specific factor. Keeping the string and no bounds is the honest answer.
|
||||
const parsed = parseReferenceRange("3,5 - 5,0 mmol/l", "mg/dL");
|
||||
expect(parsed).toEqual({
|
||||
low: null,
|
||||
high: null,
|
||||
text: "3,5 - 5,0 mmol/l",
|
||||
});
|
||||
});
|
||||
|
||||
it("matches units case- and space-insensitively", () => {
|
||||
expect(parseReferenceRange("< 5 MG/DL", "mg/dl")).toEqual({
|
||||
low: null,
|
||||
high: 5,
|
||||
text: "< 5 MG/DL",
|
||||
});
|
||||
});
|
||||
|
||||
it("tolerates a percent window against a percent reading", () => {
|
||||
expect(parseReferenceRange("4,0 - 6,0 %", "%")).toEqual({
|
||||
low: 4,
|
||||
high: 6,
|
||||
text: "4,0 - 6,0 %",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseReferenceRange — text the parser must not invent bounds for", () => {
|
||||
it.each([
|
||||
"negativ",
|
||||
"negative",
|
||||
"nicht nachweisbar",
|
||||
"siehe Befund",
|
||||
"n.n.",
|
||||
"unauffällig",
|
||||
"altersabhängig",
|
||||
"methodenabhängig",
|
||||
])("keeps %s verbatim with no bounds", (raw) => {
|
||||
expect(parseReferenceRange(raw)).toEqual({
|
||||
low: null,
|
||||
high: null,
|
||||
text: raw,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps a bare number verbatim without treating it as a bound", () => {
|
||||
// "5,0" alone states neither a ceiling nor a floor; picking one would be
|
||||
// the parser deciding what the report meant.
|
||||
expect(parseReferenceRange("5,0")).toEqual({
|
||||
low: null,
|
||||
high: null,
|
||||
text: "5,0",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps a three-number string verbatim", () => {
|
||||
expect(parseReferenceRange("3,5 - 5,0 - 7,0")).toEqual({
|
||||
low: null,
|
||||
high: null,
|
||||
text: "3,5 - 5,0 - 7,0",
|
||||
});
|
||||
});
|
||||
|
||||
it("caps an over-long string rather than storing a whole paragraph", () => {
|
||||
const parsed = parseReferenceRange("x".repeat(500));
|
||||
expect(parsed?.text).toHaveLength(120);
|
||||
expect(parsed?.low).toBeNull();
|
||||
expect(parsed?.high).toBeNull();
|
||||
});
|
||||
|
||||
it("collapses internal whitespace so the stored string stays one line", () => {
|
||||
expect(parseReferenceRange(" 3,5\n\t- 5,0 ")).toEqual({
|
||||
low: 3.5,
|
||||
high: 5,
|
||||
text: "3,5 - 5,0",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseReferenceRange — absence", () => {
|
||||
it.each([null, undefined, "", " "])("returns null for %s", (raw) => {
|
||||
expect(parseReferenceRange(raw)).toBeNull();
|
||||
});
|
||||
|
||||
it("refuses a non-finite bound", () => {
|
||||
expect(parseReferenceRange("1e999 - 2e999")).toEqual({
|
||||
low: null,
|
||||
high: null,
|
||||
text: "1e999 - 2e999",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -17,6 +17,9 @@ const baseRow: LabRow = {
|
||||
unit: "mg/dl", // lowercase legacy unit
|
||||
referenceLow: null,
|
||||
referenceHigh: 100, // stale per-row bound
|
||||
sourceReferenceLow: null,
|
||||
sourceReferenceHigh: null,
|
||||
sourceReferenceText: null,
|
||||
takenAt: new Date("2026-01-01T08:00:00.000Z"),
|
||||
source: "MANUAL",
|
||||
biomarkerId: "bm_1",
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
} from "@/lib/ai/types";
|
||||
import { AI_BUDGETS } from "@/lib/ai/ai-budgets";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { parseReferenceRange } from "@/lib/labs/parse-reference-range";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import {
|
||||
extractedLabsSchema,
|
||||
@@ -42,7 +43,8 @@ Extract every analyte reading you can read. For each reading capture:
|
||||
- value: the numeric result as a number, OR null if the result is qualitative text.
|
||||
- valueText: the qualitative result text (e.g. "negativ", "positiv", "nicht nachweisbar"), OR null if the result is numeric.
|
||||
- unit: the measurement unit (e.g. "mg/dL", "%", "ng/mL"), or null if none is printed or the reading is qualitative.
|
||||
- referenceLow / referenceHigh: the reference-range bounds as numbers, or null. A one-sided range like "< 116" sets referenceHigh only; "> 40" sets referenceLow only.
|
||||
- referenceText: the reference range EXACTLY as printed beside the value, verbatim and complete, including its operator and unit if written (for example "3,5 - 5,0", "< 116", "bis 5,0", "150.000 - 400.000 /\u00b5l", "negativ"). Copy the characters; do NOT reformat, convert, or normalise them. Null only when no range is printed for that row.
|
||||
- referenceLow / referenceHigh: the SAME range reduced to numbers, when it reduces to numbers, else null. A one-sided range like "< 116" sets referenceHigh only; "> 40" sets referenceLow only.
|
||||
- takenAt: the per-row collection date in ISO 8601 (YYYY-MM-DD) if a date is printed for that row, else null.
|
||||
- confidence: an object { analyte, value, unit, range } with a 0..1 score for how legible each field was. Use a low score when a field is blurred, ambiguous, or you had to guess.
|
||||
|
||||
@@ -51,7 +53,7 @@ Set EXACTLY ONE of value / valueText per reading. Never invent a value you canno
|
||||
Also capture reportDate: the report's collection/sample date in ISO 8601 (YYYY-MM-DD), or null.
|
||||
|
||||
Respond ONLY with a JSON object of this exact shape:
|
||||
{ "reportDate": string|null, "rows": [ { "analyte": string, "value": number|null, "valueText": string|null, "unit": string|null, "referenceLow": number|null, "referenceHigh": number|null, "takenAt": string|null, "confidence": { "analyte": number, "value": number, "unit": number, "range": number } } ] }`;
|
||||
{ "reportDate": string|null, "rows": [ { "analyte": string, "value": number|null, "valueText": string|null, "unit": string|null, "referenceText": string|null, "referenceLow": number|null, "referenceHigh": number|null, "takenAt": string|null, "confidence": { "analyte": number, "value": number, "unit": number, "range": number } } ] }`;
|
||||
|
||||
const USER_PROMPT = `Transcribe the lab report in the attached image into the JSON schema described in the system prompt. Return only the JSON object.`;
|
||||
|
||||
@@ -70,7 +72,8 @@ Extract every analyte reading you can identify. For each reading capture:
|
||||
- value: the numeric result as a number, OR null if the result is qualitative text. German reports use a decimal comma (e.g. "5,4") — emit it as 5.4.
|
||||
- valueText: the qualitative result text (e.g. "negativ", "positiv", "nicht nachweisbar"), OR null if the result is numeric.
|
||||
- unit: the measurement unit (e.g. "mg/dL", "%", "ng/mL"), or null if none is printed or the reading is qualitative.
|
||||
- referenceLow / referenceHigh: the reference-range bounds as numbers, or null. A one-sided range like "< 116" sets referenceHigh only; "> 40" sets referenceLow only.
|
||||
- referenceText: the reference range EXACTLY as printed beside the value, verbatim and complete, including its operator and unit if written (for example "3,5 - 5,0", "< 116", "bis 5,0", "150.000 - 400.000 /\u00b5l", "negativ"). Copy the characters; do NOT reformat, convert, or normalise them. Null only when no range is printed for that row.
|
||||
- referenceLow / referenceHigh: the SAME range reduced to numbers, when it reduces to numbers, else null. A one-sided range like "< 116" sets referenceHigh only; "> 40" sets referenceLow only.
|
||||
- takenAt: the per-row collection date in ISO 8601 (YYYY-MM-DD) if a date is printed for that row, else null.
|
||||
- confidence: an object { analyte, value, unit, range } with a 0..1 score for how confident you are in each field. Use a LOW score when the OCR text was garbled, ambiguous, or you had to guess — the text is lossy, so be conservative.
|
||||
|
||||
@@ -79,7 +82,7 @@ Set EXACTLY ONE of value / valueText per reading. Never invent a value you canno
|
||||
Also capture reportDate: the report's collection/sample date in ISO 8601 (YYYY-MM-DD), or null.
|
||||
|
||||
Respond ONLY with a JSON object of this exact shape:
|
||||
{ "reportDate": string|null, "rows": [ { "analyte": string, "value": number|null, "valueText": string|null, "unit": string|null, "referenceLow": number|null, "referenceHigh": number|null, "takenAt": string|null, "confidence": { "analyte": number, "value": number, "unit": number, "range": number } } ] }`;
|
||||
{ "reportDate": string|null, "rows": [ { "analyte": string, "value": number|null, "valueText": string|null, "unit": string|null, "referenceText": string|null, "referenceLow": number|null, "referenceHigh": number|null, "takenAt": string|null, "confidence": { "analyte": number, "value": number, "unit": number, "range": number } } ] }`;
|
||||
|
||||
/** A normalised ISO date string (YYYY-MM-DD) or null. */
|
||||
function normaliseDate(raw: string | null): string | null {
|
||||
@@ -130,6 +133,7 @@ async function annotateRow(
|
||||
): Promise<OcrExtractedRowDto> {
|
||||
const analyte = row.analyte.trim();
|
||||
const takenAt = normaliseDate(row.takenAt) ?? reportDate;
|
||||
const printed = parseReferenceRange(row.referenceText, row.unit);
|
||||
|
||||
// Biomarker match — case-insensitive on the catalog identity, presence-only.
|
||||
const existingBiomarker = await prisma.biomarker.findFirst({
|
||||
@@ -175,8 +179,13 @@ async function annotateRow(
|
||||
value: row.value,
|
||||
valueText: row.valueText,
|
||||
unit: row.unit,
|
||||
referenceLow: row.referenceLow,
|
||||
referenceHigh: row.referenceHigh,
|
||||
// Both readings of the printed window travel to the review screen: the
|
||||
// derived bounds where the string yields them, and the string itself
|
||||
// either way. A window the parser could not read is still the window the
|
||||
// lab printed, so the user sees it and it reaches the row.
|
||||
referenceLow: printed?.low ?? row.referenceLow,
|
||||
referenceHigh: printed?.high ?? row.referenceHigh,
|
||||
referenceText: printed?.text ?? null,
|
||||
takenAt,
|
||||
confidence: row.confidence,
|
||||
biomarkerMatch: existingBiomarker ? "existing" : "new",
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
/**
|
||||
* Free-text reference-range parser for the range a lab report prints beside a
|
||||
* value.
|
||||
*
|
||||
* A report states its window in prose, not in columns: "3,5 - 5,0", "< 5",
|
||||
* "bis 5,0", "≥ 3,5", "150.000 - 400.000 /µl", or a word like "negativ" that
|
||||
* carries no window at all. The document-reading path and the manual entry
|
||||
* form both hand that string here.
|
||||
*
|
||||
* The contract is deliberately asymmetric:
|
||||
*
|
||||
* - KEEPING the printed string is mandatory. `text` always carries what the
|
||||
* report said, whitespace-normalised and length-capped, whether or not a
|
||||
* bound could be derived from it. A window this parser cannot read is not
|
||||
* a window that gets thrown away.
|
||||
* - DERIVING bounds is best-effort and refuses on ambiguity. A string that
|
||||
* does not match one of the notations below yields `low: null, high: null`
|
||||
* beside the text. Absence reads as absence — the parser never picks a
|
||||
* plausible bound out of a string it did not understand, because a fake
|
||||
* window silently reclassifies a reading as normal or abnormal.
|
||||
*
|
||||
* Notations read (German and English, comma- or dot-decimal):
|
||||
*
|
||||
* two-sided 3.5-5.0 · 3,5 – 5,0 · 3,5 bis 5,0 · 3.5 to 5.0 · 3,5 ... 5,0
|
||||
* von 3,5 bis 5,0 · [3,5 - 5,0] · (3.5-5.0)
|
||||
* upper only < 5 · ≤ 5 · <= 5 · bis 5,0 · bis zu 5,0 · max. 5,0 ·
|
||||
* kleiner 5 · unter 5 · up to 5.0 · less than 5
|
||||
* lower only > 3.5 · ≥ 3,5 · >= 3.5 · ab 3,5 · min. 3,5 · mindestens 3,5 ·
|
||||
* größer 3,5 · über 3,5 · at least 3.5
|
||||
*
|
||||
* Units are read, never converted. A window may carry the unit the report
|
||||
* printed it in; when the caller states the reading's own unit and the two
|
||||
* disagree, the bounds are refused and only the text survives. Converting
|
||||
* mmol/L into mg/dL needs a per-analyte factor, so doing it here would be the
|
||||
* parser deciding what the report meant.
|
||||
*/
|
||||
|
||||
/** What a printed range yielded. `text` is set whenever the input was non-empty. */
|
||||
export interface ParsedReferenceRange {
|
||||
/** Lower bound, or null when the notation states none / could not be read. */
|
||||
low: number | null;
|
||||
/** Upper bound, or null when the notation states none / could not be read. */
|
||||
high: number | null;
|
||||
/** The printed string, whitespace-normalised and capped. Never null here. */
|
||||
text: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Storage cap for the verbatim string. A reference-range cell is a few
|
||||
* characters; anything longer is a mis-segmented paragraph, and truncating it
|
||||
* keeps a runaway transcription out of the column.
|
||||
*/
|
||||
export const SOURCE_REFERENCE_TEXT_MAX = 120;
|
||||
|
||||
/** Magnitude ceiling. Beyond this a "bound" is a transcription artefact. */
|
||||
const MAX_ABS_BOUND = 1e12;
|
||||
|
||||
/** One number token: optional sign, digits, any number of `.`/`,` groups. */
|
||||
const NUM = String.raw`[+-]?\d+(?:[.,]\d+)*`;
|
||||
|
||||
/** Infix separators for a two-sided window. */
|
||||
const TWO_SIDED_SEP = String.raw`(?:-|–|—|‒|…|\.{2,3}|bis\s+zu|bis|to|until)`;
|
||||
|
||||
/** Prefixes that state a ceiling only. */
|
||||
const UPPER_PREFIX = String.raw`(?:<=|<|≤|bis\s+zu|bis|max\.|maximal|max|kleiner\s+als|kleiner|unter|höchstens|hoechstens|up\s+to|less\s+than|below)`;
|
||||
|
||||
/** Prefixes that state a floor only. */
|
||||
const LOWER_PREFIX = String.raw`(?:>=|>|≥|ab|mind\.|mindestens|minimal|min\.|min|größer\s+als|größer|groesser\s+als|groesser|über|ueber|at\s+least|greater\s+than|above)`;
|
||||
|
||||
/**
|
||||
* Trailing unit tail. Deliberately whitespace-free and comma-free: a real unit
|
||||
* symbol ("mmol/l", "mg/dL", "%", "/µl", "10^9/L") never contains either, and
|
||||
* refusing anything that does keeps a third number ("3,5 - 5,0 - 7,0") from
|
||||
* being swallowed as if it were a unit.
|
||||
*/
|
||||
const UNIT_TAIL = String.raw`(?:\s*([^\s,]{1,24}))?`;
|
||||
|
||||
const TWO_SIDED_RE = new RegExp(
|
||||
`^(${NUM})\\s*${TWO_SIDED_SEP}\\s*(${NUM})${UNIT_TAIL}$`,
|
||||
"iu",
|
||||
);
|
||||
const UPPER_RE = new RegExp(`^${UPPER_PREFIX}\\s*(${NUM})${UNIT_TAIL}$`, "iu");
|
||||
const LOWER_RE = new RegExp(`^${LOWER_PREFIX}\\s*(${NUM})${UNIT_TAIL}$`, "iu");
|
||||
|
||||
/** Leading words that only introduce the window ("von 3,5 bis 5,0"). */
|
||||
const LEADING_FILLER_RE = /^(?:von|from|ref\.?|referenz|reference)\s+/iu;
|
||||
|
||||
/**
|
||||
* Read one number token into a JS number, or null when its separators are
|
||||
* ambiguous.
|
||||
*
|
||||
* Separator rules, in order:
|
||||
* 1. Both `.` and `,` present → the LAST one is the decimal mark, the other
|
||||
* groups thousands ("1.234,56" and "1,234.56" both read as 1234.56).
|
||||
* 2. One separator kind, appearing more than once → all thousands
|
||||
* ("1.234.567").
|
||||
* 3. One separator, appearing once, group ≠ 3 digits → decimal ("3,5").
|
||||
* 4. One separator, appearing once, group = 3 digits → ambiguous in the
|
||||
* general case. Read as thousands only when the group is "000", which is
|
||||
* how a German report prints a rounded count ("150.000 - 400.000 /µl");
|
||||
* reading that as 150 would flag every normal platelet count as wildly
|
||||
* high. Any other 3-digit group ("1.234" — 1234 or 1.234?) is refused.
|
||||
*
|
||||
* The limit of rule 4 is real: a report that prints "5.000" meaning five with
|
||||
* three decimal places is read as five thousand. The verbatim text is stored
|
||||
* either way, so the printed window is never lost — only the derived bound is.
|
||||
*/
|
||||
function readNumber(token: string): number | null {
|
||||
const sign = token.startsWith("-") ? -1 : 1;
|
||||
const body = token.replace(/^[+-]/u, "");
|
||||
|
||||
const dots = (body.match(/\./gu) ?? []).length;
|
||||
const commas = (body.match(/,/gu) ?? []).length;
|
||||
|
||||
let normalised: string;
|
||||
if (dots > 0 && commas > 0) {
|
||||
const decimalMark =
|
||||
body.lastIndexOf(".") > body.lastIndexOf(",") ? "." : ",";
|
||||
const grouping = decimalMark === "." ? "," : ".";
|
||||
normalised = body.split(grouping).join("").replace(decimalMark, ".");
|
||||
} else if (dots + commas === 0) {
|
||||
normalised = body;
|
||||
} else if (dots > 1 || commas > 1) {
|
||||
normalised = body.replace(/[.,]/gu, "");
|
||||
} else {
|
||||
const idx = body.search(/[.,]/u);
|
||||
const group = body.slice(idx + 1);
|
||||
if (group.length !== 3) {
|
||||
normalised = `${body.slice(0, idx)}.${group}`;
|
||||
} else if (group === "000") {
|
||||
normalised = body.slice(0, idx) + group;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const value = sign * Number(normalised);
|
||||
if (!Number.isFinite(value) || Math.abs(value) > MAX_ABS_BOUND) return null;
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Compare a printed unit against the reading's unit, ignoring case + spaces. */
|
||||
function unitsAgree(printed: string | undefined, reading: string | undefined) {
|
||||
if (!printed) return true;
|
||||
if (!reading || !reading.trim()) return true;
|
||||
const norm = (u: string) => u.replace(/\s+/gu, "").toLowerCase();
|
||||
return norm(printed) === norm(reading);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the reference range a lab report printed for one reading.
|
||||
*
|
||||
* @param raw the printed string, exactly as transcribed.
|
||||
* @param unit the reading's own unit, when known. A printed window in a
|
||||
* different unit yields text-only — never a converted bound.
|
||||
* @returns null when there was nothing printed at all.
|
||||
*/
|
||||
export function parseReferenceRange(
|
||||
raw: string | null | undefined,
|
||||
unit?: string | null,
|
||||
): ParsedReferenceRange | null {
|
||||
if (raw === null || raw === undefined) return null;
|
||||
const text = raw
|
||||
.replace(/\s+/gu, " ")
|
||||
.trim()
|
||||
.slice(0, SOURCE_REFERENCE_TEXT_MAX);
|
||||
if (text === "") return null;
|
||||
|
||||
const noBounds: ParsedReferenceRange = { low: null, high: null, text };
|
||||
|
||||
// Strip a bracket wrapper and an introducing word for MATCHING only — the
|
||||
// stored text stays exactly what the report printed.
|
||||
let candidate = text;
|
||||
const bracketed = /^[[(](.*)[\])]$/u.exec(candidate);
|
||||
if (bracketed) candidate = bracketed[1].trim();
|
||||
candidate = candidate.replace(LEADING_FILLER_RE, "").trim();
|
||||
|
||||
const two = TWO_SIDED_RE.exec(candidate);
|
||||
if (two) {
|
||||
if (!unitsAgree(two[3], unit ?? undefined)) return noBounds;
|
||||
const low = readNumber(two[1]);
|
||||
const high = readNumber(two[2]);
|
||||
if (low === null || high === null) return noBounds;
|
||||
// A transposed window is a transcription error, and swapping it would be a
|
||||
// guess about which digit the lab meant. Refuse the bounds, keep the text.
|
||||
if (low > high) return noBounds;
|
||||
return { low, high, text };
|
||||
}
|
||||
|
||||
const upper = UPPER_RE.exec(candidate);
|
||||
if (upper) {
|
||||
if (!unitsAgree(upper[2], unit ?? undefined)) return noBounds;
|
||||
const high = readNumber(upper[1]);
|
||||
return high === null ? noBounds : { low: null, high, text };
|
||||
}
|
||||
|
||||
const lower = LOWER_RE.exec(candidate);
|
||||
if (lower) {
|
||||
if (!unitsAgree(lower[2], unit ?? undefined)) return noBounds;
|
||||
const low = readNumber(lower[1]);
|
||||
return low === null ? noBounds : { low, high: null, text };
|
||||
}
|
||||
|
||||
// Anything else — "negativ", "siehe Befund", a bare number, prose — keeps its
|
||||
// text and states no window.
|
||||
return noBounds;
|
||||
}
|
||||
@@ -61,3 +61,113 @@ export function formatReferenceRange(
|
||||
if (hasLow) return `≥ ${formatNumber(low as number)}`;
|
||||
return opts?.emptyText ?? "";
|
||||
}
|
||||
|
||||
/**
|
||||
* The reference range a lab report printed beside ONE reading, as stored on
|
||||
* `LabResult.sourceReference*`.
|
||||
*/
|
||||
export interface SourceReferenceRange {
|
||||
sourceReferenceLow: number | null;
|
||||
sourceReferenceHigh: number | null;
|
||||
sourceReferenceText: string | null;
|
||||
}
|
||||
|
||||
/** Where the bounds a reading is judged against came from. */
|
||||
export type ReferenceRangeOrigin = "source" | "catalog" | "none";
|
||||
|
||||
/** The bounds one reading is judged against, plus how they were arrived at. */
|
||||
export interface EffectiveReferenceRange {
|
||||
/** The bound the verdict uses. */
|
||||
low: number | null;
|
||||
high: number | null;
|
||||
origin: ReferenceRangeOrigin;
|
||||
/** The catalog band, kept alongside so a surface can show both. */
|
||||
catalogLow: number | null;
|
||||
catalogHigh: number | null;
|
||||
/** The printed window, kept verbatim. Null when the report stated none. */
|
||||
sourceText: string | null;
|
||||
/**
|
||||
* True when the source window is in force AND the catalog states a window
|
||||
* that differs from it. This is the case a reader must be told about: the
|
||||
* same number reads differently against the two, so a surface that shows
|
||||
* only one of them is showing a partial answer.
|
||||
*/
|
||||
divergesFromCatalog: boolean;
|
||||
}
|
||||
|
||||
/** Two bounds are the same window when both ends match (null included). */
|
||||
function sameWindow(
|
||||
aLow: number | null,
|
||||
aHigh: number | null,
|
||||
bLow: number | null,
|
||||
bHigh: number | null,
|
||||
): boolean {
|
||||
return aLow === bLow && aHigh === bHigh;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the reference window ONE reading is judged against.
|
||||
*
|
||||
* A lab runs its own method on its own device, and the physician reads the
|
||||
* value against the window printed on that report. So for a reading that
|
||||
* carries a source window, the source window wins — for that reading alone.
|
||||
* The catalog band stays the net for every reading that carries none.
|
||||
*
|
||||
* A source window counts as stating a window only when it yields at least one
|
||||
* numeric bound. A printed string with no derivable bound ("negativ", "siehe
|
||||
* Befund") is carried through as `sourceText` but never displaces the catalog:
|
||||
* a range that could not be read is not an argument for having no range.
|
||||
*
|
||||
* This function is the ONLY place that precedence is expressed. Every surface
|
||||
* that classifies a reading — the API DTO, the doctor report, insights, the
|
||||
* coach snapshot, the MCP read, the FHIR export — routes through it, so a
|
||||
* reading cannot read as in-range on one surface and out on another.
|
||||
*/
|
||||
export function resolveEffectiveReferenceRange(
|
||||
catalogLow: number | null | undefined,
|
||||
catalogHigh: number | null | undefined,
|
||||
source: SourceReferenceRange | null | undefined,
|
||||
): EffectiveReferenceRange {
|
||||
const cLow = catalogLow ?? null;
|
||||
const cHigh = catalogHigh ?? null;
|
||||
const sLow = source?.sourceReferenceLow ?? null;
|
||||
const sHigh = source?.sourceReferenceHigh ?? null;
|
||||
const sourceText = source?.sourceReferenceText ?? null;
|
||||
|
||||
if (sLow !== null || sHigh !== null) {
|
||||
return {
|
||||
low: sLow,
|
||||
high: sHigh,
|
||||
origin: "source",
|
||||
catalogLow: cLow,
|
||||
catalogHigh: cHigh,
|
||||
sourceText,
|
||||
divergesFromCatalog:
|
||||
(cLow !== null || cHigh !== null) &&
|
||||
!sameWindow(sLow, sHigh, cLow, cHigh),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
low: cLow,
|
||||
high: cHigh,
|
||||
origin: cLow !== null || cHigh !== null ? "catalog" : "none",
|
||||
catalogLow: cLow,
|
||||
catalogHigh: cHigh,
|
||||
sourceText,
|
||||
divergesFromCatalog: false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify one reading against its effective window. Qualitative readings
|
||||
* (no numeric value) always report `"unknown"` — there is nothing to compare
|
||||
* against bounds, and a fabricated verdict is worse than none.
|
||||
*/
|
||||
export function classifyAgainstEffectiveRange(
|
||||
value: number | null,
|
||||
effective: EffectiveReferenceRange,
|
||||
): ReferenceRangeStatus {
|
||||
if (value === null) return "unknown";
|
||||
return classifyReferenceRange(value, effective.low, effective.high);
|
||||
}
|
||||
+138
-58
@@ -10,13 +10,40 @@
|
||||
* - If the row is unlinked (legacy / pre-backfill), it falls back to its own
|
||||
* free-text `analyte` / `unit` / `reference*` columns.
|
||||
*
|
||||
* Either way the verdict (`rangeStatus`) is computed from the RESOLVED bounds
|
||||
* via `classifyReferenceRange`. The DTO carries the resolved values so the web
|
||||
* client AND the iOS client render the same numbers — neither recomputes the
|
||||
* range or guesses the unit. This is the server-authoritative-parity rule for
|
||||
* Labs.
|
||||
* On top of that the reading's OWN printed window wins. A lab report states
|
||||
* the range its method and device produce right beside the value, and that is
|
||||
* the window a physician evaluates against; the catalog band is the net for
|
||||
* readings that carry none. `resolveEffectiveReferenceRange` owns that
|
||||
* precedence and this file is one of its callers, not a second copy of it.
|
||||
*
|
||||
* The DTO therefore carries three windows and says which one is in force:
|
||||
*
|
||||
* - `referenceLow` / `referenceHigh` — the EFFECTIVE bounds, the ones
|
||||
* `rangeStatus` was computed from. A client that reads only these fields
|
||||
* (including an iOS build that predates the source window) renders the
|
||||
* right numbers and the right verdict with no change of its own.
|
||||
* - `catalogReferenceLow` / `catalogReferenceHigh` — the catalog band, so a
|
||||
* surface can paint both and show where they part.
|
||||
* - `sourceReferenceLow` / `sourceReferenceHigh` / `sourceReferenceText` —
|
||||
* what the report printed, the last verbatim.
|
||||
*
|
||||
* `referenceOrigin` names which window is in force and
|
||||
* `referenceDivergesFromCatalog` flags the case a reader must be told about.
|
||||
* Every value is resolved server-side; neither client recomputes a range,
|
||||
* guesses a unit, or re-derives the verdict.
|
||||
*
|
||||
* The two serialisers below spell every field out rather than spreading a
|
||||
* shared helper: `written-outcome-response-consumer-guard` walks these object
|
||||
* literals to enumerate the response leaves and pair each with a client
|
||||
* reader, and a spread of a function call hides them from it. A field added
|
||||
* here must be visible there.
|
||||
*/
|
||||
import { classifyReferenceRange } from "@/lib/labs/reference-range";
|
||||
import {
|
||||
classifyAgainstEffectiveRange,
|
||||
resolveEffectiveReferenceRange,
|
||||
type EffectiveReferenceRange,
|
||||
type ReferenceRangeOrigin,
|
||||
} from "@/lib/labs/reference-range";
|
||||
|
||||
/** The minimal biomarker shape the resolver needs (no encrypted context). */
|
||||
export interface ResolvedBiomarker {
|
||||
@@ -40,6 +67,10 @@ export interface LabRow {
|
||||
unit: string;
|
||||
referenceLow: number | null;
|
||||
referenceHigh: number | null;
|
||||
/** The window the source report printed for THIS reading. */
|
||||
sourceReferenceLow: number | null;
|
||||
sourceReferenceHigh: number | null;
|
||||
sourceReferenceText: string | null;
|
||||
takenAt: Date;
|
||||
source: string;
|
||||
biomarkerId: string | null;
|
||||
@@ -48,60 +79,95 @@ export interface LabRow {
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the canonical name / unit / bounds / panel for a lab row, preferring
|
||||
* the linked biomarker over the legacy per-row fields. Returns a plain shape
|
||||
* the list + detail serialisers both build on.
|
||||
*/
|
||||
export function resolveLabFields(
|
||||
row: Pick<
|
||||
LabRow,
|
||||
| "analyte"
|
||||
| "unit"
|
||||
| "referenceLow"
|
||||
| "referenceHigh"
|
||||
| "panel"
|
||||
| "biomarkerId"
|
||||
>,
|
||||
biomarker: ResolvedBiomarker | null | undefined,
|
||||
): {
|
||||
/** The columns `resolveLabFields` needs off a lab row. */
|
||||
export type LabFieldRow = Pick<
|
||||
LabRow,
|
||||
| "analyte"
|
||||
| "unit"
|
||||
| "referenceLow"
|
||||
| "referenceHigh"
|
||||
| "sourceReferenceLow"
|
||||
| "sourceReferenceHigh"
|
||||
| "sourceReferenceText"
|
||||
| "panel"
|
||||
| "biomarkerId"
|
||||
>;
|
||||
|
||||
/** What every lab surface reads off a row: identity, unit, and one window. */
|
||||
export interface ResolvedLabFields {
|
||||
analyte: string;
|
||||
unit: string;
|
||||
panel: string | null;
|
||||
/** The bounds in force for this reading — source window first, catalog next. */
|
||||
referenceLow: number | null;
|
||||
referenceHigh: number | null;
|
||||
panel: string | null;
|
||||
} {
|
||||
if (biomarker) {
|
||||
return {
|
||||
analyte: biomarker.name,
|
||||
unit: biomarker.unit,
|
||||
referenceLow: biomarker.lowerBound,
|
||||
referenceHigh: biomarker.upperBound,
|
||||
panel: biomarker.panel,
|
||||
};
|
||||
}
|
||||
return {
|
||||
analyte: row.analyte,
|
||||
unit: row.unit,
|
||||
referenceLow: row.referenceLow,
|
||||
referenceHigh: row.referenceHigh,
|
||||
panel: row.panel,
|
||||
};
|
||||
/** The catalog band, so a surface can show both windows. */
|
||||
catalogReferenceLow: number | null;
|
||||
catalogReferenceHigh: number | null;
|
||||
/** What the source report printed for this reading. */
|
||||
sourceReferenceLow: number | null;
|
||||
sourceReferenceHigh: number | null;
|
||||
sourceReferenceText: string | null;
|
||||
referenceOrigin: ReferenceRangeOrigin;
|
||||
referenceDivergesFromCatalog: boolean;
|
||||
/** The resolved window as one value, for callers that pass it on. */
|
||||
effectiveRange: EffectiveReferenceRange;
|
||||
}
|
||||
|
||||
/**
|
||||
* v1.18.9 — the reference-range verdict for a row. A qualitative row (no
|
||||
* numeric `value`) has nothing to compare against the bounds, so it always
|
||||
* reports `"unknown"` — the neutral, no-verdict state — never a fabricated
|
||||
* in/out classification. A numeric row classifies against the resolved bounds.
|
||||
* Resolve the canonical name / unit / window / panel for a lab row.
|
||||
*
|
||||
* Two resolutions happen here, in this order:
|
||||
*
|
||||
* 1. Identity + catalog band: the linked biomarker wins over the row's legacy
|
||||
* free-text columns (an unlinked pre-backfill row falls back to its own).
|
||||
* 2. The window in force: the reading's printed source range wins over the
|
||||
* catalog band, through `resolveEffectiveReferenceRange`.
|
||||
*
|
||||
* Every lab surface calls this, so the window a reading is judged against is
|
||||
* the same on the API, the doctor report, insights, the coach, MCP and FHIR.
|
||||
*/
|
||||
function rowRangeStatus(
|
||||
value: number | null,
|
||||
referenceLow: number | null,
|
||||
referenceHigh: number | null,
|
||||
) {
|
||||
if (value === null) return "unknown" as const;
|
||||
return classifyReferenceRange(value, referenceLow, referenceHigh);
|
||||
export function resolveLabFields(
|
||||
row: LabFieldRow,
|
||||
biomarker: ResolvedBiomarker | null | undefined,
|
||||
): ResolvedLabFields {
|
||||
const identity = biomarker
|
||||
? {
|
||||
analyte: biomarker.name,
|
||||
unit: biomarker.unit,
|
||||
panel: biomarker.panel,
|
||||
catalogLow: biomarker.lowerBound,
|
||||
catalogHigh: biomarker.upperBound,
|
||||
}
|
||||
: {
|
||||
analyte: row.analyte,
|
||||
unit: row.unit,
|
||||
panel: row.panel,
|
||||
catalogLow: row.referenceLow,
|
||||
catalogHigh: row.referenceHigh,
|
||||
};
|
||||
|
||||
const effectiveRange = resolveEffectiveReferenceRange(
|
||||
identity.catalogLow,
|
||||
identity.catalogHigh,
|
||||
row,
|
||||
);
|
||||
|
||||
return {
|
||||
analyte: identity.analyte,
|
||||
unit: identity.unit,
|
||||
panel: identity.panel,
|
||||
referenceLow: effectiveRange.low,
|
||||
referenceHigh: effectiveRange.high,
|
||||
catalogReferenceLow: effectiveRange.catalogLow,
|
||||
catalogReferenceHigh: effectiveRange.catalogHigh,
|
||||
sourceReferenceLow: row.sourceReferenceLow,
|
||||
sourceReferenceHigh: row.sourceReferenceHigh,
|
||||
sourceReferenceText: effectiveRange.sourceText,
|
||||
referenceOrigin: effectiveRange.origin,
|
||||
referenceDivergesFromCatalog: effectiveRange.divergesFromCatalog,
|
||||
effectiveRange,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -123,13 +189,21 @@ export function serialiseLabResult(
|
||||
unit: resolved.unit,
|
||||
referenceLow: resolved.referenceLow,
|
||||
referenceHigh: resolved.referenceHigh,
|
||||
catalogReferenceLow: resolved.catalogReferenceLow,
|
||||
catalogReferenceHigh: resolved.catalogReferenceHigh,
|
||||
sourceReferenceLow: resolved.sourceReferenceLow,
|
||||
sourceReferenceHigh: resolved.sourceReferenceHigh,
|
||||
sourceReferenceText: resolved.sourceReferenceText,
|
||||
referenceOrigin: resolved.referenceOrigin,
|
||||
referenceDivergesFromCatalog: resolved.referenceDivergesFromCatalog,
|
||||
takenAt: row.takenAt.toISOString(),
|
||||
source: row.source,
|
||||
hasNote: row.noteEncrypted !== null,
|
||||
rangeStatus: rowRangeStatus(
|
||||
// A qualitative row has no number to place against bounds, so it reports
|
||||
// the neutral "unknown" rather than a fabricated in/out verdict.
|
||||
rangeStatus: classifyAgainstEffectiveRange(
|
||||
row.value,
|
||||
resolved.referenceLow,
|
||||
resolved.referenceHigh,
|
||||
resolved.effectiveRange,
|
||||
),
|
||||
createdAt: row.createdAt.toISOString(),
|
||||
updatedAt: row.updatedAt.toISOString(),
|
||||
@@ -156,13 +230,19 @@ export function serialiseLabResultDetail(
|
||||
unit: resolved.unit,
|
||||
referenceLow: resolved.referenceLow,
|
||||
referenceHigh: resolved.referenceHigh,
|
||||
catalogReferenceLow: resolved.catalogReferenceLow,
|
||||
catalogReferenceHigh: resolved.catalogReferenceHigh,
|
||||
sourceReferenceLow: resolved.sourceReferenceLow,
|
||||
sourceReferenceHigh: resolved.sourceReferenceHigh,
|
||||
sourceReferenceText: resolved.sourceReferenceText,
|
||||
referenceOrigin: resolved.referenceOrigin,
|
||||
referenceDivergesFromCatalog: resolved.referenceDivergesFromCatalog,
|
||||
takenAt: row.takenAt.toISOString(),
|
||||
source: row.source,
|
||||
note,
|
||||
rangeStatus: rowRangeStatus(
|
||||
rangeStatus: classifyAgainstEffectiveRange(
|
||||
row.value,
|
||||
resolved.referenceLow,
|
||||
resolved.referenceHigh,
|
||||
resolved.effectiveRange,
|
||||
),
|
||||
createdAt: row.createdAt.toISOString(),
|
||||
updatedAt: row.updatedAt.toISOString(),
|
||||
|
||||
@@ -48,7 +48,7 @@ import {
|
||||
type MetricStatusMetricId,
|
||||
} from "@/lib/insights/metric-status-registry";
|
||||
import { allSignals, getSignal } from "@/lib/signals/registry";
|
||||
import { classifyReferenceRange } from "@/lib/labs/reference-range";
|
||||
import { classifyAgainstEffectiveRange } from "@/lib/labs/reference-range";
|
||||
import { resolveLabFields } from "@/lib/labs/serialise";
|
||||
import { sanitizeValueText } from "@/lib/ai/coach/labs-snapshot";
|
||||
import { encodeOffsetCursor } from "@/lib/mcp/pagination";
|
||||
@@ -1316,6 +1316,9 @@ export async function getLabHistory(
|
||||
unit: true,
|
||||
referenceLow: true,
|
||||
referenceHigh: true,
|
||||
sourceReferenceLow: true,
|
||||
sourceReferenceHigh: true,
|
||||
sourceReferenceText: true,
|
||||
takenAt: true,
|
||||
biomarkerId: true,
|
||||
biomarker: {
|
||||
@@ -1350,14 +1353,10 @@ export async function getLabHistory(
|
||||
unit: resolved.unit,
|
||||
referenceLow: resolved.referenceLow,
|
||||
referenceHigh: resolved.referenceHigh,
|
||||
rangeStatus:
|
||||
row.value === null
|
||||
? "unknown"
|
||||
: classifyReferenceRange(
|
||||
row.value,
|
||||
resolved.referenceLow,
|
||||
resolved.referenceHigh,
|
||||
),
|
||||
rangeStatus: classifyAgainstEffectiveRange(
|
||||
row.value,
|
||||
resolved.effectiveRange,
|
||||
),
|
||||
takenAt: row.takenAt.toISOString(),
|
||||
};
|
||||
});
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
/**
|
||||
* The ingest-side plausibility gate.
|
||||
*
|
||||
* `VALUE_RANGES` is the band the application declares a metric can occupy, and
|
||||
* every path a person can drive has always enforced it. The provider sync
|
||||
* writers did not, which is how a row the application itself calls impossible
|
||||
* reached the table and poisoned every mean and personal band derived from it.
|
||||
* These cases pin the two halves of the fix: the refusal, and the fact that a
|
||||
* refusal is never silent.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { WideEventBuilder } from "@/lib/logging/event-builder";
|
||||
import { eventStorage } from "@/lib/logging/context";
|
||||
import {
|
||||
RANGE_REJECTED_META_KEY,
|
||||
classifyRangeRejection,
|
||||
dropImplausibleMeasurements,
|
||||
recordRangeRejections,
|
||||
type RangeRejectionTally,
|
||||
} from "../plausibility-gate";
|
||||
|
||||
let event: WideEventBuilder;
|
||||
|
||||
function withEvent<T>(run: () => T): T {
|
||||
return eventStorage.run(event, run);
|
||||
}
|
||||
|
||||
function tally(): RangeRejectionTally | undefined {
|
||||
return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as
|
||||
RangeRejectionTally | undefined;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
event = new WideEventBuilder("background");
|
||||
});
|
||||
|
||||
describe("classifyRangeRejection", () => {
|
||||
it("passes a value inside the metric's declared band", () => {
|
||||
// PULSE is declared 20–300; both edges are inclusive.
|
||||
expect(classifyRangeRejection("PULSE", 60)).toBeNull();
|
||||
expect(classifyRangeRejection("PULSE", 20)).toBeNull();
|
||||
expect(classifyRangeRejection("PULSE", 300)).toBeNull();
|
||||
});
|
||||
|
||||
it("names the edge an out-of-band value broke", () => {
|
||||
expect(classifyRangeRejection("PULSE", 55643821.505)).toEqual({
|
||||
type: "PULSE",
|
||||
direction: "above_max",
|
||||
});
|
||||
expect(classifyRangeRejection("PULSE", 19)).toEqual({
|
||||
type: "PULSE",
|
||||
direction: "below_min",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses a non-finite value whatever the metric", () => {
|
||||
expect(classifyRangeRejection("PULSE", Number.NaN)).toEqual({
|
||||
type: "PULSE",
|
||||
direction: "not_finite",
|
||||
});
|
||||
// No declared band, so the value itself is the only thing to judge.
|
||||
expect(
|
||||
classifyRangeRejection("NOT_A_METRIC", Number.POSITIVE_INFINITY),
|
||||
).toEqual({ type: "NOT_A_METRIC", direction: "not_finite" });
|
||||
});
|
||||
|
||||
it("passes a metric with no declared band — an absent range is an absent fact", () => {
|
||||
expect(classifyRangeRejection("NOT_A_METRIC", 9_999_999)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("dropImplausibleMeasurements", () => {
|
||||
it("keeps the in-band rows and returns them untouched", () => {
|
||||
const rows = [
|
||||
{ type: "PULSE", value: 60 },
|
||||
{ type: "WEIGHT", value: 82.4 },
|
||||
];
|
||||
const kept = withEvent(() =>
|
||||
dropImplausibleMeasurements("withings", rows, (r) => r),
|
||||
);
|
||||
|
||||
expect(kept).toEqual(rows);
|
||||
expect(kept[0]).toBe(rows[0]);
|
||||
// Nothing was refused, so the event carries no tally at all — absence
|
||||
// reads as absence.
|
||||
expect(tally()).toBeUndefined();
|
||||
expect(event.toJSON().warnings).toBeUndefined();
|
||||
});
|
||||
|
||||
it("drops the out-of-band rows and tallies them on the wide event", () => {
|
||||
const kept = withEvent(() =>
|
||||
dropImplausibleMeasurements(
|
||||
"withings",
|
||||
[
|
||||
{ type: "PULSE", value: 60 },
|
||||
{ type: "PULSE", value: 55643821.505 },
|
||||
{ type: "PULSE", value: 900 },
|
||||
{ type: "WEIGHT", value: 0.2 },
|
||||
],
|
||||
(r) => r,
|
||||
),
|
||||
);
|
||||
|
||||
expect(kept).toEqual([{ type: "PULSE", value: 60 }]);
|
||||
|
||||
const recorded = tally();
|
||||
expect(recorded).toEqual({
|
||||
total: 3,
|
||||
buckets: [
|
||||
{
|
||||
source: "withings",
|
||||
type: "PULSE",
|
||||
direction: "above_max",
|
||||
count: 2,
|
||||
},
|
||||
{
|
||||
source: "withings",
|
||||
type: "WEIGHT",
|
||||
direction: "below_min",
|
||||
count: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the refused readings out of the log line", () => {
|
||||
withEvent(() =>
|
||||
dropImplausibleMeasurements(
|
||||
"withings",
|
||||
[{ type: "PULSE", value: 55643821.505 }],
|
||||
(r) => r,
|
||||
),
|
||||
);
|
||||
|
||||
// The value is a health reading. The tally says which metric, which
|
||||
// provider and which edge — never the number itself.
|
||||
const serialised = JSON.stringify(event.toJSON());
|
||||
expect(serialised).not.toContain("55643821");
|
||||
});
|
||||
|
||||
it("lifts the event to warn once per provider", () => {
|
||||
withEvent(() => {
|
||||
dropImplausibleMeasurements(
|
||||
"withings",
|
||||
[{ type: "PULSE", value: 900 }],
|
||||
(r) => r,
|
||||
);
|
||||
dropImplausibleMeasurements(
|
||||
"withings",
|
||||
[{ type: "PULSE", value: 901 }],
|
||||
(r) => r,
|
||||
);
|
||||
dropImplausibleMeasurements(
|
||||
"fitbit",
|
||||
[{ type: "PULSE", value: 902 }],
|
||||
(r) => r,
|
||||
);
|
||||
});
|
||||
|
||||
expect(event.getLevel()).toBe("warn");
|
||||
expect(event.toJSON().warnings).toHaveLength(2);
|
||||
expect(event.toJSON().warnings?.[0]).toContain("withings");
|
||||
expect(event.toJSON().warnings?.[1]).toContain("fitbit");
|
||||
});
|
||||
|
||||
it("accumulates across calls instead of overwriting the previous tally", () => {
|
||||
// Fitbit gates once per resource leg, and the reconciler gates once per
|
||||
// row: a tally that replaced rather than merged would report only the
|
||||
// last batch, and a run that dropped hundreds would read as dropping one.
|
||||
withEvent(() => {
|
||||
dropImplausibleMeasurements(
|
||||
"fitbit",
|
||||
[{ type: "PULSE", value: 900 }],
|
||||
(r) => r,
|
||||
);
|
||||
dropImplausibleMeasurements(
|
||||
"fitbit",
|
||||
[
|
||||
{ type: "PULSE", value: 901 },
|
||||
{ type: "OXYGEN_SATURATION", value: 4 },
|
||||
],
|
||||
(r) => r,
|
||||
);
|
||||
});
|
||||
|
||||
expect(tally()).toEqual({
|
||||
total: 3,
|
||||
buckets: [
|
||||
{
|
||||
source: "fitbit",
|
||||
type: "OXYGEN_SATURATION",
|
||||
direction: "below_min",
|
||||
count: 1,
|
||||
},
|
||||
{ source: "fitbit", type: "PULSE", direction: "above_max", count: 2 },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("still drops outside a wide-event context", () => {
|
||||
// A sync running without an ambient event has nowhere to report. The row
|
||||
// is refused anyway: observability is what is lost, never the guarantee.
|
||||
const kept = dropImplausibleMeasurements(
|
||||
"nightscout",
|
||||
[
|
||||
{ type: "BLOOD_GLUCOSE", value: 110 },
|
||||
{ type: "BLOOD_GLUCOSE", value: 9_000 },
|
||||
],
|
||||
(r) => r,
|
||||
);
|
||||
expect(kept).toEqual([{ type: "BLOOD_GLUCOSE", value: 110 }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("recordRangeRejections", () => {
|
||||
it("writes nothing when there is nothing to report", () => {
|
||||
withEvent(() => recordRangeRejections("withings", []));
|
||||
expect(tally()).toBeUndefined();
|
||||
expect(event.getLevel()).toBe("info");
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,9 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { Prisma } from "@/generated/prisma/client";
|
||||
import { eventStorage } from "@/lib/logging/context";
|
||||
import { WideEventBuilder } from "@/lib/logging/event-builder";
|
||||
import { RANGE_REJECTED_META_KEY } from "../plausibility-gate";
|
||||
import {
|
||||
reconcileExternalMeasurement,
|
||||
type ExternalMeasurementWrite,
|
||||
@@ -383,3 +386,104 @@ describe("reconcileExternalMeasurement", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* The reconciler is the shared write seam for every provider that does not
|
||||
* hold Prisma itself — Polar, Oura, WHOOP, the Withings sleep leg, the Apple
|
||||
* export importer, the batch endpoint. Gating here rather than at each caller
|
||||
* is what makes a provider added tomorrow inherit the refusal instead of
|
||||
* having to remember a check it never knew about.
|
||||
*/
|
||||
describe("reconcileExternalMeasurement — plausibility gate", () => {
|
||||
it("reconciles a value inside the metric's declared band", async () => {
|
||||
const { tx, measurement } = transaction([]);
|
||||
measurement.create.mockResolvedValue(row({ id: "fresh" }));
|
||||
|
||||
// PULSE is declared 20–300.
|
||||
const verdict = await reconcileExternalMeasurement(
|
||||
tx,
|
||||
desired({ type: "PULSE", value: 112, unit: "bpm", sleepStage: null }),
|
||||
);
|
||||
|
||||
expect(verdict.status).toBe("inserted");
|
||||
expect(measurement.create).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("refuses an out-of-band value without touching the transaction", async () => {
|
||||
const { tx, measurement } = transaction([]);
|
||||
|
||||
const verdict = await reconcileExternalMeasurement(
|
||||
tx,
|
||||
desired({
|
||||
type: "PULSE",
|
||||
value: 55643821.505,
|
||||
unit: "bpm",
|
||||
sleepStage: null,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(verdict).toEqual({
|
||||
status: "rejected_range",
|
||||
rejection: { type: "PULSE", direction: "above_max" },
|
||||
});
|
||||
// No row written, no row updated, and no savepoint opened — the refusal
|
||||
// happens before the reconciler does any work at all.
|
||||
expect(measurement.create).not.toHaveBeenCalled();
|
||||
expect(measurement.update).not.toHaveBeenCalled();
|
||||
expect(tx.$executeRawUnsafe).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("refuses rather than overwriting a good stored row", async () => {
|
||||
// The dangerous shape: a live row already holds a real reading, and the
|
||||
// provider re-sends the same identity carrying an impossible value. The
|
||||
// update branch would have rewritten a good value with a bad one.
|
||||
const existing = row({
|
||||
id: "exact-row",
|
||||
type: "PULSE",
|
||||
value: 62,
|
||||
unit: "bpm",
|
||||
sleepStage: null,
|
||||
measuredAt: new Date("2026-07-20T06:30:00.000Z"),
|
||||
});
|
||||
const { tx, measurement } = transaction([existing]);
|
||||
|
||||
const verdict = await reconcileExternalMeasurement(
|
||||
tx,
|
||||
desired({
|
||||
type: "PULSE",
|
||||
value: 55643821.505,
|
||||
unit: "bpm",
|
||||
sleepStage: null,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(verdict.status).toBe("rejected_range");
|
||||
expect(measurement.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("tallies the refusal on the ambient wide event under the provider", async () => {
|
||||
const { tx } = transaction([]);
|
||||
const event = new WideEventBuilder("background");
|
||||
|
||||
await eventStorage.run(event, () =>
|
||||
reconcileExternalMeasurement(
|
||||
tx,
|
||||
desired({
|
||||
type: "PULSE",
|
||||
value: 55643821.505,
|
||||
unit: "bpm",
|
||||
source: "WHOOP",
|
||||
sleepStage: null,
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
expect(event.toJSON().meta?.[RANGE_REJECTED_META_KEY]).toEqual({
|
||||
total: 1,
|
||||
buckets: [
|
||||
{ source: "whoop", type: "PULSE", direction: "above_max", count: 1 },
|
||||
],
|
||||
});
|
||||
expect(event.getLevel()).toBe("warn");
|
||||
});
|
||||
});
|
||||
@@ -720,6 +720,16 @@ export async function streamParseExportXml(
|
||||
),
|
||||
);
|
||||
|
||||
// The reconciler carries the same plausibility gate the block above
|
||||
// already applied, so this arm means the two disagreed. Tally it under
|
||||
// the same counter the earlier guard uses and count no row.
|
||||
if (verdict.status === "rejected_range") {
|
||||
unknown[`${type}::aggregate_out_of_range`] =
|
||||
(unknown[`${type}::aggregate_out_of_range`] ?? 0) + 1;
|
||||
stat.durationMs += Date.now() - rowStart;
|
||||
continue;
|
||||
}
|
||||
|
||||
cumulativeEstimatedDays.add(dayKey);
|
||||
cumulativeEstimatedRows += 1;
|
||||
if (verdict.status === "inserted") {
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* The ingest-side half of the app's plausibility domain.
|
||||
*
|
||||
* `VALUE_RANGES` in `@/lib/validations/measurement` is the band the
|
||||
* application declares a metric can occupy — PULSE 20–300, WEIGHT 1–500,
|
||||
* SLEEP_DURATION 0–1440 minutes. Every path a person can drive enforces it:
|
||||
* the measurement routes, the batch endpoint, the CSV and Apple-export
|
||||
* importers, MCP, Telegram. The provider sync writers did not. They took the
|
||||
* number the remote handed over and wrote it, so a decode slip or a glitched
|
||||
* device landed a row the application itself calls impossible, and every mean,
|
||||
* median and personal band downstream digested it as a reading.
|
||||
*
|
||||
* `isPlausibleMetricValue` (in `value-domain.ts`) already answers the question
|
||||
* for the READ side, which is where the damage surfaced first. This module is
|
||||
* the same question asked one layer earlier, at the point of writing, so the
|
||||
* row never lands. The two ends share the one predicate on purpose: a band the
|
||||
* reader hides but the writer accepts is a row that exists and cannot be seen,
|
||||
* which is the worse of the two failures.
|
||||
*
|
||||
* ## Disposition: drop, and say so
|
||||
*
|
||||
* An out-of-band incoming value is DROPPED. Not clamped (a clamp invents a
|
||||
* reading nobody took), not stored-and-flagged (a quarantined row is still a
|
||||
* row, and every consumer would need to learn the flag), not fatal to the
|
||||
* batch (one glitched sample must not strand the sync's other readings behind
|
||||
* a held watermark). Absence reads as absence: the sample is simply not part
|
||||
* of the record.
|
||||
*
|
||||
* A silent drop, though, is indistinguishable from a provider that sent
|
||||
* nothing — so every drop is tallied onto the ambient wide event under
|
||||
* {@link RANGE_REJECTED_META_KEY}, with the metric type, the provider, and the
|
||||
* direction it broke the band in. Counts only. The value itself is a health
|
||||
* reading and never enters the log line; the direction plus the metric's own
|
||||
* declared bound is what a diagnosis actually needs.
|
||||
*
|
||||
* ## Why meta and not `action`
|
||||
*
|
||||
* A wide event holds exactly ONE action, and `setAction` replaces it. A drop
|
||||
* happens in the middle of a sync whose action is decided at the end, so an
|
||||
* `action` set here is either clobbered a moment later (the drop vanishes) or,
|
||||
* on the paths that annotate no terminal action, relabels an otherwise
|
||||
* successful sync as a rejection. Meta accumulates, survives every later
|
||||
* `setAction`, and is the shape a dashboard can sum. One warning per provider
|
||||
* per event rides along so the drop also lifts the event's level to `warn`
|
||||
* rather than hiding inside a green line.
|
||||
*/
|
||||
import { annotate, getEvent } from "@/lib/logging/context";
|
||||
import { isPlausibleMetricValue } from "@/lib/measurements/value-domain";
|
||||
import { VALUE_RANGES } from "@/lib/validations/measurement";
|
||||
|
||||
/**
|
||||
* Stable meta key the tally lands under. Pinned so dashboards can count it
|
||||
* across providers without knowing which one wrote the event.
|
||||
*/
|
||||
export const RANGE_REJECTED_META_KEY = "measurement.range_rejected";
|
||||
|
||||
/** Which edge of the declared band the value broke, or that it was not a number. */
|
||||
export type RangeRejectionDirection = "below_min" | "above_max" | "not_finite";
|
||||
|
||||
/** One refused reading, reduced to the facts that carry no health value. */
|
||||
export interface MeasurementRangeRejection {
|
||||
/** `MeasurementType` of the refused reading. */
|
||||
type: string;
|
||||
direction: RangeRejectionDirection;
|
||||
}
|
||||
|
||||
/** One (provider, metric, direction) cell of the event's running tally. */
|
||||
export interface RangeRejectionBucket {
|
||||
source: string;
|
||||
type: string;
|
||||
direction: RangeRejectionDirection;
|
||||
count: number;
|
||||
}
|
||||
|
||||
/** The pinned meta shape written under {@link RANGE_REJECTED_META_KEY}. */
|
||||
export interface RangeRejectionTally {
|
||||
total: number;
|
||||
buckets: RangeRejectionBucket[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a candidate write against the metric's declared band.
|
||||
*
|
||||
* Returns `null` when the value may be written. A type with no declared range
|
||||
* passes — an absent range is an absent fact, never a guessed one, and the
|
||||
* same asymmetry the read side uses (`isPlausibleMetricValue`). A non-finite
|
||||
* value never passes, whatever the type: NaN is not an input to anything.
|
||||
*/
|
||||
export function classifyRangeRejection(
|
||||
type: string,
|
||||
value: number,
|
||||
): MeasurementRangeRejection | null {
|
||||
if (isPlausibleMetricValue(type, value)) return null;
|
||||
if (!Number.isFinite(value)) return { type, direction: "not_finite" };
|
||||
const range = VALUE_RANGES[type];
|
||||
// `isPlausibleMetricValue` only answers false for a finite value when the
|
||||
// type HAS a range, so the lookup cannot miss here.
|
||||
return {
|
||||
type,
|
||||
direction: value < range!.min ? "below_min" : "above_max",
|
||||
};
|
||||
}
|
||||
|
||||
function readTally(): RangeRejectionTally {
|
||||
const existing = getEvent()?.toJSON().meta?.[RANGE_REJECTED_META_KEY];
|
||||
if (
|
||||
existing !== null &&
|
||||
typeof existing === "object" &&
|
||||
Array.isArray((existing as RangeRejectionTally).buckets)
|
||||
) {
|
||||
return existing as RangeRejectionTally;
|
||||
}
|
||||
return { total: 0, buckets: [] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold refusals onto the ambient wide event.
|
||||
*
|
||||
* Merges with whatever the event already carries, so a provider that gates
|
||||
* several batches (or reconciles row by row) accumulates instead of
|
||||
* overwriting. Outside a wide-event context this is a no-op — the caller still
|
||||
* drops the row, it simply has nowhere to report.
|
||||
*/
|
||||
export function recordRangeRejections(
|
||||
source: string,
|
||||
rejections: readonly MeasurementRangeRejection[],
|
||||
): void {
|
||||
if (rejections.length === 0) return;
|
||||
const evt = getEvent();
|
||||
if (!evt) return;
|
||||
|
||||
const tally = readTally();
|
||||
const firstForSource = !tally.buckets.some((b) => b.source === source);
|
||||
|
||||
for (const rejection of rejections) {
|
||||
const bucket = tally.buckets.find(
|
||||
(b) =>
|
||||
b.source === source &&
|
||||
b.type === rejection.type &&
|
||||
b.direction === rejection.direction,
|
||||
);
|
||||
if (bucket) bucket.count += 1;
|
||||
else
|
||||
tally.buckets.push({
|
||||
source,
|
||||
type: rejection.type,
|
||||
direction: rejection.direction,
|
||||
count: 1,
|
||||
});
|
||||
tally.total += 1;
|
||||
}
|
||||
|
||||
tally.buckets.sort(
|
||||
(a, b) =>
|
||||
a.source.localeCompare(b.source) ||
|
||||
a.type.localeCompare(b.type) ||
|
||||
a.direction.localeCompare(b.direction),
|
||||
);
|
||||
|
||||
annotate({ meta: { [RANGE_REJECTED_META_KEY]: tally } });
|
||||
if (firstForSource) {
|
||||
evt.addWarning(
|
||||
`${source}: refused reading(s) outside the declared plausibility range — ` +
|
||||
`dropped, see meta.${RANGE_REJECTED_META_KEY}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep the rows whose value sits inside the metric's declared band; tally and
|
||||
* drop the rest.
|
||||
*
|
||||
* `select` exists because the sync writers carry their own row shapes and none
|
||||
* of them is worth reshaping at the call site: the gate reads the two fields
|
||||
* it needs and returns the caller's own objects untouched.
|
||||
*/
|
||||
export function dropImplausibleMeasurements<T>(
|
||||
source: string,
|
||||
rows: readonly T[],
|
||||
select: (row: T) => { type: string; value: number },
|
||||
): T[] {
|
||||
const kept: T[] = [];
|
||||
const rejections: MeasurementRangeRejection[] = [];
|
||||
for (const row of rows) {
|
||||
const { type, value } = select(row);
|
||||
const rejection = classifyRangeRejection(type, value);
|
||||
if (rejection) rejections.push(rejection);
|
||||
else kept.push(row);
|
||||
}
|
||||
recordRangeRejections(source, rejections);
|
||||
return kept;
|
||||
}
|
||||
@@ -4,6 +4,11 @@ import {
|
||||
type MeasurementAggregationProvenance,
|
||||
type Prisma as PrismaTypes,
|
||||
} from "@/generated/prisma/client";
|
||||
import {
|
||||
classifyRangeRejection,
|
||||
recordRangeRejections,
|
||||
type MeasurementRangeRejection,
|
||||
} from "@/lib/measurements/plausibility-gate";
|
||||
|
||||
const SAVEPOINT = "measurement_identity_reconcile";
|
||||
|
||||
@@ -50,6 +55,18 @@ export type MeasurementReconciliationVerdict =
|
||||
| {
|
||||
status: "failed";
|
||||
error: { message: string; code?: string };
|
||||
}
|
||||
| {
|
||||
/**
|
||||
* The incoming value sits outside the metric's declared plausibility
|
||||
* band, so nothing was written and no existing row was touched. Not a
|
||||
* failure: the write was refused on purpose and the caller must carry
|
||||
* on with the rest of its batch. Every caller narrows on this status
|
||||
* because it carries no `row` — the compiler is what makes a future
|
||||
* provider handle it rather than dereference a row that does not exist.
|
||||
*/
|
||||
status: "rejected_range";
|
||||
rejection: MeasurementRangeRejection;
|
||||
};
|
||||
|
||||
export type FailedMeasurementReconciliation = Extract<
|
||||
@@ -219,12 +236,26 @@ async function retireCollision(
|
||||
* advisory lock serializes cooperative writers sharing either identity, and a
|
||||
* savepoint turns a non-benign database error into a structured `failed`
|
||||
* verdict without poisoning the caller's surrounding transaction.
|
||||
*
|
||||
* This is also the plausibility gate for every provider that reconciles rather
|
||||
* than writing Prisma directly. Gating HERE rather than at each caller is the
|
||||
* point: a provider added tomorrow inherits the refusal by construction, and
|
||||
* cannot land an impossible reading by forgetting a check it never knew about.
|
||||
*/
|
||||
export async function reconcileExternalMeasurement(
|
||||
tx: PrismaTypes.TransactionClient,
|
||||
desired: ExternalMeasurementWrite,
|
||||
options: MeasurementReconciliationOptions = {},
|
||||
): Promise<MeasurementReconciliationVerdict> {
|
||||
const rejection = classifyRangeRejection(desired.type, desired.value);
|
||||
if (rejection) {
|
||||
recordRangeRejections(
|
||||
desired.source ? String(desired.source).toLowerCase() : "external",
|
||||
[rejection],
|
||||
);
|
||||
return { status: "rejected_range", rejection };
|
||||
}
|
||||
|
||||
await tx.$executeRawUnsafe(`SAVEPOINT ${SAVEPOINT}`);
|
||||
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
/**
|
||||
* The Nightscout pull writes BLOOD_GLUCOSE rows directly, so it does not
|
||||
* inherit the shared reconciler's gate. It is also first-write-wins: a row
|
||||
* that lands here is immutable, so an impossible sample never corrects itself
|
||||
* on a later pass. These cases pin the refusal and its annotation.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const {
|
||||
upsertMock,
|
||||
findManyMock,
|
||||
emitArrivalsMock,
|
||||
recomputeMock,
|
||||
invalidateMock,
|
||||
} = vi.hoisted(() => ({
|
||||
upsertMock: vi.fn(),
|
||||
findManyMock: vi.fn(),
|
||||
emitArrivalsMock: vi.fn(),
|
||||
recomputeMock: vi.fn(),
|
||||
invalidateMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: { measurement: { upsert: upsertMock, findMany: findManyMock } },
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/arrivals/measurement-emit", () => ({
|
||||
emitInsertedMeasurementArrivals: emitArrivalsMock,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/rollups/measurement-rollups", () => ({
|
||||
collapseToTypeDayKeys: (rows: Array<{ type: string; measuredAt: Date }>) =>
|
||||
rows.map((r) => ({ type: r.type, measuredAt: r.measuredAt })),
|
||||
recomputeBucketsForMeasurement: recomputeMock,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/insights/comprehensive-generate", () => ({
|
||||
invalidateStatusInsightsForTypes: invalidateMock,
|
||||
}));
|
||||
|
||||
import { eventStorage } from "@/lib/logging/context";
|
||||
import { WideEventBuilder } from "@/lib/logging/event-builder";
|
||||
import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate";
|
||||
import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate";
|
||||
|
||||
import { upsertNightscoutEntries } from "../sync";
|
||||
|
||||
/** BLOOD_GLUCOSE is declared 20–800 mg/dL. */
|
||||
const REAL_ENTRY = { id: "abc", sgv: 112, date: 1718000000000 };
|
||||
/** A sensor fault, or an mg/dL field carrying something else entirely. */
|
||||
const IMPOSSIBLE_ENTRY = { id: "def", sgv: 41_600, date: 1718000300000 };
|
||||
|
||||
let event: WideEventBuilder;
|
||||
|
||||
function tally(): RangeRejectionTally | undefined {
|
||||
return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as
|
||||
RangeRejectionTally | undefined;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
findManyMock.mockResolvedValue([]);
|
||||
upsertMock.mockResolvedValue({ id: "row-1" });
|
||||
emitArrivalsMock.mockResolvedValue(undefined);
|
||||
recomputeMock.mockResolvedValue(undefined);
|
||||
invalidateMock.mockResolvedValue(undefined);
|
||||
event = new WideEventBuilder("background");
|
||||
});
|
||||
|
||||
describe("upsertNightscoutEntries — plausibility gate", () => {
|
||||
it("writes a reading inside the metric's declared band", async () => {
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertNightscoutEntries("user-1", [REAL_ENTRY]),
|
||||
);
|
||||
|
||||
expect(result).toEqual({ inserted: 1, failedRows: 0 });
|
||||
expect(upsertMock).toHaveBeenCalledTimes(1);
|
||||
expect(tally()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refuses an out-of-band reading and annotates the drop", async () => {
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertNightscoutEntries("user-1", [REAL_ENTRY, IMPOSSIBLE_ENTRY]),
|
||||
);
|
||||
|
||||
expect(result).toEqual({ inserted: 1, failedRows: 0 });
|
||||
expect(upsertMock).toHaveBeenCalledTimes(1);
|
||||
expect(
|
||||
(upsertMock.mock.calls[0]![0] as { create: { value: number } }).create
|
||||
.value,
|
||||
).toBe(112);
|
||||
expect(tally()).toEqual({
|
||||
total: 1,
|
||||
buckets: [
|
||||
{
|
||||
source: "nightscout",
|
||||
type: "BLOOD_GLUCOSE",
|
||||
direction: "above_max",
|
||||
count: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(event.getLevel()).toBe("warn");
|
||||
});
|
||||
|
||||
it("writes nothing when the whole page is out of band", async () => {
|
||||
const result = await eventStorage.run(event, () =>
|
||||
upsertNightscoutEntries("user-1", [IMPOSSIBLE_ENTRY]),
|
||||
);
|
||||
|
||||
expect(result).toEqual({ inserted: 0, failedRows: 0 });
|
||||
expect(upsertMock).not.toHaveBeenCalled();
|
||||
expect(tally()?.total).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -22,6 +22,7 @@
|
||||
import { prisma } from "@/lib/db";
|
||||
import type { MeasurementType } from "@/generated/prisma/client";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate";
|
||||
import {
|
||||
markSyncFailureRecorded,
|
||||
recordSyncFailure,
|
||||
@@ -221,7 +222,16 @@ export async function upsertNightscoutEntries(
|
||||
if (entries.length === 0) return { inserted: 0, failedRows: 0 };
|
||||
|
||||
const type = "BLOOD_GLUCOSE" as MeasurementType;
|
||||
const mapped = entries.map((entry) => mapSgvEntryToMeasurement(entry));
|
||||
// A CGM that reports a glucose the app's own band calls impossible is
|
||||
// reporting a sensor fault or a unit slip, not a reading. Drop it: the row
|
||||
// would otherwise be immutable (first-write-wins), so an impossible sample
|
||||
// that lands here never corrects itself on a later pass.
|
||||
const mapped = dropImplausibleMeasurements(
|
||||
"nightscout",
|
||||
entries.map((entry) => mapSgvEntryToMeasurement(entry)),
|
||||
(m) => ({ type, value: m.value }),
|
||||
);
|
||||
if (mapped.length === 0) return { inserted: 0, failedRows: 0 };
|
||||
|
||||
// Probe which keys already exist so a re-confirmed reading (the upsert's
|
||||
// no-op update branch) is not miscounted as a fresh import and the arrival
|
||||
|
||||
@@ -44,7 +44,13 @@ listLabResultsSchema.meta({
|
||||
const rangeStatusEnum = z.enum(["in-range", "below", "above", "unknown"]).meta({
|
||||
id: "LabReferenceRangeStatus",
|
||||
description:
|
||||
"Server-computed, NEUTRAL reference-range verdict. `unknown` when the lab reported no usable bounds. Inclusive bounds: a value on the limit reads in-range. The badge that renders this must stay calm and informative — not an alarming red.",
|
||||
"Server-computed, NEUTRAL reference-range verdict, always against `referenceLow` / `referenceHigh` (the window in force for that reading). `unknown` when no usable bounds are on file. Inclusive bounds: a value on the limit reads in-range. The badge that renders this must stay calm and informative — not an alarming red.",
|
||||
});
|
||||
|
||||
const referenceOriginEnum = z.enum(["source", "catalog", "none"]).meta({
|
||||
id: "LabReferenceRangeOrigin",
|
||||
description:
|
||||
"Which window `referenceLow` / `referenceHigh` came from. `source` = the range printed on this reading's own report, which wins for that reading because it is the window the physician evaluated the value against. `catalog` = the user's Biomarker band, the net for readings with no printed range. `none` = no window on file.",
|
||||
});
|
||||
|
||||
const labResultRow = z
|
||||
@@ -60,6 +66,13 @@ const labResultRow = z
|
||||
unit: z.string(),
|
||||
referenceLow: z.number().nullable(),
|
||||
referenceHigh: z.number().nullable(),
|
||||
catalogReferenceLow: z.number().nullable(),
|
||||
catalogReferenceHigh: z.number().nullable(),
|
||||
sourceReferenceLow: z.number().nullable(),
|
||||
sourceReferenceHigh: z.number().nullable(),
|
||||
sourceReferenceText: z.string().nullable(),
|
||||
referenceOrigin: referenceOriginEnum,
|
||||
referenceDivergesFromCatalog: z.boolean(),
|
||||
takenAt: z.string(),
|
||||
source: z.string(),
|
||||
hasNote: z.boolean(),
|
||||
@@ -70,7 +83,7 @@ const labResultRow = z
|
||||
.meta({
|
||||
id: "LabResult",
|
||||
description:
|
||||
'A stored lab result. Exactly one of `value` (numeric) or `valueText` (qualitative, e.g. "negativ") is non-null; `rangeStatus` is always `unknown` for a qualitative row (no number to classify). `biomarkerId` links the user-scoped catalog marker (null for legacy free-text rows); when linked, `analyte` / `unit` / `referenceLow` / `referenceHigh` are RESOLVED server-side from the Biomarker — the client renders them and never recomputes the range. The encrypted note is never echoed in list rows; `hasNote` flags its presence and the single-resource GET returns the decrypted `note`.',
|
||||
"A stored lab result. Exactly one of `value` (numeric) or `valueText` (qualitative, e.g. \"negativ\") is non-null; `rangeStatus` is always `unknown` for a qualitative row (no number to classify). `biomarkerId` links the user-scoped catalog marker (null for legacy free-text rows); when linked, `analyte` / `unit` are RESOLVED server-side from the Biomarker.\n\nThree reference windows ride on the row and the server says which one applies. `referenceLow` / `referenceHigh` are the window IN FORCE and the one `rangeStatus` was computed from — read these to render the range and you are correct without doing any resolution of your own. `sourceReferenceLow` / `sourceReferenceHigh` / `sourceReferenceText` are the range this reading's own report printed (the text verbatim, kept even when no numbers could be read out of it). `catalogReferenceLow` / `catalogReferenceHigh` are the Biomarker band. `referenceOrigin` names the winner and `referenceDivergesFromCatalog` is true when the report's window and the catalog band state different limits — a surface showing only one of the two is showing a partial answer. The client renders these values and never recomputes the range. The encrypted note is never echoed in list rows; `hasNote` flags its presence and the single-resource GET returns the decrypted `note`.",
|
||||
});
|
||||
|
||||
const labResultDetail = labResultRow
|
||||
|
||||
@@ -53,6 +53,10 @@ const extractedRow = z
|
||||
unit: z.string().nullable(),
|
||||
referenceLow: z.number().nullable(),
|
||||
referenceHigh: z.number().nullable(),
|
||||
referenceText: z.string().nullable().meta({
|
||||
description:
|
||||
'The reference range EXACTLY as printed on the report, verbatim ("3,5 - 5,0", "< 116", "bis 5,0", "negativ"). `referenceLow` / `referenceHigh` carry the same window reduced to numbers WHEN it reduces to numbers; this field carries it either way, so a window stated in words is not lost between the report and the reading.',
|
||||
}),
|
||||
takenAt: z.string().nullable(),
|
||||
confidence: extractConfidence,
|
||||
biomarkerMatch: z.enum(["new", "existing"]),
|
||||
@@ -87,6 +91,13 @@ const committedRow = z
|
||||
unit: z.string(),
|
||||
referenceLow: z.number().nullable(),
|
||||
referenceHigh: z.number().nullable(),
|
||||
catalogReferenceLow: z.number().nullable(),
|
||||
catalogReferenceHigh: z.number().nullable(),
|
||||
sourceReferenceLow: z.number().nullable(),
|
||||
sourceReferenceHigh: z.number().nullable(),
|
||||
sourceReferenceText: z.string().nullable(),
|
||||
referenceOrigin: z.enum(["source", "catalog", "none"]),
|
||||
referenceDivergesFromCatalog: z.boolean(),
|
||||
takenAt: z.string(),
|
||||
source: z.string(),
|
||||
hasNote: z.boolean(),
|
||||
@@ -94,7 +105,11 @@ const committedRow = z
|
||||
createdAt: z.string(),
|
||||
updatedAt: z.string(),
|
||||
})
|
||||
.meta({ id: "OcrCommittedLabResult" });
|
||||
.meta({
|
||||
id: "OcrCommittedLabResult",
|
||||
description:
|
||||
"A written reading. Its reference window resolves exactly as it does on `/api/labs` — see `LabResult`.",
|
||||
});
|
||||
|
||||
const commitResponse = z
|
||||
.object({
|
||||
|
||||
@@ -575,6 +575,10 @@ export async function upsertOuraMeasurements(
|
||||
for (let index = 0; index < readings.length; index++) {
|
||||
const reading = readings[index]!;
|
||||
const verdict = verdicts[index]!;
|
||||
// Refused by the plausibility gate: nothing was written and nothing was
|
||||
// touched, so the reading counts as neither imported nor a rollup dirt
|
||||
// mark. The drop is already on the wide event.
|
||||
if (verdict.status === "rejected_range") continue;
|
||||
for (const dirty of verdict.dirtyIdentities ?? []) {
|
||||
touched.push(dirty);
|
||||
}
|
||||
|
||||
@@ -364,6 +364,10 @@ export async function upsertPolarMeasurements(
|
||||
for (let index = 0; index < readings.length; index++) {
|
||||
const reading = readings[index]!;
|
||||
const verdict = verdicts[index]!;
|
||||
// Refused by the plausibility gate: nothing was written and nothing was
|
||||
// touched, so the reading counts as neither imported nor a rollup dirt
|
||||
// mark. The drop is already on the wide event.
|
||||
if (verdict.status === "rejected_range") continue;
|
||||
for (const dirty of verdict.dirtyIdentities ?? []) {
|
||||
touched.push(dirty);
|
||||
}
|
||||
|
||||
@@ -255,11 +255,29 @@ const moodEntrySchema = z
|
||||
mood: z.string().min(1),
|
||||
score: z.number().int().min(0).max(10),
|
||||
tags: moodEntryTagsSchema,
|
||||
// The free text, in both storage shapes. A portable export carries the
|
||||
// decrypted `note`; a disaster-recovery file carries `noteEncrypted` plus
|
||||
// whatever legacy plaintext the row still holds. Same pair, and same
|
||||
// handling on restore, as `Measurement.notes` / `notesEncrypted`.
|
||||
note: z.string().nullable().optional(),
|
||||
noteEncrypted: base64BytesSchema.nullable().optional(),
|
||||
source: z.string().min(1).optional(),
|
||||
externalId: z.string().nullable().optional(),
|
||||
loggedAt: isoDateTime,
|
||||
// The IANA zone the `date` string is anchored to. Absent means the legacy
|
||||
// Europe/Berlin reading, which is what a row written before v1.4.25 means
|
||||
// by a NULL here — so an old file keeps its old meaning exactly.
|
||||
tz: z.string().nullable().optional(),
|
||||
syncedAt: isoDateTime.optional(),
|
||||
syncVersion: z.number().int().optional(),
|
||||
deletedAt: isoDateTime.nullable().optional(),
|
||||
createdAt: isoDateTime.optional(),
|
||||
updatedAt: isoDateTime.optional(),
|
||||
factors: z.array(moodFactorSchema).default([]),
|
||||
// BINARY structured-tag keys. Separate from `factors`, which keeps its
|
||||
// RATED-only meaning, so every file written before this one parses
|
||||
// unchanged and restores identically: absent reads as none.
|
||||
structuredTags: z.array(z.string().min(1)).default([]),
|
||||
})
|
||||
.passthrough();
|
||||
|
||||
@@ -690,6 +708,11 @@ const labResultBackupSchema = z
|
||||
unit: z.string().min(1),
|
||||
referenceLow: z.number().nullable().optional(),
|
||||
referenceHigh: z.number().nullable().optional(),
|
||||
// Optional so a backup written before these columns existed still
|
||||
// restores; absent reads as "this reading had no source window on file".
|
||||
sourceReferenceLow: z.number().nullable().optional(),
|
||||
sourceReferenceHigh: z.number().nullable().optional(),
|
||||
sourceReferenceText: z.string().nullable().optional(),
|
||||
takenAt: isoDateTime,
|
||||
source: z.string().min(1),
|
||||
biomarkerName: z.string().nullable().optional(),
|
||||
|
||||
@@ -167,6 +167,15 @@ export const extractedFactRawSchema = z.object({
|
||||
unit: nullableText(80),
|
||||
referenceLow: z.number().finite().nullable().catch(null),
|
||||
referenceHigh: z.number().finite().nullable().catch(null),
|
||||
/**
|
||||
* Observation: the reference range EXACTLY as printed, verbatim. A report
|
||||
* writes its window in prose ("3,5 - 5,0", "< 5", "bis 5,0", "negativ"), and
|
||||
* `referenceLow` / `referenceHigh` above can only carry the cases that
|
||||
* reduce to two numbers. This field is what stops every other case from
|
||||
* being dropped on the floor — the string is kept whether or not the parser
|
||||
* gets bounds out of it.
|
||||
*/
|
||||
referenceText: nullableText(120),
|
||||
/** MedicationStatement: dose + stated status (verbatim). */
|
||||
dose: nullableText(120),
|
||||
medicationStatus: nullableText(64),
|
||||
@@ -237,6 +246,14 @@ export interface ObservationFactData {
|
||||
unit: string | null;
|
||||
referenceLow: number | null;
|
||||
referenceHigh: number | null;
|
||||
/**
|
||||
* The reference range as printed on the report, verbatim.
|
||||
*
|
||||
* Optional because it genuinely is: a fact staged before this field existed
|
||||
* carries no key for it, and that is an honest absence rather than a shim
|
||||
* for a caller we do not ship. Read it as `?? null`.
|
||||
*/
|
||||
referenceText?: string | null;
|
||||
effectiveDate: string | null;
|
||||
}
|
||||
|
||||
@@ -379,6 +396,7 @@ const observationEditSchema = z
|
||||
unit: optText(80),
|
||||
referenceLow: z.number().finite().nullable().optional(),
|
||||
referenceHigh: z.number().finite().nullable().optional(),
|
||||
referenceText: optText(120),
|
||||
effectiveDate: isoDate,
|
||||
})
|
||||
// Numeric XOR qualitative, mirroring the lab-result discipline. A null/absent
|
||||
|
||||
@@ -78,6 +78,13 @@ export const extractedRowSchema = z.object({
|
||||
unit: z.string().trim().max(80).nullable().catch(null),
|
||||
referenceLow: z.number().finite().nullable().catch(null),
|
||||
referenceHigh: z.number().finite().nullable().catch(null),
|
||||
/**
|
||||
* The reference range EXACTLY as printed, verbatim. The two numeric bounds
|
||||
* above can only carry a window that reduces to two numbers; a report that
|
||||
* prints "bis 5,0" or "negativ" states a window all the same, and this is
|
||||
* what keeps it instead of dropping it.
|
||||
*/
|
||||
referenceText: z.string().trim().max(120).nullable().catch(null),
|
||||
takenAt: z.string().nullable().catch(null),
|
||||
confidence: extractedConfidenceSchema,
|
||||
});
|
||||
@@ -107,6 +114,8 @@ export interface OcrExtractedRowDto {
|
||||
unit: string | null;
|
||||
referenceLow: number | null;
|
||||
referenceHigh: number | null;
|
||||
/** The reference range as printed on the report, verbatim. */
|
||||
referenceText: string | null;
|
||||
takenAt: string | null;
|
||||
confidence: {
|
||||
analyte: number;
|
||||
@@ -150,6 +159,17 @@ export const ocrCommitRowSchema = z
|
||||
unit: z.string().trim().min(1).max(40).optional(),
|
||||
referenceLow,
|
||||
referenceHigh,
|
||||
/**
|
||||
* The window the scanned report printed for this row. Confirmed by the
|
||||
* human on the review screen alongside everything else, and written onto
|
||||
* the reading rather than only used to seed a new catalog marker.
|
||||
*/
|
||||
referenceText: z
|
||||
.string()
|
||||
.trim()
|
||||
.max(120)
|
||||
.optional()
|
||||
.or(z.literal("").transform(() => undefined)),
|
||||
takenAt: takenAtField,
|
||||
})
|
||||
// Numeric XOR qualitative: exactly one of value / valueText.
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
*/
|
||||
import { z } from "zod/v4";
|
||||
|
||||
import { SOURCE_REFERENCE_TEXT_MAX } from "@/lib/labs/parse-reference-range";
|
||||
import { validateEntryInstant } from "@/lib/validations/entry-instant";
|
||||
|
||||
/**
|
||||
@@ -63,6 +64,29 @@ const optionalNote = z
|
||||
const referenceLow = z.number().finite().optional();
|
||||
const referenceHigh = z.number().finite().optional();
|
||||
|
||||
/**
|
||||
* The reference range the SOURCE REPORT printed for this one reading.
|
||||
*
|
||||
* Distinct from `referenceLow` / `referenceHigh` above, which seed (and are
|
||||
* stamped from) the catalog marker's band. A lab runs its own method on its
|
||||
* own device, so the window a physician evaluates against is the one on that
|
||||
* report; these fields carry it, and `resolveEffectiveReferenceRange` lets it
|
||||
* win for that reading alone.
|
||||
*
|
||||
* `sourceReferenceText` is what was printed, verbatim. It is accepted with or
|
||||
* without derivable bounds: a window stated as "negativ" or "siehe Befund"
|
||||
* still belongs on the record, and dropping it because no number could be read
|
||||
* out of it is exactly the loss this field exists to stop.
|
||||
*/
|
||||
const sourceReferenceLow = z.number().finite().optional();
|
||||
const sourceReferenceHigh = z.number().finite().optional();
|
||||
const sourceReferenceText = z
|
||||
.string()
|
||||
.trim()
|
||||
.max(SOURCE_REFERENCE_TEXT_MAX)
|
||||
.optional()
|
||||
.or(z.literal("").transform(() => undefined));
|
||||
|
||||
/**
|
||||
* v1.18.1 — optional link to a user-scoped `Biomarker` (the catalog row).
|
||||
* When set, the structured-entry path resolves the unit + reference range
|
||||
@@ -101,6 +125,9 @@ export const createLabResultSchema = z
|
||||
unit: requiredText(40).optional(),
|
||||
referenceLow,
|
||||
referenceHigh,
|
||||
sourceReferenceLow,
|
||||
sourceReferenceHigh,
|
||||
sourceReferenceText,
|
||||
takenAt: takenAtField,
|
||||
note: optionalNote,
|
||||
source,
|
||||
@@ -142,6 +169,19 @@ export const createLabResultSchema = z
|
||||
message: "referenceLow must not exceed referenceHigh",
|
||||
path: ["referenceLow"],
|
||||
},
|
||||
)
|
||||
// Same guard on the source window. A transposed window is a client error
|
||||
// here for the same reason: nothing downstream can tell it from a real one.
|
||||
.refine(
|
||||
(d) =>
|
||||
d.valueText !== undefined ||
|
||||
d.sourceReferenceLow === undefined ||
|
||||
d.sourceReferenceHigh === undefined ||
|
||||
d.sourceReferenceLow <= d.sourceReferenceHigh,
|
||||
{
|
||||
message: "sourceReferenceLow must not exceed sourceReferenceHigh",
|
||||
path: ["sourceReferenceLow"],
|
||||
},
|
||||
);
|
||||
|
||||
export type CreateLabResultInput = z.infer<typeof createLabResultSchema>;
|
||||
@@ -164,6 +204,9 @@ export const updateLabResultSchema = z
|
||||
unit: requiredText(40).optional(),
|
||||
referenceLow: referenceLow.or(z.null()),
|
||||
referenceHigh: referenceHigh.or(z.null()),
|
||||
sourceReferenceLow: sourceReferenceLow.or(z.null()),
|
||||
sourceReferenceHigh: sourceReferenceHigh.or(z.null()),
|
||||
sourceReferenceText: sourceReferenceText.or(z.null()),
|
||||
takenAt: takenAtField.optional(),
|
||||
note: optionalNote.or(z.null()),
|
||||
})
|
||||
@@ -183,6 +226,18 @@ export const updateLabResultSchema = z
|
||||
message: "referenceLow must not exceed referenceHigh",
|
||||
path: ["referenceLow"],
|
||||
},
|
||||
)
|
||||
.refine(
|
||||
(d) =>
|
||||
d.sourceReferenceLow === undefined ||
|
||||
d.sourceReferenceLow === null ||
|
||||
d.sourceReferenceHigh === undefined ||
|
||||
d.sourceReferenceHigh === null ||
|
||||
d.sourceReferenceLow <= d.sourceReferenceHigh,
|
||||
{
|
||||
message: "sourceReferenceLow must not exceed sourceReferenceHigh",
|
||||
path: ["sourceReferenceLow"],
|
||||
},
|
||||
);
|
||||
|
||||
export type UpdateLabResultInput = z.infer<typeof updateLabResultSchema>;
|
||||
@@ -246,7 +301,12 @@ export type ListLabResultsInput = z.infer<typeof listLabResultsSchema>;
|
||||
* implementation. Re-exported here for the existing import sites.
|
||||
*/
|
||||
export {
|
||||
classifyAgainstEffectiveRange,
|
||||
classifyReferenceRange,
|
||||
formatReferenceRange,
|
||||
resolveEffectiveReferenceRange,
|
||||
type EffectiveReferenceRange,
|
||||
type ReferenceRangeOrigin,
|
||||
type ReferenceRangeStatus,
|
||||
type SourceReferenceRange,
|
||||
} from "@/lib/labs/reference-range";
|
||||
@@ -480,6 +480,10 @@ export async function upsertWhoopMeasurements(
|
||||
for (let index = chunkStart; index < chunkEnd; index++) {
|
||||
const reading = readings[index]!;
|
||||
const verdict = verdicts[index - chunkStart]!;
|
||||
// Refused by the plausibility gate: nothing was written and nothing was
|
||||
// touched, so the reading counts as neither imported nor a rollup dirt
|
||||
// mark. The drop is already on the wide event.
|
||||
if (verdict.status === "rejected_range") continue;
|
||||
for (const dirty of verdict.dirtyIdentities ?? []) {
|
||||
touched.push(dirty);
|
||||
touchedTypes.add(dirty.type);
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
/**
|
||||
* Withings writes the measurement table directly — the measures leg row by
|
||||
* row, the activity leg as a planned batch — so neither inherits the shared
|
||||
* reconciler's gate. These cases pin that both legs now clear the app's own
|
||||
* declared plausibility band before anything reaches Prisma, and that a
|
||||
* refusal lands on the wide event rather than vanishing.
|
||||
*
|
||||
* The measures leg matters twice over: it does not only insert, it OVERWRITES
|
||||
* the stored value of an existing slot, so an ungated impossible reading could
|
||||
* replace a real one.
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
measurement: {
|
||||
findFirst: vi.fn(),
|
||||
findMany: vi.fn(),
|
||||
create: vi.fn(),
|
||||
createMany: vi.fn(),
|
||||
update: vi.fn(),
|
||||
},
|
||||
withingsConnection: {
|
||||
findUnique: vi.fn(),
|
||||
update: vi.fn(),
|
||||
},
|
||||
$queryRaw: vi.fn(),
|
||||
$transaction: vi.fn((ops: unknown) =>
|
||||
Array.isArray(ops) ? Promise.all(ops) : Promise.resolve(),
|
||||
),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/crypto", () => ({
|
||||
decrypt: vi.fn((v: string) => v),
|
||||
encrypt: vi.fn((v: string) => v),
|
||||
}));
|
||||
|
||||
vi.mock("../client", async () => {
|
||||
const actual = await vi.importActual<typeof import("../client")>("../client");
|
||||
return {
|
||||
...actual,
|
||||
fetchMeasurements: vi.fn(),
|
||||
refreshAccessToken: vi.fn(),
|
||||
subscribeWebhook: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../credentials", () => ({
|
||||
getUserWithingsCredentials: vi.fn(async () => ({
|
||||
clientId: "cid",
|
||||
clientSecret: "secret",
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/integrations/status", () => ({
|
||||
isReauthRequired: vi.fn().mockResolvedValue(false),
|
||||
parkIntegrationAtReauth: vi.fn(),
|
||||
recordSyncFailure: vi.fn().mockResolvedValue(undefined),
|
||||
recordSyncSuccess: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/rollups/measurement-rollups", async () => {
|
||||
const actual = await vi.importActual<
|
||||
typeof import("@/lib/rollups/measurement-rollups")
|
||||
>("@/lib/rollups/measurement-rollups");
|
||||
return {
|
||||
...actual,
|
||||
recomputeBucketsForMeasurement: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@/lib/insights/comprehensive-generate", () => ({
|
||||
invalidateStatusInsightsForTypes: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/cache/invalidate", () => ({
|
||||
invalidateUserDashboardSnapshot: vi.fn(),
|
||||
}));
|
||||
|
||||
import { prisma } from "@/lib/db";
|
||||
import { eventStorage } from "@/lib/logging/context";
|
||||
import { WideEventBuilder } from "@/lib/logging/event-builder";
|
||||
import { RANGE_REJECTED_META_KEY } from "@/lib/measurements/plausibility-gate";
|
||||
import type { RangeRejectionTally } from "@/lib/measurements/plausibility-gate";
|
||||
|
||||
import { fetchMeasurements } from "../client";
|
||||
import { syncUserActivity } from "../sync-activity";
|
||||
import { syncUserMeasurements } from "../sync";
|
||||
|
||||
let event: WideEventBuilder;
|
||||
|
||||
function tally(): RangeRejectionTally | undefined {
|
||||
return event.toJSON().meta?.[RANGE_REJECTED_META_KEY] as
|
||||
RangeRejectionTally | undefined;
|
||||
}
|
||||
|
||||
function installActivityFetch(
|
||||
entries: Array<{ date: string; steps?: number; distance?: number }>,
|
||||
) {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => ({
|
||||
status: 200,
|
||||
headers: new Headers(),
|
||||
json: async () => ({
|
||||
status: 0,
|
||||
body: { activities: entries, more: false, offset: 0 },
|
||||
}),
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
event = new WideEventBuilder("background");
|
||||
vi.mocked(prisma.withingsConnection.findUnique).mockResolvedValue({
|
||||
id: "conn-1",
|
||||
withingsUserId: "wu-1",
|
||||
accessToken: "enc-access",
|
||||
refreshToken: "enc-refresh",
|
||||
scope: "user.metrics,user.activity",
|
||||
tokenExpiresAt: new Date(Date.now() + 60 * 60 * 1000),
|
||||
lastSyncedAt: new Date(Date.now() - 60 * 60 * 1000),
|
||||
} as never);
|
||||
vi.mocked(prisma.withingsConnection.update).mockResolvedValue({} as never);
|
||||
vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null);
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.measurement.create).mockResolvedValue({
|
||||
id: "m-1",
|
||||
type: "PULSE",
|
||||
measuredAt: new Date(),
|
||||
} as never);
|
||||
vi.mocked(prisma.measurement.createMany).mockResolvedValue({
|
||||
count: 1,
|
||||
} as never);
|
||||
vi.mocked(prisma.measurement.update).mockResolvedValue({} as never);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe("syncUserMeasurements — plausibility gate", () => {
|
||||
it("writes a reading inside the metric's declared band", async () => {
|
||||
vi.mocked(fetchMeasurements).mockResolvedValue([
|
||||
{
|
||||
type: "PULSE",
|
||||
value: 112,
|
||||
measuredAt: new Date("2026-05-16T08:00:00Z"),
|
||||
},
|
||||
] as never);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
syncUserMeasurements("user-1"),
|
||||
);
|
||||
|
||||
expect(result).toEqual({ imported: 1, failed: false });
|
||||
expect(prisma.measurement.create).toHaveBeenCalledTimes(1);
|
||||
expect(tally()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refuses an out-of-band reading and annotates the drop", async () => {
|
||||
vi.mocked(fetchMeasurements).mockResolvedValue([
|
||||
{
|
||||
type: "PULSE",
|
||||
value: 112,
|
||||
measuredAt: new Date("2026-05-16T08:00:00Z"),
|
||||
},
|
||||
{
|
||||
type: "PULSE",
|
||||
value: 55643821.505,
|
||||
measuredAt: new Date("2026-05-16T08:05:00Z"),
|
||||
},
|
||||
] as never);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
syncUserMeasurements("user-1"),
|
||||
);
|
||||
|
||||
// Only the real reading landed, and the run is still a clean success: one
|
||||
// impossible sample must not strand the rest behind a held watermark.
|
||||
expect(result).toEqual({ imported: 1, failed: false });
|
||||
expect(prisma.measurement.create).toHaveBeenCalledTimes(1);
|
||||
expect(tally()).toEqual({
|
||||
total: 1,
|
||||
buckets: [
|
||||
{ source: "withings", type: "PULSE", direction: "above_max", count: 1 },
|
||||
],
|
||||
});
|
||||
expect(event.getLevel()).toBe("warn");
|
||||
});
|
||||
|
||||
it("does not overwrite a stored reading with an out-of-band one", async () => {
|
||||
// The slot already holds a real pulse; the provider re-sends the same
|
||||
// instant carrying a decode slip. Ungated, the update branch rewrote the
|
||||
// good value with the bad one.
|
||||
vi.mocked(prisma.measurement.findFirst).mockResolvedValue({
|
||||
id: "existing-row",
|
||||
} as never);
|
||||
vi.mocked(fetchMeasurements).mockResolvedValue([
|
||||
{
|
||||
type: "PULSE",
|
||||
value: 55643821.505,
|
||||
measuredAt: new Date("2026-05-16T08:00:00Z"),
|
||||
},
|
||||
] as never);
|
||||
|
||||
const result = await eventStorage.run(event, () =>
|
||||
syncUserMeasurements("user-1"),
|
||||
);
|
||||
|
||||
expect(result).toEqual({ imported: 0, failed: false });
|
||||
expect(prisma.measurement.update).not.toHaveBeenCalled();
|
||||
expect(tally()?.total).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("syncUserActivity — plausibility gate", () => {
|
||||
it("plans a daily total inside the metric's declared band", async () => {
|
||||
installActivityFetch([{ date: "2026-05-16", steps: 8_400 }]);
|
||||
|
||||
const imported = await eventStorage.run(event, () =>
|
||||
syncUserActivity("user-1"),
|
||||
);
|
||||
|
||||
expect(imported).toBe(1);
|
||||
expect(prisma.measurement.createMany).toHaveBeenCalledTimes(1);
|
||||
expect(tally()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refuses an out-of-band daily total and annotates the drop", async () => {
|
||||
// ACTIVITY_STEPS is declared 0–200 000.
|
||||
installActivityFetch([
|
||||
{ date: "2026-05-16", steps: 8_400 },
|
||||
{ date: "2026-05-17", steps: 9_900_000 },
|
||||
]);
|
||||
|
||||
const imported = await eventStorage.run(event, () =>
|
||||
syncUserActivity("user-1"),
|
||||
);
|
||||
|
||||
expect(imported).toBe(1);
|
||||
const created = vi.mocked(prisma.measurement.createMany).mock.calls[0]?.[0]
|
||||
?.data as Array<{ value: number }>;
|
||||
expect(created).toHaveLength(1);
|
||||
expect(created[0]!.value).toBe(8_400);
|
||||
expect(tally()).toEqual({
|
||||
total: 1,
|
||||
buckets: [
|
||||
{
|
||||
source: "withings",
|
||||
type: "ACTIVITY_STEPS",
|
||||
direction: "above_max",
|
||||
count: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(event.getLevel()).toBe("warn");
|
||||
});
|
||||
});
|
||||
@@ -39,6 +39,7 @@ import { prisma } from "@/lib/db";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
import { safeFetch } from "@/lib/safe-fetch";
|
||||
import { getUnitForType } from "@/lib/validations/measurement";
|
||||
import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate";
|
||||
import {
|
||||
collapseToTypeDayKeys,
|
||||
recomputeBucketsForMeasurement,
|
||||
@@ -342,7 +343,7 @@ export async function syncUserActivity(
|
||||
value: number;
|
||||
externalId: string;
|
||||
}
|
||||
const planned: PlannedRow[] = [];
|
||||
const collected: PlannedRow[] = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.date) continue;
|
||||
const measuredAt = activityMeasuredAt(entry.date);
|
||||
@@ -352,7 +353,7 @@ export async function syncUserActivity(
|
||||
// valid (a day of rest still records 0 steps). Only skip
|
||||
// undefined / null / NaN.
|
||||
if (raw == null || !Number.isFinite(raw)) continue;
|
||||
planned.push({
|
||||
collected.push({
|
||||
type,
|
||||
measuredAt,
|
||||
value: raw,
|
||||
@@ -361,6 +362,15 @@ export async function syncUserActivity(
|
||||
}
|
||||
}
|
||||
|
||||
// A daily-activity total the app's own band calls impossible (a step count
|
||||
// past 200 000, a negative distance) is a provider artefact, not a day. Drop
|
||||
// it before the plan is read against the table, so it can neither insert a
|
||||
// fresh impossible slot nor overwrite a good stored one.
|
||||
const planned = dropImplausibleMeasurements("withings", collected, (p) => ({
|
||||
type: p.type,
|
||||
value: p.value,
|
||||
}));
|
||||
|
||||
if (planned.length > 0) {
|
||||
const slotKey = (type: MeasurementType, measuredAt: Date) =>
|
||||
`${type}|${measuredAt.getTime()}`;
|
||||
|
||||
@@ -474,6 +474,9 @@ export async function syncUserSleep(
|
||||
if (verdict.status === "failed") {
|
||||
throw new MeasurementReconciliationError(verdict);
|
||||
}
|
||||
// Refused by the plausibility gate: nothing was written and nothing
|
||||
// was touched. The drop is already on the wide event.
|
||||
if (verdict.status === "rejected_range") continue;
|
||||
for (const dirty of verdict.dirtyIdentities ?? []) {
|
||||
touched.push(dirty);
|
||||
}
|
||||
@@ -542,6 +545,9 @@ export async function syncUserSleep(
|
||||
if (verdict.status === "failed") {
|
||||
throw new MeasurementReconciliationError(verdict);
|
||||
}
|
||||
// Refused by the plausibility gate: nothing was written and
|
||||
// nothing was touched. The drop is already on the wide event.
|
||||
if (verdict.status === "rejected_range") continue;
|
||||
for (const dirty of verdict.dirtyIdentities ?? []) {
|
||||
touched.push(dirty);
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import { getUserWithingsCredentials } from "./credentials";
|
||||
import { annotate, getEvent } from "@/lib/logging/context";
|
||||
import { isP2002 } from "@/lib/prisma-errors";
|
||||
import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit";
|
||||
import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate";
|
||||
import {
|
||||
isReauthRequired,
|
||||
recordSyncFailure,
|
||||
@@ -229,14 +230,24 @@ export async function syncUserMeasurements(
|
||||
)
|
||||
: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); // 30 days back
|
||||
|
||||
let measures: Awaited<ReturnType<typeof fetchMeasurements>>;
|
||||
let fetched: Awaited<ReturnType<typeof fetchMeasurements>>;
|
||||
try {
|
||||
measures = await fetchMeasurements(tokenInfo.accessToken, startDate);
|
||||
fetched = await fetchMeasurements(tokenInfo.accessToken, startDate);
|
||||
} catch (err) {
|
||||
await recordWithingsSyncFailure(userId, err, WITHINGS_LEG_MEASURES);
|
||||
throw err;
|
||||
}
|
||||
|
||||
// The app's own plausibility band applies to a provider reading exactly as
|
||||
// it applies to a typed one. A remote that reports a pulse in the millions
|
||||
// is reporting a decode slip, not a heartbeat, and the row would poison
|
||||
// every mean and personal band derived from it. Refuse it here, before the
|
||||
// update branch below can overwrite a good stored value with it.
|
||||
const measures = dropImplausibleMeasurements("withings", fetched, (m) => ({
|
||||
type: m.type,
|
||||
value: m.value,
|
||||
}));
|
||||
|
||||
let imported = 0;
|
||||
// v1.4.39.1 — track every (type, measuredAt) we touched so the
|
||||
// persistent rollup tier can be re-folded at the end of the sync.
|
||||
|
||||
Reference in new issue
Block a user