Merge pull request #1293 from haritowa/documents-fact-date-from-document

src/app/api/documents/inbound/[id]/confirm/route.ts

@@ -43,7 +43,8 @@ async function confirmDocument(
 
   const document = await prisma.inboundDocument.findFirst({
     where: { id, userId: user.id, deletedAt: null },
-    select: { id: true },
+    // The two dates date a value that states none of its own.
+    select: { id: true, reportDate: true, documentDate: true },
   });
   if (!document) {
     return apiError("Document not found", 404, {
@@ -94,10 +95,15 @@ async function confirmDocument(
     try {
       // v1.43 — an OBSERVATION may name the marker the person chose
       // on the review screen; the label is then remembered as its alias.
-      const ref = await commitApprovedFact(user.id, fact, {
-        biomarkerId: decision.biomarkerId,
-        rememberAlias: decision.rememberAlias,
-      });
+      const ref = await commitApprovedFact(
+        user.id,
+        fact,
+        {
+          biomarkerId: decision.biomarkerId,
+          rememberAlias: decision.rememberAlias,
+        },
+        document,
+      );
       await prisma.extractedFact.update({
         where: { id: fact.id },
         data: {

src/lib/documents/__tests__/commit.test.ts

@@ -12,6 +12,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
 vi.mock("@/lib/db", () => ({
   prisma: {
     labResult: { create: vi.fn() },
+    illnessEpisode: { create: vi.fn() },
   },
 }));
 
@@ -27,8 +28,16 @@ import { commitApprovedFact, FactCommitError } from "@/lib/documents/commit";
 import { prisma } from "@/lib/db";
 import { resolveOrMintBiomarker } from "@/lib/labs/biomarker-store";
 import { decryptFactData } from "@/lib/documents/store";
-import type { ExtractedFact } from "@/generated/prisma/client";
+import type {
+  ExtractedFact,
+  IllnessEpisode,
+  LabResult,
+} from "@/generated/prisma/client";
 import { userDayKey } from "@/lib/tz/format";
+import type {
+  ConditionFactData,
+  ObservationFactData,
+} from "@/lib/validations/inbound-documents";
 
 function observationFact(): ExtractedFact {
   return {
@@ -213,3 +222,181 @@ describe("commitApprovedFact — OBSERVATION unit integrity", () => {
     ).resolves.toEqual({ recordType: "labResult", recordId: "lr-2" });
   });
 });
+
+describe("commitApprovedFact — OBSERVATION date when the value states none", () => {
+  // A lab report that prints the collection date once in its header carries
+  // no date beside each value. Mutation check: drop the document day from
+  // `takenAt` → the report-date, filing-date and impossible-date cases file
+  // the reading under today.
+  function undated(effectiveDate: string | null = null) {
+    const data: ObservationFactData = {
+      label: "Ferritin",
+      code: null,
+      codeSystem: null,
+      value: 80,
+      valueText: null,
+      unit: "ng/mL",
+      referenceLow: null,
+      referenceHigh: null,
+      effectiveDate,
+    };
+    vi.mocked(decryptFactData).mockReturnValue(data);
+    vi.mocked(resolveOrMintBiomarker).mockResolvedValue({
+      id: "bm-3",
+      name: "Ferritin",
+      unit: "ng/mL",
+      lowerBound: null,
+      upperBound: null,
+      panel: null,
+      analyteKey: null,
+    });
+    vi.mocked(prisma.labResult.create).mockResolvedValue({
+      id: "lr-3",
+    } as LabResult);
+  }
+
+  /** The dates as the confirm route reads them off the stored document. */
+  function dates(reportDate: string | null, documentDate: string | null) {
+    return {
+      // An older report date sits at UTC midnight, newer ones at noon UTC.
+      reportDate: reportDate ? new Date(`${reportDate}T00:00:00.000Z`) : null,
+      documentDate: documentDate
+        ? new Date(`${documentDate}T12:00:00.000Z`)
+        : null,
+    };
+  }
+
+  function storedTakenAt(): Date {
+    return vi.mocked(prisma.labResult.create).mock.calls[0]![0].data
+      .takenAt as Date;
+  }
+
+  it("keeps the value's own date", async () => {
+    undated("2021-03-04");
+
+    await commitApprovedFact(
+      "user-1",
+      observationFact(),
+      {},
+      dates("2021-03-01", "2026-10-20"),
+    );
+
+    expect(storedTakenAt().toISOString()).toBe("2021-03-04T12:00:00.000Z");
+  });
+
+  it("takes the report date the document states over its filing date", async () => {
+    undated();
+
+    await commitApprovedFact(
+      "user-1",
+      observationFact(),
+      {},
+      dates("2021-03-01", "2026-10-20"),
+    );
+
+    expect(storedTakenAt().toISOString()).toBe("2021-03-01T12:00:00.000Z");
+  });
+
+  it("takes the filing date when the document states no report date", async () => {
+    undated();
+
+    await commitApprovedFact(
+      "user-1",
+      observationFact(),
+      {},
+      dates(null, "2021-02-15"),
+    );
+
+    expect(storedTakenAt().toISOString()).toBe("2021-02-15T12:00:00.000Z");
+  });
+
+  it.each([
+    ["not a date", "March 2021"],
+    ["a day the month does not have", "2021-02-30"],
+    ["a month that does not exist", "2021-13-01"],
+  ])(
+    "reads a value whose own date is %s as undated",
+    async (_label, effectiveDate) => {
+      undated(effectiveDate);
+
+      await commitApprovedFact(
+        "user-1",
+        observationFact(),
+        {},
+        dates("2021-03-01", null),
+      );
+
+      // Not 2 March for 30 February, and no Invalid Date for month 13.
+      expect(storedTakenAt().toISOString()).toBe("2021-03-01T12:00:00.000Z");
+    },
+  );
+
+  it("files the reading under now when neither the value nor the document has a date", async () => {
+    undated();
+
+    const before = Date.now();
+    await commitApprovedFact(
+      "user-1",
+      observationFact(),
+      {},
+      dates(null, null),
+    );
+    const after = Date.now();
+
+    const takenAt = storedTakenAt().getTime();
+    expect(takenAt).toBeGreaterThanOrEqual(before);
+    expect(takenAt).toBeLessThanOrEqual(after);
+  });
+});
+
+describe("commitApprovedFact — CONDITION onset", () => {
+  // The onset goes through the same stated-day check as a lab value's date.
+  function condition(onsetDate: string | null) {
+    const data: ConditionFactData = {
+      label: "Iron deficiency",
+      code: null,
+      codeSystem: null,
+      clinicalStatus: null,
+      verificationStatus: null,
+      onsetDate,
+    };
+    vi.mocked(decryptFactData).mockReturnValue(data);
+    vi.mocked(prisma.illnessEpisode.create).mockResolvedValue({
+      id: "ep-1",
+    } as IllnessEpisode);
+  }
+
+  function conditionFact(): ExtractedFact {
+    return {
+      id: "fact-2",
+      factType: "CONDITION",
+      dataEncrypted: new Uint8Array(),
+    } as unknown as ExtractedFact;
+  }
+
+  function storedOnset(): Date {
+    return vi.mocked(prisma.illnessEpisode.create).mock.calls[0]![0].data
+      .onsetAt as Date;
+  }
+
+  it("keeps a stated onset day", async () => {
+    condition("2021-03-04");
+
+    await commitApprovedFact("user-1", conditionFact());
+
+    expect(storedOnset().toISOString()).toBe("2021-03-04T12:00:00.000Z");
+  });
+
+  it("files an onset that is not a real day under now, not an invalid date", async () => {
+    // 2021-13-01 used to become an invalid date that failed the whole confirm.
+    condition("2021-13-01");
+
+    const before = Date.now();
+    await commitApprovedFact("user-1", conditionFact());
+    const after = Date.now();
+
+    const onset = storedOnset().getTime();
+    expect(onset).toBeGreaterThanOrEqual(before);
+    expect(onset).toBeLessThanOrEqual(after);
+  });
+});

src/lib/documents/commit.ts

@@ -44,7 +44,11 @@ import type {
   MedicationStatementFactData,
   ObservationFactData,
 } from "@/lib/validations/inbound-documents";
-import { dateOnlyAtNoonUtc, isDateOnlyKey } from "@/lib/tz/date-only";
+import {
+  dateOnlyAtNoonUtc,
+  dateOnlyKey,
+  isCalendarDateKey,
+} from "@/lib/tz/date-only";
 
 /** A per-fact commit failure the caller maps to a per-fact 422 entry. */
 export class FactCommitError extends Error {
@@ -61,14 +65,41 @@ export interface CommittedRecordRef {
   recordId: string;
 }
 
+/** A stated day, or null when it is not a real calendar date. */
+function statedDay(date: string | null): string | null {
+  return date && isCalendarDateKey(date) ? date : null;
+}
+
 /**
  * A stated YYYY-MM-DD as the instant a date-only value is stored at (noon
- * UTC), or now when the document states none. UTC midnight read back as the
+ * UTC), else the fallback day, else now. UTC midnight read back as the
  * previous day anywhere west of UTC.
  */
-function statedDateOrNow(date: string | null): Date {
-  if (date && isDateOnlyKey(date)) return dateOnlyAtNoonUtc(date);
-  return new Date();
+function statedDateOrNow(
+  date: string | null,
+  fallback: string | null = null,
+): Date {
+  const day = statedDay(date) ?? fallback;
+  return day ? dateOnlyAtNoonUtc(day) : new Date();
+}
+
+/** The two dates a stored document carries, as the confirm route reads them. */
+export interface FactDocumentDates {
+  reportDate: Date | null;
+  documentDate: Date | null;
+}
+
+/**
+ * The day a value from this document took place when it states none of its
+ * own: the report date the model read off the document, else the document's
+ * filing date, the same order the day view and the timeline place a document
+ * by. A lab report that prints the collection date once in its header carries
+ * no date beside each value, and every reading used to be filed under the day
+ * it was confirmed.
+ */
+function documentDay(dates: FactDocumentDates | null): string | null {
+  const day = dates?.reportDate ?? dates?.documentDate ?? null;
+  return day ? dateOnlyKey(day) : null;
 }
 
 /**
@@ -84,6 +115,7 @@ async function commitObservation(
   userId: string,
   data: ObservationFactData,
   choice: ObservationChoice = {},
+  fallbackDay: string | null = null,
 ): Promise<CommittedRecordRef> {
   const isQualitative = typeof data.value !== "number";
   // A numeric reading needs a unit so the catalog has something to mint with —
@@ -228,7 +260,7 @@ async function commitObservation(
       sourceReferenceLow: storedSourceLow,
       sourceReferenceHigh: storedSourceHigh,
       sourceReferenceText: isQualitative ? null : sourceText,
-      takenAt: statedDateOrNow(data.effectiveDate),
+      takenAt: statedDateOrNow(data.effectiveDate, fallbackDay),
       source: "DOCUMENT",
       noteEncrypted: null,
     },
@@ -317,12 +349,18 @@ export async function commitApprovedFact(
   userId: string,
   fact: ExtractedFact,
   choice: ObservationChoice = {},
+  documentDates: FactDocumentDates | null = null,
 ): Promise<CommittedRecordRef> {
   // Decrypt the staged payload at confirm time to write it into the normal
   // structured store. Fail-closed: a bad key id throws and aborts the commit.
   const data = decryptFactData(fact.dataEncrypted);
   if (fact.factType === "OBSERVATION") {
-    return commitObservation(userId, data as ObservationFactData, choice);
+    return commitObservation(
+      userId,
+      data as ObservationFactData,
+      choice,
+      documentDay(documentDates),
+    );
   }
   if (fact.factType === "CONDITION") {
     return commitCondition(userId, data as ConditionFactData);

tests/integration/document-undated-lab-value.test.ts

@@ -0,0 +1,222 @@
+/**
+ * A lab value without its own date, confirmed from a stored document, against
+ * a real Postgres and through the real confirm route.
+ *
+ * A lab report usually prints the collection date once, in its header. The
+ * extraction puts that date on the document (`reportDate`) and leaves each
+ * value's own date empty, so the confirm step has to date the value by its
+ * document. It used to file every such reading under the moment it was
+ * confirmed, so a report from years ago showed up as today's results.
+ *
+ * The route reads the document's two dates once and hands them to the commit
+ * step: the report date first, else the filing date (`documentDate`), the order
+ * the day view places a document by. A value's own date still wins, but only
+ * when it names a real calendar day.
+ */
+import { NextRequest } from "next/server";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import { cookieJar } from "./mock-next-headers";
+import { getPrismaClient, truncateAllTables } from "./setup";
+
+const USER_ID = "user-undated-lab-value";
+
+vi.mock("next/headers", async () => {
+  const { cookieJar, headerJar } = await import("./mock-next-headers");
+  return {
+    headers: vi.fn(async () => ({
+      get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
+    })),
+    cookies: vi.fn(async () => ({
+      get: (name: string) => {
+        const value = cookieJar.get(name);
+        return value ? { name, value } : undefined;
+      },
+      set: (name: string, value: string) => {
+        cookieJar.set(name, value);
+      },
+      delete: (name: string) => {
+        cookieJar.delete(name);
+      },
+    })),
+  };
+});
+
+vi.mock("@/lib/db-compat", () => ({
+  ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
+}));
+
+beforeEach(async () => {
+  await truncateAllTables(getPrismaClient());
+  cookieJar.clear();
+  const prisma = getPrismaClient();
+  await prisma.user.create({
+    data: {
+      id: USER_ID,
+      username: "undated-lab-value",
+      email: "undated-lab-value@example.test",
+      modulePreferencesJson: { inboundDocuments: true, labs: true },
+    },
+  });
+  const session = await prisma.session.create({
+    data: { userId: USER_ID, expiresAt: new Date(Date.now() + 3_600_000) },
+  });
+  cookieJar.set("healthlog_session", session.id);
+  for (const [name, unit] of [
+    ["Ferritin", "ng/mL"],
+    ["Glucose", "mmol/L"],
+    ["Haemoglobin", "g/L"],
+  ]) {
+    await prisma.biomarker.create({ data: { userId: USER_ID, name, unit } });
+  }
+});
+
+const noon = (day: string) => new Date(`${day}T12:00:00.000Z`);
+
+async function storedDocument(dates: {
+  reportDate: string | null;
+  documentDate: string | null;
+}) {
+  return getPrismaClient().inboundDocument.create({
+    data: {
+      userId: USER_ID,
+      filename: "befund.pdf",
+      mimeType: "application/pdf",
+      byteSize: 1024,
+      contentEncrypted: Buffer.from("stand-in for the stored bytes"),
+      status: "EXTRACTED",
+      reportDate: dates.reportDate ? noon(dates.reportDate) : null,
+      documentDate: dates.documentDate ? noon(dates.documentDate) : null,
+    },
+  });
+}
+
+async function stagedValue(
+  documentId: string,
+  label: string,
+  value: number,
+  unit: string,
+  effectiveDate: string | null,
+) {
+  const { encryptFactData, encryptFactProvenance } =
+    await import("@/lib/documents/store");
+  return getPrismaClient().extractedFact.create({
+    data: {
+      documentId,
+      userId: USER_ID,
+      factType: "OBSERVATION",
+      status: "PENDING",
+      confidence: 0.95,
+      needsReview: false,
+      dataEncrypted: encryptFactData({
+        label,
+        code: null,
+        codeSystem: null,
+        value,
+        valueText: null,
+        unit,
+        referenceLow: null,
+        referenceHigh: null,
+        referenceText: null,
+        effectiveDate,
+      }),
+      provenanceEncrypted: encryptFactProvenance({
+        sourceText: `${label} ${value} ${unit}`,
+        anchored: true,
+        sourceOffset: 0,
+        page: 0,
+        confidence: 0.95,
+      }),
+    },
+  });
+}
+
+async function confirm(documentId: string, factIds: string[]) {
+  const { POST } =
+    await import("@/app/api/documents/inbound/[id]/confirm/route");
+  const res = await POST(
+    new NextRequest(
+      `http://localhost/api/documents/inbound/${documentId}/confirm`,
+      {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({
+          decisions: factIds.map((factId) => ({ factId, action: "approve" })),
+        }),
+      },
+    ),
+    { params: Promise.resolve({ id: documentId }) },
+  );
+  expect(res.status).toBe(200);
+  const body = await res.json();
+  expect(body.data.failed).toEqual([]);
+  expect(body.data.approved).toHaveLength(factIds.length);
+  return body.data.approved as { factId: string; recordId: string }[];
+}
+
+async function takenAtOf(recordId: string) {
+  const row = await getPrismaClient().labResult.findUniqueOrThrow({
+    where: { id: recordId },
+  });
+  return row.takenAt.toISOString();
+}
+
+describe("an undated lab value is dated by its document", () => {
+  it("takes the report date over the filing date, keeps a value's own date, and ignores an impossible one", async () => {
+    const today = new Date().toISOString().slice(0, 10);
+    // Collection date printed once in the header; filed today (the upload-day
+    // default for documentDate).
+    const doc = await storedDocument({
+      reportDate: "2021-03-01",
+      documentDate: today,
+    });
+    const undated = await stagedValue(doc.id, "Ferritin", 80, "ng/mL", null);
+    const ownDate = await stagedValue(
+      doc.id,
+      "Glucose",
+      5.1,
+      "mmol/L",
+      "2021-02-20",
+    );
+    const impossible = await stagedValue(
+      doc.id,
+      "Haemoglobin",
+      142,
+      "g/L",
+      "2021-02-30",
+    );
+
+    const approved = await confirm(doc.id, [
+      undated.id,
+      ownDate.id,
+      impossible.id,
+    ]);
+    const recordOf = (factId: string) =>
+      approved.find((a) => a.factId === factId)!.recordId;
+
+    expect(await takenAtOf(recordOf(undated.id))).toBe(
+      "2021-03-01T12:00:00.000Z",
+    );
+    expect(await takenAtOf(recordOf(ownDate.id))).toBe(
+      "2021-02-20T12:00:00.000Z",
+    );
+    // Not 2 March: a day the month does not have is no date at all.
+    expect(await takenAtOf(recordOf(impossible.id))).toBe(
+      "2021-03-01T12:00:00.000Z",
+    );
+  });
+
+  it("takes the filing date when the document has no report date", async () => {
+    const doc = await storedDocument({
+      reportDate: null,
+      documentDate: "2021-02-15",
+    });
+    const undated = await stagedValue(doc.id, "Ferritin", 80, "ng/mL", null);
+
+    const [approved] = await confirm(doc.id, [undated.id]);
+
+    expect(await takenAtOf(approved!.recordId)).toBe(
+      "2021-02-15T12:00:00.000Z",
+    );
+  });
+});