fix(labs): keep a deleted Health Records result deleted once its row is gone

The import recognised a deleted result by the identity on its own
tombstone, so anything that removed the row brought the result back
with the next Apple Health export or resend: deleting the whole marker,
which hard-deletes its readings, the nightly tombstone purge, and a
portable backup, which leaves deleted readings out.

The identities now move into an import ledger (LabResultImportKey,
migration 0501) in the same transaction as the marker delete and the
purge, and the import checks it beside the stored rows. The backup
carries the ledger, a portable file also the identities of deleted
readings still waiting as tombstones; the restore writes it back
without the identities a restored reading holds. The data wipe clears
it, account deletion cascades.
This commit is contained in:
Marc-André Bombeck committed 2026-11-17 21:20:55 +01:00
1 parent 0fedabde13
commit f3662a8dc8
38 files changed
+2356 -1635

No files matched your search

+1 -1
View File
@@ -19,7 +19,7 @@
stay under for any nutrient on the energy card; the chart draws your number
instead of the reference. Targets travel in the full backup and are read
and set through `GET` and `PATCH /api/nutrients/targets`. (#1402)
- Lab results from Apple Health's Health Records now land in Labs. An Apple Health export used to skip every clinical record; the lab results among them are now read, matched to your existing markers (by name, alias or LOINC code, with a reading in another unit converted and the printed value kept), and filed under their panel. A result like "<0.5 mg/L" is kept as printed, the report's own reference range comes along, and preliminary or withdrawn results are left out. Importing the same results again writes nothing, a result you delete stays deleted, and a reading you already typed or scanned is not doubled. If you imported an archive before, upload it once more to pick up its lab results. For the iOS app, `POST /api/labs/fhir-batch` takes the same results straight from HealthKit. Thanks to @camiracode for asking for this. (#1400)
- Lab results from Apple Health's Health Records now land in Labs. An Apple Health export used to skip every clinical record; the lab results among them are now read, matched to your existing markers (by name, alias or LOINC code, with a reading in another unit converted and the printed value kept), and filed under their panel. A result like "<0.5 mg/L" is kept as printed, the report's own reference range comes along, and preliminary or withdrawn results are left out. Importing the same results again writes nothing, a result you delete stays deleted (also when you delete its whole marker, long after the deletion, and across a backup restore), and a reading you already typed or scanned is not doubled. If you imported an archive before, upload it once more to pick up its lab results. For the iOS app, `POST /api/labs/fhir-batch` takes the same results straight from HealthKit. Thanks to @camiracode for asking for this. (#1400)
### Changed
+22
View File
@@ -45678,6 +45678,27 @@ components:
- readingCount
- mergedAt
additionalProperties: {}
labResultImportKeys:
default: []
type: array
items:
type: object
properties:
source:
type: string
minLength: 1
maxLength: 32
externalId:
type: string
minLength: 1
maxLength: 128
createdAt:
type: string
minLength: 1
required:
- source
- externalId
additionalProperties: {}
illnessEpisodes:
default: []
type: array
@@ -49012,6 +49033,7 @@ components:
- labResults
- biomarkers
- biomarkerMerges
- labResultImportKeys
- illnessEpisodes
- symptomDefinitions
- lifeEvents
@@ -0,0 +1,32 @@
-- The identities of Health Records lab readings the person deleted, kept once
-- the reading's row is gone (#1400). A deleted reading remembers its identity
-- on its own tombstone; the nightly purge and deleting a whole marker remove
-- the row, and with it the only memory that the person deleted it, so the
-- next Apple Health export wrote the reading again. Those two now move the
-- identity here first. One row per (user, source, identity): the digest, no
-- lab content.
--
-- A new, empty table; nothing is rewritten. IF NOT EXISTS keeps a replay on a
-- database that already ran a build carrying it harmless. Reversibility:
-- dropping the table forgets the deletions it remembers; readings already
-- purged would be imported again on the next export.
--
-- @retry-on-lock-timeout
DO $migration$
BEGIN
PERFORM set_config('lock_timeout', '5s', true);
CREATE TABLE IF NOT EXISTS "lab_result_import_keys" (
"id" TEXT NOT NULL,
"user_id" TEXT NOT NULL,
"source" TEXT NOT NULL,
"external_id" TEXT NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "lab_result_import_keys_pkey" PRIMARY KEY ("id"),
CONSTRAINT "lab_result_import_keys_user_id_fkey" FOREIGN KEY ("user_id")
REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE
);
CREATE UNIQUE INDEX IF NOT EXISTS "lab_result_import_keys_user_id_source_external_id_key"
ON "lab_result_import_keys" ("user_id", "source", "external_id");
END
$migration$;
+29 -1
View File
@@ -1083,6 +1083,9 @@ model User {
// v1.39.2 — source keys of imported documents that the tombstone purge
// removed, so a re-run of an importer cannot bring a deleted document back.
documentImportKeys DocumentImportKey[]
// v1.47.7 — identities of Health Records lab readings the person deleted,
// kept once the reading's row is gone. See LabResultImportKey.
labResultImportKeys LabResultImportKey[]
// v1.39.2 — further source keys an import answered with an existing
// document (same bytes under another key). See DocumentSourceAlias.
documentSourceAliases DocumentSourceAlias[]
@@ -6666,7 +6669,8 @@ model LabResult {
/// it has none). A digest, not the identifier and not ciphertext. NULL for
/// every reading typed, scanned or imported from CSV. Read by
/// `writeFhirLabReadings` (`src/lib/labs/fhir-import/write.ts`), which treats
/// a reading it already holds under this id, deleted or not, as a duplicate.
/// a reading it already holds under this id, deleted or not, as a duplicate;
/// once a deleted reading's row is gone, `LabResultImportKey` holds the id.
externalId String? @map("external_id")
/// Optional free-text note, AES-256-GCM encrypted at rest via
/// `src/lib/crypto.ts` (Bytes column, `*Encrypted` suffix per convention).
@@ -6692,6 +6696,30 @@ model LabResult {
@@map("lab_results")
}
/// v1.47.7 (#1400) — the identity of a Health Records lab reading the person
/// deleted, once the reading's row is gone. While a deleted reading waits as
/// a tombstone, its own `externalId` is the memory that the person deleted it
/// and the import answers "duplicate". When the row leaves the database (the
/// tombstone purge, or deleting the reading's whole marker) the identity moves
/// here in the same transaction, so an Apple Health export or the phone
/// sending the result again still writes nothing. Read by
/// `writeFhirLabReadings`, carried by the backup. Holds the digest only, no
/// lab content.
model LabResultImportKey {
id String @id @default(cuid())
userId String @map("user_id")
/// The `LabResult.source` the identity arrived under ("APPLE_HEALTH").
source String
/// The `LabResult.externalId` digest.
externalId String @map("external_id")
createdAt DateTime @default(now()) @map("created_at")
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
@@unique([userId, source, externalId])
@@map("lab_result_import_keys")
}
// ─── Biomarker catalog ───────────────────────────────────────
/// v1.19.0 — user-scoped catalog of blood/lab biomarkers. Defining a marker
@@ -1,7 +1,7 @@
{
"$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
"count": 84,
"total": 93813,
"total": 93815,
"entries": {
"app/medications/page-client.tsx": 842,
"components/charts/mood-chart.tsx": 1166,
@@ -42,7 +42,7 @@
"lib/dashboard-layout.ts": 977,
"lib/dashboard/snapshot.ts": 1548,
"lib/day/records.ts": 1172,
"lib/export/backup-plan.ts": 838,
"lib/export/backup-plan.ts": 840,
"lib/export/coach-backup.ts": 1070,
"lib/export/profile-backup.ts": 1019,
"lib/fitbit/client.ts": 1262,
@@ -22,13 +22,18 @@ vi.mock("@/lib/api-handler", async (importOriginal) => {
};
});
vi.mock("@/lib/db", () => ({
prisma: {
vi.mock("@/lib/db", () => {
const prisma = {
biomarker: { findFirst: vi.fn(), delete: vi.fn() },
labResult: { deleteMany: vi.fn() },
$transaction: vi.fn(async (ops: Promise<unknown>[]) => Promise.all(ops)),
},
}));
labResult: { deleteMany: vi.fn(), findMany: vi.fn(async () => []) },
labResultImportKey: { createMany: vi.fn() },
$transaction: vi.fn(),
};
prisma.$transaction.mockImplementation(
async (fn: (tx: typeof prisma) => unknown) => fn(prisma),
);
return { prisma };
});
vi.mock("@/lib/auth/audit", () => ({
auditLog: vi.fn().mockResolvedValue(undefined),
}));
@@ -79,11 +84,20 @@ describe("DELETE /api/biomarkers/[id] record authority", () => {
expect(requireAuth).not.toHaveBeenCalled();
});
it("narrows both delete legs to the resolved record", async () => {
it("narrows every delete leg to the resolved record", async () => {
await DELETE(delReq(), params);
expect(prisma.labResult.deleteMany).toHaveBeenCalledWith({
where: { userId: "record-1", biomarkerId: "bm-1" },
});
// The identities the import ledger keeps come from the same rows.
expect(prisma.labResult.findMany).toHaveBeenCalledWith(
expect.objectContaining({
where: expect.objectContaining({
userId: "record-1",
biomarkerId: "bm-1",
}),
}),
);
});
it("refuses when the record authority refuses, deleting nothing", async () => {
@@ -17,10 +17,11 @@ vi.mock("@/lib/db", () => {
},
biomarkerMerge: { findFirst: vi.fn(async () => null) },
labResult: { deleteMany: vi.fn(), findMany: vi.fn(async () => []) },
labResultImportKey: { createMany: vi.fn() },
auditLog: { create: vi.fn().mockResolvedValue({}) },
};
// `$transaction` is set in `beforeEach`: the batch form for the delete, the
// callback form for the edit.
// `$transaction` is set in `beforeEach`: the callback form for the delete
// and the edit, the batch form where a caller still uses it.
return { prisma: { ...prisma, $transaction: vi.fn() } };
});
@@ -158,6 +159,12 @@ describe("DELETE /api/biomarkers/[id]", () => {
vi.mocked(prisma.labResult.deleteMany).mockResolvedValue({
count: 3,
} as never);
vi.mocked(prisma.labResult.findMany).mockResolvedValue([
{ source: "APPLE_HEALTH", externalId: "fhir:a" },
] as never);
vi.mocked(prisma.labResultImportKey.createMany).mockResolvedValue({
count: 1,
} as never);
vi.mocked(prisma.biomarker.delete).mockResolvedValue(EXISTING as never);
const res = await DELETE(delReq(), params("bm-1"));
@@ -171,6 +178,13 @@ describe("DELETE /api/biomarkers/[id]", () => {
expect(prisma.biomarker.delete).toHaveBeenCalledWith({
where: { id: "bm-1" },
});
// A Health Records reading's identity outlives it in the import ledger.
expect(prisma.labResultImportKey.createMany).toHaveBeenCalledWith({
data: [
{ userId: "user-1", source: "APPLE_HEALTH", externalId: "fhir:a" },
],
skipDuplicates: true,
});
expect(prisma.$transaction).toHaveBeenCalledTimes(1);
});
+12 -8
View File
@@ -27,6 +27,7 @@ import { matchAnalyteKey } from "@/lib/labs/analyte-match";
import { sameLabUnit } from "@/lib/labs/unit-normalise";
import { annotate } from "@/lib/logging/context";
import { computeLabTrend } from "@/lib/labs/lab-trend";
import { retireLabImportKeys } from "@/lib/labs/fhir-import/import-keys";
/** Upper bound on readings the trend line reads. */
const LAB_TREND_READ_CAP = 2000;
@@ -319,14 +320,17 @@ export const DELETE = apiHandler(
// Hard delete the marker AND its readings in one transaction. Deleting a
// biomarker is a deliberate "remove this and its history" action, so the
// readings go with it rather than lingering as orphaned, unlinked rows.
// Both legs are userId-narrowed so a cross-user id can never reach another
// account's data.
const [removedReadings] = await prisma.$transaction([
prisma.labResult.deleteMany({
where: { userId: user.id, biomarkerId: id },
}),
prisma.biomarker.delete({ where: { id } }),
]);
// Every leg is userId-narrowed so a cross-user id can never reach another
// account's data. A reading imported from Health Records leaves its
// identity in the import ledger first, so the next export or resend does
// not write it again.
const removedReadings = await prisma.$transaction(async (tx) => {
const where = { userId: user.id, biomarkerId: id };
await retireLabImportKeys(tx, user.id, where);
const removed = await tx.labResult.deleteMany({ where });
await tx.biomarker.delete({ where: { id } });
return removed;
});
invalidateUserHealthScore(user.id);
await auditLog("biomarker.delete", {
@@ -95,6 +95,7 @@ vi.mock("@/lib/db", () => ({
biomarker: { findMany: vi.fn() },
biomarkerAlias: { findMany: vi.fn() },
biomarkerMerge: { findMany: vi.fn() },
labResultImportKey: { findMany: vi.fn(async () => []) },
illnessEpisode: { findMany: vi.fn() },
allergy: { findMany: vi.fn() },
familyHistoryEntry: { findMany: vi.fn() },
@@ -69,6 +69,7 @@ vi.mock("@/lib/db", () => ({
biomarker: { findMany: vi.fn() },
biomarkerAlias: { findMany: vi.fn() },
biomarkerMerge: { findMany: vi.fn() },
labResultImportKey: { findMany: vi.fn(async () => []) },
allergy: { findMany: vi.fn() },
familyHistoryEntry: { findMany: vi.fn() },
workout: { findMany: vi.fn() },
@@ -86,6 +86,7 @@ vi.mock("@/lib/db", () => ({
biomarker: { findMany: vi.fn().mockResolvedValue([]) },
biomarkerAlias: { findMany: vi.fn().mockResolvedValue([]) },
biomarkerMerge: { findMany: vi.fn().mockResolvedValue([]) },
labResultImportKey: { findMany: vi.fn().mockResolvedValue([]) },
illnessEpisode: { findMany: vi.fn().mockResolvedValue([]) },
allergy: { findMany: vi.fn().mockResolvedValue([]) },
familyHistoryEntry: { findMany: vi.fn().mockResolvedValue([]) },
+4 -1
View File
@@ -81,7 +81,10 @@ export async function importAppleHealthClinicalLabs(input: {
userId: string;
tz: string;
references: readonly ClinicalLabReference[];
prisma?: Pick<PrismaClient, "labResult" | "biomarker" | "biomarkerAlias">;
prisma?: Pick<
PrismaClient,
"labResult" | "biomarker" | "biomarkerAlias" | "labResultImportKey"
>;
limits?: AppleHealthClinicalArchiveLimits;
}): Promise<FhirLabCounts> {
const counts = emptyFhirLabCounts();
+4
View File
@@ -136,6 +136,10 @@ export const WIPE_MODELS = [
// ── Clinical record ─────────────────────────────────────────────────────
"MentalHealthAssessment",
"LabResult",
// The identities of Health Records readings the person deleted (#1400).
// Kept, they would make the first import after the wipe skip results for
// a decision about a record that no longer exists.
"LabResultImportKey",
// v1.43 — a marker's other spellings and the merges folded into it. Both
// cascade from the marker; named before it so their counts read true.
"BiomarkerAlias",
@@ -464,6 +464,7 @@ function makePrisma() {
documentConditionLink: { findMany: vi.fn().mockResolvedValue([]) },
extractedFact: { findMany: vi.fn().mockResolvedValue([]) },
documentImportKey: { findMany: vi.fn().mockResolvedValue([]) },
labResultImportKey: { findMany: vi.fn().mockResolvedValue([]) },
inboundDocument: { findMany: vi.fn().mockResolvedValue([]) },
// The ECG strips. Empty for the same reason as the sections above:
// an account whose watch has never taken one is the ordinary case the
@@ -75,6 +75,7 @@ function buildMockPrisma() {
},
biomarkerAlias: { findMany: vi.fn().mockResolvedValue([]) },
biomarkerMerge: { findMany: vi.fn().mockResolvedValue([]) },
labResultImportKey: { findMany: vi.fn().mockResolvedValue([]) },
illnessEpisode: {
findMany: vi.fn().mockResolvedValue([
{
@@ -473,6 +474,7 @@ describe("buildRecordsBackupSection", () => {
expect(counts).toEqual({
labResults: 1,
labResultImportKeys: 0,
biomarkers: 1,
illnessEpisodes: 2,
illnessDayLogs: 1,
@@ -167,6 +167,7 @@ function sourceClient() {
biomarker: empty,
biomarkerAlias: empty,
biomarkerMerge: empty,
labResultImportKey: empty,
illnessEpisode: empty,
allergy: empty,
familyHistoryEntry: empty,
+2
View File
@@ -101,6 +101,7 @@ export const BACKED_UP_MODELS = [
// ── Clinical record ───────────────────────────────────────────────────────
"MentalHealthAssessment",
"LabResult",
"LabResultImportKey", // Deleted Health Records readings: `labs-backup.ts`.
"Biomarker",
"Allergy",
"FamilyHistoryEntry",
@@ -365,6 +366,7 @@ export const TWO_ENDED_MODELS = [
"MoodEntryTagLink",
"MoodTag",
"LabResult",
"LabResultImportKey",
"Biomarker",
"Allergy",
"FamilyHistoryEntry",
+1
View File
@@ -157,6 +157,7 @@ const SUMMARY_COUNTS = [
"cycles",
"cycleDayLogs",
"labResults",
"labResultImportKeys",
"nutrientDays",
"biomarkers",
"illnessEpisodes",
+12
View File
@@ -117,3 +117,15 @@ export const biomarkerMergeBackupSchema = z
mergedAt: isoDateTime,
})
.passthrough();
/**
* The identity of a Health Records reading the person deleted (the import
* ledger). Bounded like the reading's own `externalId`.
*/
export const labResultImportKeyBackupSchema = z
.object({
source: z.string().min(1).max(32),
externalId: z.string().min(1).max(128),
createdAt: isoDateTime.optional(),
})
.passthrough();
+137 -31
View File
@@ -145,11 +145,28 @@ export interface BiomarkerMergeBackupEntry {
mergedAt: string;
}
/** The lab slice of the records section: results, markers, merge ledger. */
/**
* The identity of a Health Records reading the person deleted (the import
* ledger, `LabResultImportKey`). A digest, no lab content.
*/
export interface LabResultImportKeyBackupEntry {
source: string;
externalId: string;
createdAt: string;
}
/**
* The lab slice of the records section: results, markers, merge ledger, and
* the identities of the Health Records readings the person deleted.
*/
export async function buildLabsBackupSlice(
prisma: Pick<
PrismaClient,
"labResult" | "biomarker" | "biomarkerAlias" | "biomarkerMerge"
| "labResult"
| "biomarker"
| "biomarkerAlias"
| "biomarkerMerge"
| "labResultImportKey"
>,
userId: string,
disasterRecovery: boolean,
@@ -157,35 +174,42 @@ export async function buildLabsBackupSlice(
labResults: LabResultBackupEntry[];
biomarkers: BiomarkerBackupEntry[];
biomarkerMerges: BiomarkerMergeBackupEntry[];
labResultImportKeys: LabResultImportKeyBackupEntry[];
}> {
const [labResultRows, biomarkerRows, biomarkerAliasRows, biomarkerMergeRows] =
await Promise.all([
prisma.labResult.findMany({
where: disasterRecovery ? { userId } : { userId, deletedAt: null },
orderBy: [{ takenAt: "desc" }, { id: "asc" }],
include: { biomarker: { select: { name: true } } },
}),
prisma.biomarker.findMany({
where: { userId },
orderBy: [{ name: "asc" }, { id: "asc" }],
}),
prisma.biomarkerAlias.findMany({
where: { userId },
orderBy: [{ biomarkerId: "asc" }, { nameFolded: "asc" }, { id: "asc" }],
select: {
id: true,
biomarkerId: true,
name: true,
source: true,
createdAt: true,
},
}),
prisma.biomarkerMerge.findMany({
where: { userId },
orderBy: [{ mergedAt: "asc" }, { id: "asc" }],
include: { target: { select: { name: true } } },
}),
]);
const [
labResultRows,
biomarkerRows,
biomarkerAliasRows,
biomarkerMergeRows,
labResultImportKeys,
] = await Promise.all([
prisma.labResult.findMany({
where: disasterRecovery ? { userId } : { userId, deletedAt: null },
orderBy: [{ takenAt: "desc" }, { id: "asc" }],
include: { biomarker: { select: { name: true } } },
}),
prisma.biomarker.findMany({
where: { userId },
orderBy: [{ name: "asc" }, { id: "asc" }],
}),
prisma.biomarkerAlias.findMany({
where: { userId },
orderBy: [{ biomarkerId: "asc" }, { nameFolded: "asc" }, { id: "asc" }],
select: {
id: true,
biomarkerId: true,
name: true,
source: true,
createdAt: true,
},
}),
prisma.biomarkerMerge.findMany({
where: { userId },
orderBy: [{ mergedAt: "asc" }, { id: "asc" }],
include: { target: { select: { name: true } } },
}),
buildDeletedLabImportKeys(prisma, userId, disasterRecovery),
]);
const labResults: LabResultBackupEntry[] = labResultRows.map((r) => ({
...(disasterRecovery
@@ -288,7 +312,56 @@ export async function buildLabsBackupSlice(
aliases: aliasesByMarker.get(b.id) ?? [],
}));
return { labResults, biomarkers, biomarkerMerges };
return { labResults, biomarkers, biomarkerMerges, labResultImportKeys };
}
/**
* What the person deleted among their Health Records readings, as the import
* ledger would hold it once every deletion is purged: the ledger's own rows,
* then, in a portable file, the identities of the deleted readings still
* waiting as tombstones. A portable file does not carry those readings, as it
* carries no tombstone; without their identities a restore onto a fresh
* instance would let the next import write each one again. A disaster-recovery
* file carries the tombstones themselves. An identity is carried once.
*/
async function buildDeletedLabImportKeys(
prisma: Pick<PrismaClient, "labResult" | "labResultImportKey">,
userId: string,
disasterRecovery: boolean,
): Promise<LabResultImportKeyBackupEntry[]> {
const [ledger, tombstones] = await Promise.all([
prisma.labResultImportKey.findMany({
where: { userId },
orderBy: [{ createdAt: "asc" }, { id: "asc" }],
select: { source: true, externalId: true, createdAt: true },
}),
disasterRecovery
? Promise.resolve([])
: prisma.labResult.findMany({
where: {
userId,
deletedAt: { not: null },
externalId: { not: null },
},
orderBy: [{ deletedAt: "asc" }, { id: "asc" }],
select: { source: true, externalId: true, deletedAt: true },
}),
]);
const out: LabResultImportKeyBackupEntry[] = [];
const seen = new Set<string>();
const add = (source: string, externalId: string, at: Date) => {
const key = JSON.stringify([source, externalId]);
if (seen.has(key)) return;
seen.add(key);
out.push({ source, externalId, createdAt: at.toISOString() });
};
for (const row of ledger) add(row.source, row.externalId, row.createdAt);
for (const row of tombstones) {
if (row.externalId && row.deletedAt) {
add(row.source, row.externalId, row.deletedAt);
}
}
return out;
}
// ── Restore ──────────────────────────────────────────────────────────────────
@@ -298,6 +371,11 @@ export async function clearLabResults(ctx: RestoreContext): Promise<void> {
where: { userId: ctx.ownerId },
});
ctx.cleared.labResults = labResults.count;
// The ledger is what the file says was deleted: replaced, like the rest.
const keys = await ctx.tx.labResultImportKey.deleteMany({
where: { userId: ctx.ownerId },
});
ctx.cleared.labResultImportKeys = keys.count;
}
export async function clearBiomarkers(ctx: RestoreContext): Promise<void> {
@@ -487,4 +565,32 @@ export async function restoreLabResults(ctx: RestoreContext): Promise<void> {
const labSourceId = lab.id ?? lab.ref;
if (labSourceId) labResultIdBySourceId.set(labSourceId, createdLab.id);
}
await restoreLabImportKeys(ctx);
}
/**
* The import ledger, after the results: an identity a restored reading holds
* (live, or a tombstone in a disaster-recovery file) is that reading's, and
* the ledger only remembers identities whose row is gone.
*/
async function restoreLabImportKeys(ctx: RestoreContext): Promise<void> {
const { tx, ownerId, payload } = ctx;
const held = new Set(
payload.labResults.flatMap((lab) =>
lab.externalId ? [JSON.stringify([lab.source, lab.externalId])] : [],
),
);
const keys = (payload.labResultImportKeys ?? []).filter(
(key) => !held.has(JSON.stringify([key.source, key.externalId])),
);
if (keys.length === 0) return;
await tx.labResultImportKey.createMany({
data: keys.map((key) => ({
userId: ownerId,
source: key.source,
externalId: key.externalId,
...(key.createdAt ? { createdAt: new Date(key.createdAt) } : {}),
})),
skipDuplicates: true,
});
}
+5
View File
@@ -39,6 +39,7 @@ import {
type BiomarkerBackupEntry,
type BiomarkerMergeBackupEntry,
type LabResultBackupEntry,
type LabResultImportKeyBackupEntry,
} from "@/lib/export/labs-backup";
import {
buildWorkoutsBackupSlice,
@@ -71,6 +72,7 @@ export interface RecordsBackupSection {
labResults: LabResultBackupEntry[];
biomarkers: BiomarkerBackupEntry[];
biomarkerMerges: BiomarkerMergeBackupEntry[];
labResultImportKeys: LabResultImportKeyBackupEntry[];
illnessEpisodes: CanonicalIllnessEpisode[];
allergies: CanonicalAllergy[];
familyHistory: FamilyHistoryEntryDTO[];
@@ -87,6 +89,7 @@ export interface RecordsBackupSection {
export interface RecordsBackupCounts {
labResults: number;
labResultImportKeys: number;
biomarkers: number;
illnessEpisodes: number;
illnessDayLogs: number;
@@ -107,6 +110,7 @@ export async function buildRecordsBackupSection(
| "biomarker"
| "biomarkerAlias"
| "biomarkerMerge"
| "labResultImportKey"
| "illnessEpisode"
| "allergy"
| "familyHistoryEntry"
@@ -157,6 +161,7 @@ export function countRecordsBackupSection(
): RecordsBackupCounts {
return {
labResults: section.labResults.length,
labResultImportKeys: section.labResultImportKeys.length,
biomarkers: section.biomarkers.length,
illnessEpisodes: section.illnessEpisodes.length,
illnessDayLogs: section.illnessEpisodes.reduce(
+1
View File
@@ -63,6 +63,7 @@ export const RESTORE_CLEARED_KEYS = [
"cycleDayLogs",
"cycleProfile",
"labResults",
"labResultImportKeys",
"biomarkers",
"nutrientDays",
"illnessEpisodes",
+27 -5
View File
@@ -34,10 +34,11 @@
* so a new nulling key into a purged model cannot ship without one.
*/
// account-deletion(file): a retention sweep over rows already deleted, with its own retention clock; it does no work for an account.
import type { PrismaClient } from "@/generated/prisma/client";
import type { Prisma, PrismaClient } from "@/generated/prisma/client";
import { TOMBSTONE_RETENTION_DAYS } from "@/lib/auth/native-client";
import { getCoachMessageRetentionDays } from "@/lib/jobs/coach-message-cleanup";
import { DOCUMENT_PURGE_GRACE_DAYS } from "@/lib/jobs/document-purge-queue";
import { retireLabImportKeys } from "@/lib/labs/fhir-import/import-keys";
import {
purgeTombstonesByAccount,
type TombstonePurgeOutcome,
@@ -72,6 +73,15 @@ export type SoftDeleteRetention =
keep?: string;
/** The live references that hold a tombstone back. */
referencedBy?: readonly LiveReferrer[];
/**
* Runs in the purge's transaction before the delete, over the rows the
* delete is about to take: for what a tombstone remembers that must
* outlive the row.
*/
beforePurge?: (
tx: Prisma.TransactionClient,
rows: { ids: string[]; userId: string; cutoff: Date },
) => Promise<unknown>;
/** Why the horizon or the condition differs from the default. */
why?: string;
};
@@ -129,7 +139,17 @@ export const SOFT_DELETE_RETENTION: Readonly<
live("encounters", "reminder_id"),
live("vaccination_records", "reminder_id"),
]),
LabResult: standard("lab_results"),
LabResult: {
kind: "purge",
table: "lab_results",
days: () => TOMBSTONE_RETENTION_DAYS,
beforePurge: (tx, { ids, userId, cutoff }) =>
retireLabImportKeys(tx, userId, {
id: { in: ids },
deletedAt: { not: null, lt: cutoff },
}),
why: "A deleted Health Records reading's identity is what keeps the next import from writing it again; the purge moves it into the import ledger (`LabResultImportKey`) before the row goes.",
},
CustomMetric: standard("custom_metrics"),
CustomMetricEntry: standard("custom_metric_entries"),
CoachPlan: standard("coach_plans"),
@@ -217,15 +237,17 @@ export async function purgeSoftDeletedTable(
),
// The row is re-checked inside the delete: a restore between the read and
// the delete un-tombstones it, and then it stays.
deleteIds: (tx, ids, userId) =>
tx.$executeRawUnsafe(
deleteIds: async (tx, ids, userId) => {
await entry.beforePurge?.(tx, { ids, userId, cutoff });
return tx.$executeRawUnsafe(
`DELETE FROM ${table} t
WHERE t."id" = ANY($1::text[]) AND t."user_id" = $2
AND t."deleted_at" IS NOT NULL AND t."deleted_at" < $3 ${keep}`,
ids,
userId,
cutoff,
),
);
},
});
return { ...outcome, model, table: entry.table };
}
+63
View File
@@ -0,0 +1,63 @@
/**
* The import ledger of Health Records lab readings the person deleted
* (`LabResultImportKey`, #1400).
*
* A deleted reading first waits as a tombstone, and its own `externalId` is
* the memory that the person deleted it: the import finds the row and answers
* "duplicate". Two things remove the row for good, the nightly tombstone purge
* and deleting the reading's whole marker, and both used to take that memory
* with them, so the next Apple Health export (which carries the whole history
* every time) wrote the reading again. Each now calls
* {@link retireLabImportKeys} in the same transaction as the delete, which
* moves the identities of the rows about to go into the ledger.
*
* The ledger holds the digest only, never lab content. It is read by
* {@link heldLabImportKeys} (the import's duplicate check) and carried by the
* backup (`src/lib/export/labs-backup.ts`), together with the identities of
* the deleted readings a portable file leaves out.
*/
import type { Prisma, PrismaClient } from "@/generated/prisma/client";
type LedgerClient = Pick<PrismaClient, "labResult" | "labResultImportKey">;
/**
* Move the identities of the lab rows `where` selects into the ledger, before
* the caller deletes those rows. Rows without an identity are not imported
* from anywhere and have nothing to remember. Run it inside the delete's
* transaction so the identity and the row never both go missing.
*/
export async function retireLabImportKeys(
tx: LedgerClient,
userId: string,
where: Prisma.LabResultWhereInput,
): Promise<number> {
const rows = await tx.labResult.findMany({
where: { ...where, userId, externalId: { not: null } },
select: { source: true, externalId: true },
});
if (rows.length === 0) return 0;
const { count } = await tx.labResultImportKey.createMany({
data: rows.map((row) => ({
userId,
source: row.source,
externalId: row.externalId!,
})),
skipDuplicates: true,
});
return count;
}
/** The identities among `externalIds` the ledger holds as deleted. */
export async function heldLabImportKeys(
client: Pick<PrismaClient, "labResultImportKey">,
userId: string,
source: string,
externalIds: readonly string[],
): Promise<Set<string>> {
if (externalIds.length === 0) return new Set();
const rows = await client.labResultImportKey.findMany({
where: { userId, source, externalId: { in: [...externalIds] } },
select: { externalId: true },
});
return new Set(rows.map((row) => row.externalId));
}
+17 -5
View File
@@ -11,8 +11,10 @@
*
* Two duplicate checks, in this order:
* 1. the identity (`externalId`) under `source = APPLE_HEALTH`, against
* every row including the deleted ones: a result the person deleted
* stays deleted however often the export or the phone sends it again;
* every row including the deleted ones and against the import ledger
* that keeps a deleted reading's identity once its row is gone
* (`./import-keys.ts`): a result the person deleted stays deleted
* however often the export or the phone sends it again;
* 2. the reading itself (marker or name, day, value) against the live
* record, so a result already typed, scanned or imported from a CSV is
* not written a second time.
@@ -43,6 +45,7 @@ import {
import { readingKeys } from "@/lib/labs/reading-keys";
import { encryptNoteToBytes } from "@/lib/labs/store";
import { heldLabImportKeys } from "./import-keys";
import {
APPLE_HEALTH_LAB_SOURCE,
type FhirLabWriteStatus,
@@ -53,7 +56,7 @@ const CHUNK = 200;
type LabWriteClient = Pick<
PrismaClient,
"labResult" | "biomarker" | "biomarkerAlias"
"labResult" | "biomarker" | "biomarkerAlias" | "labResultImportKey"
>;
export interface FhirLabWriteResult {
@@ -118,18 +121,27 @@ export async function writeFhirLabReadings(args: {
seenIds.add(reading.externalId);
fresh.push({ at, reading });
}
const held = new Set(
const freshIds = fresh.map((f) => f.reading.externalId);
const held = new Set<string | null>(
(
await client.labResult.findMany({
where: {
userId,
source: APPLE_HEALTH_LAB_SOURCE,
externalId: { in: fresh.map((f) => f.reading.externalId) },
externalId: { in: freshIds },
},
select: { externalId: true },
})
).map((row) => row.externalId),
);
for (const id of await heldLabImportKeys(
client,
userId,
APPLE_HEALTH_LAB_SOURCE,
freshIds,
)) {
held.add(id);
}
// 2. The marker, and whether the unit fits it.
const planned: Pending[] = [];
+3
View File
@@ -40,6 +40,8 @@ export interface BackupSummary {
cycleDayLogs: number;
/** Lab results in the backup. */
labResults: number;
/** Identities of Health Records lab readings the person deleted. */
labResultImportKeys: number;
nutrientDays: number;
/** User-scoped biomarker catalog entries in the backup. */
biomarkers: number;
@@ -146,6 +148,7 @@ export function summarizeBackup(payload: BackupPayload): BackupSummary {
cycleDayLogs: payload.cycleDayLogs.length,
nutrientDays: payload.nutrientDays.length,
labResults: payload.labResults.length,
labResultImportKeys: payload.labResultImportKeys.length,
biomarkers: payload.biomarkers.length,
illnessEpisodes: payload.illnessEpisodes.length,
illnessDayLogs: payload.illnessEpisodes.reduce(
+4
View File
@@ -67,6 +67,7 @@ import { intradayProfileBackupSchema } from "@/lib/export/intraday-profile-backu
import {
biomarkerBackupSchema,
biomarkerMergeBackupSchema,
labResultImportKeyBackupSchema,
labResultBackupSchema,
} from "@/lib/export/labs-backup-schema";
import { lifeEventBackupSchema } from "@/lib/export/life-events-backup-schema";
@@ -167,6 +168,9 @@ export const backupPayloadSchema = z
biomarkers: z.array(biomarkerBackupSchema).default([]),
// Defaulted: a file written before v1.43 carries no key.
biomarkerMerges: z.array(biomarkerMergeBackupSchema).default([]),
// The identities of Health Records readings the person deleted.
// Defaulted: a file written before the lab import ledger carries no key.
labResultImportKeys: z.array(labResultImportKeyBackupSchema).default([]),
illnessEpisodes: z.array(illnessEpisodeBackupSchema).default([]),
// Defaulted: a file written before v1.40 carries no key.
symptomDefinitions: z.array(symptomDefinitionBackupSchema).default([]),
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -18,6 +18,7 @@
"cycleDayLogs": 2,
"nutrientDays": 1,
"labResults": 2,
"labResultImportKeys": 1,
"biomarkers": 1,
"illnessEpisodes": 4,
"illnessDayLogs": 2,
@@ -75,6 +76,7 @@
"cycleDayLogs": 2,
"cycleProfile": 1,
"labResults": 2,
"labResultImportKeys": 1,
"biomarkers": 1,
"nutrientDays": 1,
"illnessEpisodes": 4,
@@ -292,6 +294,7 @@
"cycleDayLogs": 2,
"cycleProfile": 1,
"labResults": 2,
"labResultImportKeys": 1,
"biomarkers": 1,
"nutrientDays": 1,
"illnessEpisodes": 4,
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -18,6 +18,7 @@
"cycleDayLogs": 0,
"nutrientDays": 0,
"labResults": 0,
"labResultImportKeys": 0,
"biomarkers": 0,
"illnessEpisodes": 1,
"illnessDayLogs": 1,
@@ -136,6 +137,7 @@
"cycleDayLogs": 0,
"cycleProfile": 0,
"labResults": 0,
"labResultImportKeys": 0,
"biomarkers": 0,
"nutrientDays": 0,
"illnessEpisodes": 0,
@@ -353,6 +355,7 @@
"cycleDayLogs": 0,
"cycleProfile": 0,
"labResults": 0,
"labResultImportKeys": 0,
"biomarkers": 0,
"nutrientDays": 0,
"illnessEpisodes": 0,
@@ -651,6 +654,7 @@
}
],
"LabResult": [],
"LabResultImportKey": [],
"Biomarker": [],
"Allergy": [],
"FamilyHistoryEntry": [],
@@ -18,6 +18,7 @@
"cycleDayLogs": 0,
"nutrientDays": 1,
"labResults": 1,
"labResultImportKeys": 0,
"biomarkers": 1,
"illnessEpisodes": 2,
"illnessDayLogs": 1,
@@ -75,6 +76,7 @@
"cycleDayLogs": 0,
"cycleProfile": 0,
"labResults": 0,
"labResultImportKeys": 0,
"biomarkers": 0,
"nutrientDays": 0,
"illnessEpisodes": 0,
@@ -292,6 +294,7 @@
"cycleDayLogs": 0,
"cycleProfile": 0,
"labResults": 0,
"labResultImportKeys": 0,
"biomarkers": 0,
"nutrientDays": 0,
"illnessEpisodes": 0,
@@ -745,6 +748,7 @@
"deletedAt": null
}
],
"LabResultImportKey": [],
"Biomarker": [
{
"id": "<id:14>",
@@ -1660,6 +1664,7 @@
}
],
"biomarkerMerges": [],
"labResultImportKeys": [],
"illnessEpisodes": [
{
"id": "legacy-episode-child",
File diff suppressed because it is too large. Load diff
+11
View File
@@ -551,6 +551,17 @@ export async function seedEveryTwoEndedModel(
takenAt: AT("2026-06-30T09:00:00.000Z"),
},
});
// A Health Records reading the person deleted and the purge then removed:
// only the import ledger remembers it, and a restore that dropped the row
// would let the next Apple Health import write it again.
await prisma.labResultImportKey.create({
data: {
userId: OWNER_ID,
source: "APPLE_HEALTH",
externalId: "fhir:round-trip-deleted-result",
createdAt: AT("2026-07-04T09:00:00.000Z"),
},
});
// What the drug above is FOR, pinned to the analyte just created. The two
// rows are seeded at opposite ends of this fixture on purpose: the drug is
// restored hundreds of lines before the analyte, so this pairing is what
@@ -192,6 +192,8 @@ const COUNT_BACK: Record<
p.moodEntryTagLink.count({ where: { moodEntry: { userId } } }),
MoodTag: (p, userId) => p.moodTag.count({ where: { userId } }),
LabResult: (p, userId) => p.labResult.count({ where: { userId } }),
LabResultImportKey: (p, userId) =>
p.labResultImportKey.count({ where: { userId } }),
Biomarker: (p, userId) => p.biomarker.count({ where: { userId } }),
BiomarkerAlias: (p, userId) => p.biomarkerAlias.count({ where: { userId } }),
BiomarkerMerge: (p, userId) => p.biomarkerMerge.count({ where: { userId } }),
@@ -0,0 +1,254 @@
/**
* A Health Records lab result the person deleted stays deleted.
*
* The Apple Health export carries the whole clinical history every time, and
* the phone resends what it holds, so the import recognises a reading by the
* identity it arrived under. That memory lived only on the reading's own row,
* and three ways of losing the row brought the reading back on the next
* import:
*
* - deleting the whole marker, which removes its readings outright;
* - the nightly purge, which removes a deleted reading for good once its
* tombstone has aged out;
* - a portable backup, which leaves deleted readings out, restored onto an
* instance that has never seen the account.
*
* The identity of a deleted reading is now kept in the import ledger
* (`LabResultImportKey`) the moment it is deleted, and taken out again when
* the deletion is undone. Against Postgres, through the real routes.
*/
import { NextRequest } from "next/server";
import { beforeEach, describe, expect, it, vi } from "vitest";
process.env.ENCRYPTION_KEY ??=
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
import { encrypt } from "@/lib/crypto";
import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
import { purgeSoftDeletedTable } from "@/lib/jobs/soft-delete-retention";
import { writeFhirLabReadings } from "@/lib/labs/fhir-import/write";
import type { MappedFhirLab } from "@/lib/labs/fhir-import/types";
import { parseBackupPayload } from "@/lib/validations/backup";
import { POST as RESTORE } from "./restore-job-driver";
import { cookieJar, headerJar } from "./mock-next-headers";
import { getPrismaClient, truncateAllTables } from "./setup";
vi.mock("next/headers", async () => {
const { cookieJar, headerJar } = await import("./mock-next-headers");
return {
headers: vi.fn(async () => ({
get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
})),
cookies: vi.fn(async () => ({
get: (name: string) => {
const value = cookieJar.get(name);
return value ? { name, value } : undefined;
},
set: (name: string, value: string) => cookieJar.set(name, value),
delete: (name: string) => cookieJar.delete(name),
})),
};
});
vi.mock("@/lib/cache/invalidate", async (importOriginal) => {
const actual =
await importOriginal<typeof import("@/lib/cache/invalidate")>();
return { ...actual, invalidateUserData: vi.fn() };
});
const prisma = getPrismaClient();
let USER = "";
beforeEach(async () => {
await truncateAllTables(prisma);
cookieJar.clear();
headerJar.clear();
const user = await prisma.user.create({
data: {
username: "deleted-stays-deleted",
email: "deleted-stays-deleted@example.test",
role: "ADMIN",
timezone: "UTC",
modulePreferencesJson: { labs: true },
},
});
USER = user.id;
await signIn();
});
async function signIn() {
const session = await prisma.session.create({
data: { userId: USER, expiresAt: new Date(Date.now() + 600_000) },
});
cookieJar.set("healthlog_session", session.id);
}
function reading(n: number, over: Partial<MappedFhirLab> = {}): MappedFhirLab {
return {
externalId: `fhir:clinic:obs-${n}`,
analyte: "Ferritin",
loinc: null,
catalogKey: null,
value: 80 + n,
valueText: null,
unit: "ng/mL",
takenAt: new Date(Date.UTC(2026, 5, n, 7)),
sourceReferenceLow: 30,
sourceReferenceHigh: 400,
sourceReferenceText: null,
panel: null,
note: null,
...over,
};
}
const BATCH = [reading(1), reading(2), reading(3, { analyte: "Glucose" })];
async function importBatch() {
return writeFhirLabReadings({
userId: USER,
tz: "UTC",
readings: BATCH,
prisma,
});
}
async function liveReadings() {
return prisma.labResult.findMany({
where: { userId: USER, deletedAt: null },
select: { analyte: true, externalId: true },
orderBy: { externalId: "asc" },
});
}
async function deleteMarker(name: string) {
const marker = await prisma.biomarker.findFirstOrThrow({
where: { userId: USER, name },
});
const { DELETE } = await import("@/app/api/biomarkers/[id]/route");
const res = await DELETE(
new NextRequest(`http://localhost/api/biomarkers/${marker.id}`, {
method: "DELETE",
}),
{ params: Promise.resolve({ id: marker.id }) },
);
expect(res.status).toBe(200);
}
async function deleteReading(externalId: string) {
const row = await prisma.labResult.findFirstOrThrow({
where: { userId: USER, externalId },
});
const { DELETE } = await import("@/app/api/labs/[id]/route");
const res = await DELETE(
new NextRequest(`http://localhost/api/labs/${row.id}`, {
method: "DELETE",
}),
{ params: Promise.resolve({ id: row.id }) },
);
expect(res.status).toBe(200);
return row.id;
}
describe("a deleted Health Records reading", () => {
it("does not come back after its whole marker is deleted", async () => {
expect((await importBatch()).inserted).toBe(3);
await deleteMarker("Ferritin");
expect(await liveReadings()).toEqual([
{ analyte: "Glucose", externalId: "fhir:clinic:obs-3" },
]);
const again = await importBatch();
expect(again.inserted).toBe(0);
expect(again.duplicate).toBe(3);
expect(await liveReadings()).toHaveLength(1);
expect(
await prisma.biomarker.count({
where: { userId: USER, name: "Ferritin" },
}),
).toBe(0);
});
it("does not come back once the purge has removed its tombstone", async () => {
await importBatch();
await deleteReading("fhir:clinic:obs-1");
// Long past the retention horizon.
const later = new Date(Date.now() + 400 * 86_400_000);
const purged = await purgeSoftDeletedTable(prisma, "LabResult", later);
expect(purged.deleted).toBe(1);
expect(
await prisma.labResult.count({
where: { userId: USER, externalId: "fhir:clinic:obs-1" },
}),
).toBe(0);
const again = await importBatch();
expect(again.inserted).toBe(0);
expect(await liveReadings()).toHaveLength(2);
});
it("comes back to life only through its own undo", async () => {
await importBatch();
const id = await deleteReading("fhir:clinic:obs-1");
const { POST } = await import("@/app/api/labs/restore/route");
const res = await POST(
new NextRequest("http://localhost/api/labs/restore", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ ids: [id] }),
}),
);
expect(res.status).toBe(200);
expect(await liveReadings()).toHaveLength(3);
// Deleted again later, it stays deleted again, purge or not.
await deleteReading("fhir:clinic:obs-1");
await purgeSoftDeletedTable(
prisma,
"LabResult",
new Date(Date.now() + 400 * 86_400_000),
);
expect((await importBatch()).inserted).toBe(0);
expect(await liveReadings()).toHaveLength(2);
});
it("stays deleted through a portable backup restored onto a fresh instance", async () => {
await importBatch();
await deleteReading("fhir:clinic:obs-1");
await deleteMarker("Glucose");
const { payload: built } = await buildFullBackupPayload(prisma, USER, {
purpose: "portable-export",
});
const parsed = parseBackupPayload(JSON.stringify(built));
// The fresh instance: the account as the file brings it, nothing else.
await prisma.labResult.deleteMany({ where: { userId: USER } });
await prisma.biomarker.deleteMany({ where: { userId: USER } });
await prisma.labResultImportKey.deleteMany({ where: { userId: USER } });
const backup = await prisma.dataBackup.create({
data: {
userId: USER,
type: "LAB_IMPORT_LEDGER_ROUND_TRIP",
data: encrypt(JSON.stringify(parsed)),
},
});
await signIn();
const response = await RESTORE(
new Request(`http://localhost/api/admin/backups/${backup.id}/restore`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ confirm: "RESTORE" }),
}) as unknown as Parameters<typeof RESTORE>[0],
{ params: Promise.resolve({ id: backup.id }) },
);
expect(response.status).toBe(200);
expect(await liveReadings()).toEqual([
{ analyte: "Ferritin", externalId: "fhir:clinic:obs-2" },
]);
const again = await importBatch();
expect(again.inserted).toBe(0);
expect(await liveReadings()).toHaveLength(1);
});
});