fix(retention): keep a tombstone while a live record still points at it

The nightly purge nulled every non-cascading key into a purged row, so a
live visit lost the practitioner it was with, a live dose its visit, a
symptom its condition. Each purge entry now lists the keys that do not
cascade, and a tombstone stays while a live row holds one. The register
test derives those keys from the schema's relations, so a new nulling key
into a purged model fails until it is listed.
This commit is contained in:
Marc-André Bombeck committed 2026-10-20 16:01:13 +02:00
1 parent 4a0db28713
commit ec2c067e60
4 files changed
+365 -24

No files matched your search

+44 -4
View File
@@ -41,6 +41,20 @@ export interface DeclaredField {
physicalName: string;
}
/** The owning side of a relation: the field that carries `@relation(fields:)`. */
export interface DeclaredRelation {
/** The relation field's name. */
name: string;
/** The model it points at. */
target: string;
/** Whether the relation field is optional (`Type?`). */
optional: boolean;
/** The scalar fields holding the key, by their datamodel names. */
fields: string[];
/** The declared `onDelete`, or null when Prisma's default applies. */
onDelete: string | null;
}
export interface DeclaredModel {
name: string;
mappedName: string | null;
@@ -48,6 +62,8 @@ export interface DeclaredModel {
physicalName: string;
/** Scalar + enum fields only — relation fields carry no column. */
fields: DeclaredField[];
/** Relations this model owns the key of. */
relations: DeclaredRelation[];
}
export interface DeclaredSchema {
@@ -58,7 +74,9 @@ export interface DeclaredSchema {
const BLOCK_MAP = /^\s*@@map\("([^"]+)"\)/m;
const FIELD_MAP = /@map\("([^"]+)"\)/;
/** `name Type[]? @attr…` — the two leading tokens are what matters. */
const FIELD_LINE = /^\s*(\w+)\s+(\w+)(?:\[\])?\??\s*(.*)$/;
const FIELD_LINE = /^\s*(\w+)\s+(\w+)(?:\[\])?(\??)\s*(.*)$/;
const RELATION_FIELDS = /@relation\([^)]*\bfields:\s*\[([^\]]*)\]/;
const ON_DELETE = /@relation\([^)]*\bonDelete:\s*(\w+)/;
/** Drop `//` and `///` lines so prose can never read as a declaration. */
function stripComments(block: string): string {
@@ -112,15 +130,31 @@ export function readDeclaredSchema(
const models = rawModels.map(({ name, body }) => {
const mappedName = BLOCK_MAP.exec(body)?.[1] ?? null;
const fields: DeclaredField[] = [];
const relations: DeclaredRelation[] = [];
for (const line of body.split("\n")) {
if (/^\s*@@/.test(line)) continue;
const match = FIELD_LINE.exec(line);
if (!match) continue;
const [, fieldName, baseType, rest] = match;
const [, fieldName, baseType, optional, rest] = match;
// A field whose type is another model is a relation, not a column. The
// back-reference side carries no `@relation` attribute at all, so the
// type is what decides it — keying on `@relation` would miss half.
if (modelNames.has(baseType)) continue;
if (modelNames.has(baseType)) {
const keys = RELATION_FIELDS.exec(rest)?.[1];
if (keys !== undefined) {
relations.push({
name: fieldName,
target: baseType,
optional: optional === "?",
fields: keys
.split(",")
.map((key) => key.trim())
.filter(Boolean),
onDelete: ON_DELETE.exec(rest)?.[1] ?? null,
});
}
continue;
}
const fieldMapped = FIELD_MAP.exec(rest)?.[1] ?? null;
fields.push({
name: fieldName,
@@ -128,7 +162,13 @@ export function readDeclaredSchema(
physicalName: fieldMapped ?? fieldName,
});
}
return { name, mappedName, physicalName: mappedName ?? name, fields };
return {
name,
mappedName,
physicalName: mappedName ?? name,
fields,
relations,
};
});
return { enums, models };
@@ -6,6 +6,13 @@
* tombstones stayed for as long as the instance ran. The set is derived from
* `prisma/schema.prisma` (a `deletedAt` field), so a model that starts
* soft-deleting cannot ship without saying how long its tombstones live.
*
* The second half holds the live references. A key into a purged model that
* does not cascade is a live record pointing at the tombstone (a visit naming
* a deleted practitioner); the purge would null it and the record would stop
* showing what it showed. Each such key must be one of the entry's
* `referencedBy`, which keeps the tombstone while a live row holds it, and the
* set is derived from the schema's `@relation` lines in both directions.
*/
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
@@ -19,10 +26,67 @@ import {
import { TOMBSTONE_RETENTION_DAYS } from "@/lib/auth/native-client";
import {
SOFT_DELETE_RETENTION,
purgeCondition,
purgeModels,
type LiveReferrer,
type SoftDeleteRetention,
} from "@/lib/jobs/soft-delete-retention";
/**
* Every key into a purged model that does not cascade, as the register spells
* a referrer. A cascading key belongs to the row it points at (a visit's link
* rows) and goes with it; any other `onDelete` (SetNull, Prisma's default for
* an optional relation, Restrict, NoAction) leaves a live row behind.
*/
function nullingKeys(
schema: DeclaredSchema,
register: Readonly<Record<string, SoftDeleteRetention>>,
): Map<string, LiveReferrer[]> {
const byTarget = new Map<string, LiveReferrer[]>();
for (const model of schema.models) {
const softDeletes = model.fields.some((f) => f.name === "deletedAt");
for (const relation of model.relations) {
if (register[relation.target]?.kind !== "purge") continue;
if (relation.onDelete === "Cascade") continue;
for (const key of relation.fields) {
const column =
model.fields.find((f) => f.name === key)?.physicalName ?? key;
const list = byTarget.get(relation.target) ?? [];
list.push({ table: model.physicalName, column, softDeletes });
byTarget.set(relation.target, list);
}
}
}
return byTarget;
}
const refKey = (ref: LiveReferrer) =>
`${ref.table}.${ref.column}${ref.softDeletes ? "" : " (no tombstone)"}`;
function referrerProblems(
schema: DeclaredSchema,
register: Readonly<Record<string, SoftDeleteRetention>>,
): string[] {
const found: string[] = [];
const derived = nullingKeys(schema, register);
for (const [model, entry] of Object.entries(register)) {
if (entry.kind !== "purge") continue;
const want = new Set((derived.get(model) ?? []).map(refKey));
const have = new Set((entry.referencedBy ?? []).map(refKey));
for (const key of want) {
if (!have.has(key)) {
found.push(`${model}: ${key} does not cascade and is not a referrer`);
}
}
for (const key of have) {
if (!want.has(key)) {
found.push(`${model}: referrer ${key} is not a nulling key`);
}
}
}
return found;
}
function problems(
schema: DeclaredSchema,
register: Readonly<Record<string, SoftDeleteRetention>>,
@@ -91,6 +155,94 @@ describe("soft-delete retention register", () => {
}
});
it("holds every tombstone a live row still points at", () => {
const derived = nullingKeys(schema, SOFT_DELETE_RETENTION);
// A reader that stopped seeing relations would derive nothing and pass.
const count = [...derived.values()].reduce((n, l) => n + l.length, 0);
expect(count).toBeGreaterThanOrEqual(9);
expect(derived.get("Practitioner")?.map(refKey)).toEqual([
"encounters.practitioner_id",
"vaccination_records.practitioner_id",
]);
const found = referrerProblems(schema, SOFT_DELETE_RETENTION);
expect(found, found.join("\n")).toEqual([]);
});
it("turns every referrer into a live-row condition", () => {
const entry = SOFT_DELETE_RETENTION.Practitioner;
if (entry.kind !== "purge") throw new Error("Practitioner is purged");
const sql = purgeCondition(entry);
expect(sql).toContain(
`NOT EXISTS (SELECT 1 FROM "encounters" r WHERE r."practitioner_id" = t."id" AND r."deleted_at" IS NULL)`,
);
const symptoms = SOFT_DELETE_RETENTION.IllnessEpisode;
if (symptoms.kind !== "purge") throw new Error("IllnessEpisode is purged");
// A table without tombstones: every row is live.
expect(purgeCondition(symptoms)).toContain(
`NOT EXISTS (SELECT 1 FROM "symptom_events" r WHERE r."episode_id" = t."id")`,
);
expect(() =>
purgeCondition({
kind: "purge",
table: "widgets",
days: () => 75,
referencedBy: [{ table: "x; DROP", column: "a", softDeletes: true }],
}),
).toThrow("Bad referrer");
});
it("counter-check: names a new nulling key and a stale referrer", () => {
const dir = mkdtempSync(join(tmpdir(), "hl-retention-"));
try {
const path = join(dir, "schema.prisma");
writeFileSync(
path,
`model Widget {
id String @id
userId String @map("user_id")
deletedAt DateTime? @map("deleted_at")
@@map("widgets")
}
model Note {
id String @id
widgetId String? @map("widget_id")
ownerId String? @map("owner_id")
deletedAt DateTime? @map("deleted_at")
widget Widget? @relation(fields: [widgetId], references: [id], onDelete: SetNull)
owner Widget? @relation("Owned", fields: [ownerId], references: [id])
@@map("notes")
}
model Part {
id String @id
widgetId String @map("widget_id")
widget Widget @relation(fields: [widgetId], references: [id], onDelete: Cascade)
@@map("parts")
}
`,
);
const synthetic = readDeclaredSchema(path);
const found = referrerProblems(synthetic, {
Widget: {
kind: "purge",
table: "widgets",
days: () => 75,
referencedBy: [
{ table: "notes", column: "widget_id", softDeletes: true },
{ table: "gone", column: "widget_id", softDeletes: false },
],
},
});
// The SetNull key is handled, the default-`onDelete` key is not, the
// cascading key is owned, and the stale referrer is named.
expect(found).toEqual([
"Widget: notes.owner_id does not cascade and is not a referrer",
"Widget: referrer gone.widget_id (no tombstone) is not a nulling key",
]);
} finally {
rmSync(dir, { recursive: true });
}
});
it("counter-check: names a new soft-delete model, a stale entry and a wrong table", () => {
const dir = mkdtempSync(join(tmpdir(), "hl-retention-"));
try {
+81 -10
View File
@@ -24,9 +24,14 @@
* purged measurement.
*
* The delete runs per account under the restore lock, in bounded batches
* (`purgeTombstonesByAccount`). Foreign keys into these tables cascade or set
* null (none restricts), so a purged visit takes its link rows with it and a
* live vaccination that named it simply loses the pointer.
* (`purgeTombstonesByAccount`). A key that cascades belongs to the row it
* points at: a purged visit takes its link rows with it. A key that does not
* cascade is a live record's reference, and a live record keeps what it shows:
* a visit still names the practitioner it was with after the address-book
* entry is deleted, so that entry is not purged while a live visit or dose
* names it. Those references are each entry's `referencedBy`, and
* `soft-delete-retention-register.test.ts` derives the set from the schema,
* so a new nulling key into a purged model cannot ship without one.
*/
import type { PrismaClient } from "@/generated/prisma/client";
import { TOMBSTONE_RETENTION_DAYS } from "@/lib/auth/native-client";
@@ -37,6 +42,19 @@ import {
type TombstonePurgeOutcome,
} from "@/lib/jobs/measurement-tombstone-cleanup";
/**
* A key into a purged model that does not cascade. While a row that is not
* itself tombstoned holds it, the row it names stays.
*/
export interface LiveReferrer {
/** The physical table holding the key; a literal, spliced into the SQL. */
table: string;
/** The key column; a literal, spliced into the SQL. */
column: string;
/** Whether the referring table soft-deletes (a tombstoned row is not live). */
softDeletes: boolean;
}
export type SoftDeleteRetention =
| { kind: "sync-feed"; days: number }
| { kind: "document-grace"; days: number }
@@ -51,14 +69,31 @@ export type SoftDeleteRetention =
* the alias `t`. For a tombstone something live still depends on.
*/
keep?: string;
/** The live references that hold a tombstone back. */
referencedBy?: readonly LiveReferrer[];
/** Why the horizon or the condition differs from the default. */
why?: string;
};
const standard = (table: string): SoftDeleteRetention => ({
const standard = (
table: string,
referencedBy?: readonly LiveReferrer[],
): SoftDeleteRetention => ({
kind: "purge",
table,
days: () => TOMBSTONE_RETENTION_DAYS,
...(referencedBy ? { referencedBy } : {}),
});
const live = (table: string, column: string): LiveReferrer => ({
table,
column,
softDeletes: true,
});
const always = (table: string, column: string): LiveReferrer => ({
table,
column,
softDeletes: false,
});
export const SOFT_DELETE_RETENTION: Readonly<
@@ -75,6 +110,7 @@ export const SOFT_DELETE_RETENTION: Readonly<
kind: "purge",
table: "menstrual_cycles",
days: () => TOMBSTONE_RETENTION_DAYS,
referencedBy: [live("cycle_day_logs", "cycle_id")],
keep: `NOT EXISTS (SELECT 1 FROM "menstrual_cycles" a
WHERE a."id" = t."absorbed_into_id"
AND a."deleted_at" IS NULL)`,
@@ -88,26 +124,61 @@ export const SOFT_DELETE_RETENTION: Readonly<
},
MentalHealthAssessment: standard("mental_health_assessments"),
MeasurementReminder: standard("measurement_reminders"),
MeasurementReminder: standard("measurement_reminders", [
live("encounters", "reminder_id"),
live("vaccination_records", "reminder_id"),
]),
LabResult: standard("lab_results"),
CustomMetric: standard("custom_metrics"),
CustomMetricEntry: standard("custom_metric_entries"),
CoachPlan: standard("coach_plans"),
CoachReminder: standard("coach_reminders"),
IllnessEpisode: standard("illness_episodes"),
IllnessEpisode: standard("illness_episodes", [
live("illness_episodes", "parent_condition_id"),
always("symptom_events", "episode_id"),
]),
IllnessDayLog: standard("illness_day_logs"),
Allergy: standard("allergies"),
FamilyHistoryEntry: standard("family_history_entries"),
Encounter: standard("encounters"),
Practitioner: standard("practitioners"),
Encounter: standard("encounters", [
live("vaccination_records", "encounter_id"),
]),
Practitioner: standard("practitioners", [
live("encounters", "practitioner_id"),
live("vaccination_records", "practitioner_id"),
]),
VaccinationRecord: standard("vaccination_records"),
CustomVaccine: standard("custom_vaccines"),
CustomVaccine: standard("custom_vaccines", [
live("vaccination_records", "custom_vaccine_id"),
]),
LifeEvent: standard("life_events"),
};
const TABLE = /^[a-z_]+$/;
const DAY_MS = 86_400_000;
/**
* The condition a purgeable row must meet, over the alias `t`: the entry's
* own `keep`, and no live row holding a key to it. Every identifier comes
* from the register's literals and is asserted against a closed pattern.
*/
export function purgeCondition(
entry: Extract<SoftDeleteRetention, { kind: "purge" }>,
): string {
const parts: string[] = [];
if (entry.keep) parts.push(entry.keep);
for (const ref of entry.referencedBy ?? []) {
if (!TABLE.test(ref.table) || !TABLE.test(ref.column)) {
throw new Error(`Bad referrer ${ref.table}.${ref.column}`);
}
const alive = ref.softDeletes ? ` AND r."deleted_at" IS NULL` : "";
parts.push(
`NOT EXISTS (SELECT 1 FROM "${ref.table}" r WHERE r."${ref.column}" = t."id"${alive})`,
);
}
return parts.map((part) => `AND ${part}`).join(" ");
}
/** One table's purge, for the run report. */
export interface SoftDeletePurgeLeg extends TombstonePurgeOutcome {
model: string;
@@ -130,7 +201,7 @@ export async function purgeSoftDeletedTable(
}
const table = `"${entry.table}"`;
const cutoff = new Date(now.getTime() - entry.days() * DAY_MS);
const keep = entry.keep ? `AND ${entry.keep}` : "";
const keep = purgeCondition(entry);
const outcome = await purgeTombstonesByAccount({
prisma,
findRows: (take, skipUserIds) =>
+88 -10
View File
@@ -3,10 +3,11 @@
* retention register hands to `soft-delete-purge`.
*
* Proves, against the migrated schema: every table's purge statement runs; an
* expired tombstone goes while a recent one and a live row stay; a purged
* visit cascades its links and leaves a live vaccination that named it in
* place; a start folded into a live cycle stays; the coach facts keep their
* longer horizon.
* expired tombstone goes while a recent one and a live row stay; a tombstone
* a live record still points at stays (a deleted practitioner a live visit
* names, a deleted visit a live dose names) and goes once nothing live names
* it; a purged visit cascades its links; a start folded into a live cycle
* stays; the coach facts keep their longer horizon.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
@@ -75,34 +76,111 @@ describe("soft-delete purge", () => {
expect(left.map((r) => r.id)).toEqual(["lab-live", "lab-recent"]);
});
it("cascades a purged visit's links and unhooks a live vaccination", async () => {
it("keeps a deleted practitioner a live visit still names", async () => {
const practitioner = (id: string) =>
prisma.practitioner.create({
data: { id, userId: USER, name: id, deletedAt: EXPIRED },
});
await practitioner("doc-live-visit");
await practitioner("doc-gone-visit");
await practitioner("doc-unused");
await prisma.encounter.create({
data: {
id: "visit-expired",
id: "visit-live",
userId: USER,
occurredAt: new Date("2026-01-01T08:00:00.000Z"),
deletedAt: EXPIRED,
practitionerId: "doc-live-visit",
},
});
// A visit that is itself a tombstone holds nothing back.
await prisma.encounter.create({
data: {
id: "visit-tombstoned",
userId: USER,
occurredAt: new Date("2026-01-02T08:00:00.000Z"),
practitionerId: "doc-gone-visit",
deletedAt: RECENT,
},
});
const leg = await purgeSoftDeletedTable(prisma, "Practitioner", NOW);
expect(leg.deleted).toBe(2);
const left = await prisma.practitioner.findMany({ select: { id: true } });
expect(left.map((r) => r.id)).toEqual(["doc-live-visit"]);
expect(
await prisma.encounter.findUniqueOrThrow({
where: { id: "visit-live" },
select: { practitionerId: true },
}),
).toEqual({ practitionerId: "doc-live-visit" });
});
it("purges the practitioner once the visit naming it is gone", async () => {
await prisma.practitioner.create({
data: { id: "doc", userId: USER, name: "doc", deletedAt: EXPIRED },
});
await prisma.encounter.create({
data: {
id: "visit",
userId: USER,
occurredAt: new Date("2026-01-01T08:00:00.000Z"),
practitionerId: "doc",
},
});
await purgeSoftDeletedTable(prisma, "Practitioner", NOW);
expect(await prisma.practitioner.count()).toBe(1);
await prisma.encounter.update({
where: { id: "visit" },
data: { deletedAt: EXPIRED },
});
// The visit goes first in the run's order, and the practitioner with it.
await purgeSoftDeletedRows(prisma, NOW);
expect(await prisma.encounter.count()).toBe(0);
expect(await prisma.practitioner.count()).toBe(0);
});
it("keeps a deleted visit a live dose names and cascades a purged visit's links", async () => {
const visit = (id: string) =>
prisma.encounter.create({
data: {
id,
userId: USER,
occurredAt: new Date("2026-01-01T08:00:00.000Z"),
deletedAt: EXPIRED,
},
});
await visit("visit-named");
await visit("visit-linked");
await prisma.vaccinationRecord.create({
data: {
id: "vacc-live",
userId: USER,
occurredAt: new Date("2026-01-01T08:00:00.000Z"),
encounterId: "visit-expired",
encounterId: "visit-named",
},
});
await prisma.encounterVaccinationLink.create({
data: {
userId: USER,
encounterId: "visit-linked",
vaccinationId: "vacc-live",
},
});
const leg = await purgeSoftDeletedTable(prisma, "Encounter", NOW);
expect(leg.deleted).toBe(1);
expect(await prisma.encounter.count()).toBe(0);
const left = await prisma.encounter.findMany({ select: { id: true } });
expect(left.map((r) => r.id)).toEqual(["visit-named"]);
expect(await prisma.encounterVaccinationLink.count()).toBe(0);
expect(
await prisma.vaccinationRecord.findUniqueOrThrow({
where: { id: "vacc-live" },
select: { encounterId: true, deletedAt: true },
}),
).toEqual({ encounterId: null, deletedAt: null });
).toEqual({ encounterId: "visit-named", deletedAt: null });
});
it("keeps a start folded into a live cycle until that cycle goes", async () => {