From 722945702a55a273c5d1e604a20b7315699aa5d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 27 Sep 2026 13:07:17 +0200 Subject: [PATCH] fix(privacy): clear the last readable copies of practitioner contact details The encryption backfill now also removes the two copies of a practitioner's contact details that outlived the sealing. An appointment reminder's copy of the address is cleared, since the reminder resolves it from the practitioner when it fires. An address-book edit's audit row written before the sealing kept the old phone number and address in its previous values; those values leave the row and the changed fields stay named. Rows of a deleted account, which no per-account job reaches, are scrubbed by the discovery pass itself. A route whose readable geometry is a JSON null rather than SQL NULL had nothing to seal but matched discovery every night. The pass now sets it to SQL NULL so it drops out. --- .../free-text-encryption-backfill.test.ts | 40 ++++- src/lib/jobs/free-text-encryption-backfill.ts | 148 ++++++++++++++++- src/lib/jobs/reminder/register-maintenance.ts | 10 +- .../free-text-encryption-backfill.test.ts | 155 ++++++++++++++++++ 4 files changed, 349 insertions(+), 4 deletions(-) diff --git a/src/lib/jobs/__tests__/free-text-encryption-backfill.test.ts b/src/lib/jobs/__tests__/free-text-encryption-backfill.test.ts index 96a3ed459..c75aa4b8e 100644 --- a/src/lib/jobs/__tests__/free-text-encryption-backfill.test.ts +++ b/src/lib/jobs/__tests__/free-text-encryption-backfill.test.ts @@ -92,13 +92,20 @@ vi.mock("@/lib/db", () => { return { prisma: { ...delegates, + // Nothing to clear or scrub in this in-memory store; the real-Postgres + // test covers both. + measurementReminder: { updateMany: async () => ({ count: 0 }) }, + auditLog: { findMany: async () => [] }, $transaction: async (fn: (tx: typeof delegates) => unknown) => fn(delegates), }, }; }); -import { runFreeTextEncryptionBackfillForUser } from "@/lib/jobs/free-text-encryption-backfill"; +import { + runFreeTextEncryptionBackfillForUser, + scrubContactAuditDetails, +} from "@/lib/jobs/free-text-encryption-backfill"; const KEY = "a".repeat(64); const TRACK = { @@ -202,6 +209,8 @@ describe("runFreeTextEncryptionBackfillForUser", () => { metricNotesMigrated: 2, practitionerContactsMigrated: 2, routeGeometriesMigrated: 1, + appointmentAddressesCleared: 0, + contactAuditRowsScrubbed: 0, }); const c1 = store.conversations.find((r) => r.id === "c1")!; @@ -300,6 +309,8 @@ describe("runFreeTextEncryptionBackfillForUser", () => { metricNotesMigrated: 0, practitionerContactsMigrated: 0, routeGeometriesMigrated: 0, + appointmentAddressesCleared: 0, + contactAuditRowsScrubbed: 0, }); expect(store.conversations.find((r) => r.id === "c1")!.titleEncrypted).toBe( sealed, @@ -316,3 +327,30 @@ describe("runFreeTextEncryptionBackfillForUser", () => { }); }); }); + +describe("scrubContactAuditDetails", () => { + it("removes the phone number and address from previous and keeps them named", () => { + const out = scrubContactAuditDetails( + JSON.stringify({ + practitionerId: "p1", + fields: ["location", "name"], + previous: { location: "Hauptstr. 1", name: "Alt", phone: "+49 30 1" }, + }), + ); + expect(JSON.parse(out!)).toEqual({ + practitionerId: "p1", + fields: ["location", "name", "phone"], + previous: { name: "Alt" }, + }); + }); + + it("leaves a row without contact values, or that is not this JSON, alone", () => { + expect( + scrubContactAuditDetails( + JSON.stringify({ fields: ["location"], previous: {} }), + ), + ).toBeNull(); + expect(scrubContactAuditDetails("not json")).toBeNull(); + expect(scrubContactAuditDetails(null)).toBeNull(); + }); +}); diff --git a/src/lib/jobs/free-text-encryption-backfill.ts b/src/lib/jobs/free-text-encryption-backfill.ts index 60c238fa4..60d769060 100644 --- a/src/lib/jobs/free-text-encryption-backfill.ts +++ b/src/lib/jobs/free-text-encryption-backfill.ts @@ -13,6 +13,15 @@ * track of an outdoor workout, sealed as one binary value). A route row has no * user id of its own; it is found through its workout. * + * The same pass removes the two readable copies of a practitioner's contact + * details that outlived the sealing: the address an appointment reminder used + * to copy onto `MeasurementReminder.location` (the tick now resolves it from + * the practitioner when it fires), and the old phone number and address an + * edit's audit row recorded in `previous` before the contact fields were + * sealed (the row keeps the changed-field names). A route whose readable + * column holds a JSON null rather than SQL NULL has nothing to seal; it is set + * to SQL NULL so discovery stops finding it. + * * Modelled on `med-notes-encryption-backfill.ts`: a discovery query enqueues * one job per user still holding an un-migrated row, the per-user handler walks * that user's rows, and the pass is idempotent across reboots — once a row is @@ -76,6 +85,93 @@ export interface FreeTextEncryptionBackfillSummary { practitionerContactsMigrated: number; /** v1.39.4 — workout GPS tracks sealed. */ routeGeometriesMigrated: number; + /** v1.39.4 — appointment reminders whose readable address copy was cleared. */ + appointmentAddressesCleared: number; + /** v1.39.4 — edit audit rows whose old phone number / address was removed. */ + contactAuditRowsScrubbed: number; +} + +/** The audit action whose older rows carried contact values. */ +const CONTACT_AUDIT_ACTION = "practitioner.contact.update"; + +/** The sealed contact fields an audit row may name but never carry. */ +const SEALED_CONTACT_FIELDS = ["location", "phone"] as const; + +/** + * An edit audit row's `details` without the sealed contact values: the keys + * leave `previous` and stay named in `fields`. Returns null when there is + * nothing to remove (or the value is not the JSON this action writes), so the + * caller leaves the row alone. + */ +export function scrubContactAuditDetails( + details: string | null, +): string | null { + if (!details) return null; + let parsed: unknown; + try { + parsed = JSON.parse(details); + } catch { + return null; + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return null; + } + const record = parsed as Record; + const previous = record.previous; + if (!previous || typeof previous !== "object" || Array.isArray(previous)) { + return null; + } + const carried = SEALED_CONTACT_FIELDS.filter((field) => + Object.prototype.hasOwnProperty.call(previous, field), + ); + if (carried.length === 0) return null; + const kept = { ...(previous as Record) }; + for (const field of carried) delete kept[field]; + const fields = new Set( + Array.isArray(record.fields) + ? record.fields.filter((f): f is string => typeof f === "string") + : [], + ); + for (const field of carried) fields.add(field); + return JSON.stringify({ + ...record, + fields: [...fields].sort(), + previous: kept, + }); +} + +/** Rewrite one audit row's details without its contact values. */ +async function scrubContactAuditRow(id: string): Promise { + return prisma.$transaction(async (tx) => { + const fresh = await tx.auditLog.findUnique({ + where: { id }, + select: { details: true }, + }); + const scrubbed = scrubContactAuditDetails(fresh?.details ?? null); + if (scrubbed === null) return false; + await tx.auditLog.update({ where: { id }, data: { details: scrubbed } }); + return true; + }); +} + +/** + * Audit rows of this action that may still carry a contact value in + * `previous`. The text match is a prefilter only (a key is followed by a + * colon, a field name in `fields` is not); the row is parsed before it is + * rewritten. + */ +function contactAuditCandidates(userId: string | null) { + return prisma.auditLog.findMany({ + where: { + userId, + action: CONTACT_AUDIT_ACTION, + OR: SEALED_CONTACT_FIELDS.map((field) => ({ + details: { contains: `"${field}":` }, + })), + }, + select: { id: true }, + take: PAGE_SIZE, + }); } /** @@ -158,7 +254,17 @@ async function migrateRouteGeometry(id: string): Promise { where: { id }, select: { geometry: true }, }); - if (!fresh || fresh.geometry === null) return false; + if (!fresh) return false; + if (fresh.geometry === null) { + // The page only lists rows whose readable column is set, so a null here + // is a JSON null (or a racing pass's SQL NULL). Nothing to seal; set it + // to SQL NULL so the row leaves discovery. + await tx.workoutRoute.update({ + where: { id }, + data: { geometry: Prisma.DbNull }, + }); + return true; + } await tx.workoutRoute.update({ where: { id }, data: { @@ -264,6 +370,20 @@ export async function runFreeTextEncryptionBackfillForUser( once, ); + // An appointment reminder's address is resolved from the practitioner when + // it fires; the readable copy earlier releases wrote here goes. + const appointmentAddressesCleared = ( + await prisma.measurementReminder.updateMany({ + where: { userId, origin: "ENCOUNTER", location: { not: null } }, + data: { location: null }, + }) + ).count; + const contactAuditRowsScrubbed = await drain( + () => contactAuditCandidates(userId), + scrubContactAuditRow, + once, + ); + annotate({ action: { name: "free_text.encryption.backfill", @@ -272,6 +392,8 @@ export async function runFreeTextEncryptionBackfillForUser( metric_notes_migrated: metricNotesMigrated, practitioner_contacts_migrated: practitionerContactsMigrated, route_geometries_migrated: routeGeometriesMigrated, + appointment_addresses_cleared: appointmentAddressesCleared, + contact_audit_rows_scrubbed: contactAuditRowsScrubbed, }, }, }); @@ -281,13 +403,18 @@ export async function runFreeTextEncryptionBackfillForUser( metricNotesMigrated, practitionerContactsMigrated, routeGeometriesMigrated, + appointmentAddressesCleared, + contactAuditRowsScrubbed, }; } /** * Discovery. Finds every user still holding a readable value in any of the * columns above and enqueues one backfill job per account. The 0357 and 0361 - * partial indexes match the predicates, so the scan is index-only and converges to empty. pg-boss + * partial indexes match the predicates, so the scan is index-only and converges to empty. + * The appointment-address and audit predicates have no partial index: the + * first scans the reminder table, the second reads one action through the + * `(action, created_at)` index; both converge to empty the same way. pg-boss * `singletonKey` coalesces duplicate sends. Best-effort: errors come back * through the result value so worker boot never fails on a miss. */ @@ -301,6 +428,15 @@ export async function enqueueBootTimeFreeTextEncryptionBackfill( } try { + // Audit rows of an account deleted since have no owner, so no per-user + // job reaches them: they are scrubbed here. The set is bounded by the + // audit retention and converges to empty. + await drain( + () => contactAuditCandidates(null), + scrubContactAuditRow, + () => {}, + ); + const found = await Promise.all([ prisma.$queryRaw<{ user_id: string }[]>` SELECT DISTINCT user_id FROM coach_conversations @@ -315,6 +451,14 @@ export async function enqueueBootTimeFreeTextEncryptionBackfill( SELECT DISTINCT w.user_id FROM workout_routes r JOIN workouts w ON w.id = r.workout_id WHERE r.geometry IS NOT NULL`, + prisma.$queryRaw<{ user_id: string }[]>` + SELECT DISTINCT user_id FROM measurement_reminders + WHERE origin = 'ENCOUNTER' AND location IS NOT NULL`, + prisma.$queryRaw<{ user_id: string }[]>` + SELECT DISTINCT user_id FROM audit_logs + WHERE action = ${CONTACT_AUDIT_ACTION} + AND user_id IS NOT NULL + AND (details LIKE '%"location":%' OR details LIKE '%"phone":%')`, ]); const userIds = new Set(); diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index 83828963a..460ac70ec 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -1249,6 +1249,8 @@ export async function registerMaintenanceQueues( let notes = 0; let contacts = 0; let routes = 0; + let appointmentAddresses = 0; + let contactAudits = 0; for (const job of jobs) { const { userId } = job.data; if (!userId) { @@ -1268,15 +1270,19 @@ export async function registerMaintenanceQueues( metricNotesMigrated, practitionerContactsMigrated, routeGeometriesMigrated, + appointmentAddressesCleared, + contactAuditRowsScrubbed, } = await runFreeTextEncryptionBackfillForUser(userId); users++; titles += conversationTitlesMigrated; notes += metricNotesMigrated; contacts += practitionerContactsMigrated; routes += routeGeometriesMigrated; + appointmentAddresses += appointmentAddressesCleared; + contactAudits += contactAuditRowsScrubbed; workerLog( "info", - `[free-text-encryption-backfill] user=${userId} titles=${conversationTitlesMigrated} notes=${metricNotesMigrated} contacts=${practitionerContactsMigrated} routes=${routeGeometriesMigrated}`, + `[free-text-encryption-backfill] user=${userId} titles=${conversationTitlesMigrated} notes=${metricNotesMigrated} contacts=${practitionerContactsMigrated} routes=${routeGeometriesMigrated} appointment_addresses=${appointmentAddressesCleared} contact_audits=${contactAuditRowsScrubbed}`, ); } catch (err) { recordError(); @@ -1298,6 +1304,8 @@ export async function registerMaintenanceQueues( metric_notes_migrated: notes, practitioner_contacts_migrated: contacts, route_geometries_migrated: routes, + appointment_addresses_cleared: appointmentAddresses, + contact_audit_rows_scrubbed: contactAudits, }); }, ); diff --git a/tests/integration/free-text-encryption-backfill.test.ts b/tests/integration/free-text-encryption-backfill.test.ts index 7ce676292..5b76a751f 100644 --- a/tests/integration/free-text-encryption-backfill.test.ts +++ b/tests/integration/free-text-encryption-backfill.test.ts @@ -16,6 +16,8 @@ */ import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { Prisma } from "@/generated/prisma/client"; + import { setGlobalBoss } from "@/lib/jobs/boss-instance"; import { encryptToBytes } from "@/lib/ai/coach/bytes-codec"; import { readNote } from "@/lib/crypto/note-cipher"; @@ -122,12 +124,16 @@ describe("free-text encryption backfill (real Postgres)", () => { metricNotesMigrated: 0, practitionerContactsMigrated: 0, routeGeometriesMigrated: 0, + appointmentAddressesCleared: 0, + contactAuditRowsScrubbed: 0, }); expect(await runFreeTextEncryptionBackfillForUser("ft-b")).toEqual({ conversationTitlesMigrated: 0, metricNotesMigrated: 1, practitionerContactsMigrated: 0, routeGeometriesMigrated: 0, + appointmentAddressesCleared: 0, + contactAuditRowsScrubbed: 0, }); const conversations = await prisma.coachConversation.findMany({ @@ -171,6 +177,8 @@ describe("free-text encryption backfill (real Postgres)", () => { metricNotesMigrated: 0, practitionerContactsMigrated: 0, routeGeometriesMigrated: 0, + appointmentAddressesCleared: 0, + contactAuditRowsScrubbed: 0, }); }); @@ -253,12 +261,16 @@ describe("free-text encryption backfill (real Postgres)", () => { metricNotesMigrated: 0, practitionerContactsMigrated: 2, routeGeometriesMigrated: 0, + appointmentAddressesCleared: 0, + contactAuditRowsScrubbed: 0, }); expect(await runFreeTextEncryptionBackfillForUser("pc-b")).toEqual({ conversationTitlesMigrated: 0, metricNotesMigrated: 0, practitionerContactsMigrated: 0, routeGeometriesMigrated: 1, + appointmentAddressesCleared: 0, + contactAuditRowsScrubbed: 0, }); const practitioners = await prisma.practitioner.findMany({ @@ -317,9 +329,152 @@ describe("free-text encryption backfill (real Postgres)", () => { metricNotesMigrated: 0, practitionerContactsMigrated: 0, routeGeometriesMigrated: 0, + appointmentAddressesCleared: 0, + contactAuditRowsScrubbed: 0, }); }); + it("clears appointment address copies, scrubs old contact audit rows, and settles a JSON-null track", async () => { + const prisma = getPrismaClient(); + await Promise.all([seedUser("ad-a"), seedUser("ad-b")]); + + // ad-a: an appointment reminder holding a readable copy of the address, + // and a checkup whose own location is the person's words and stays. + await prisma.measurementReminder.create({ + data: { + id: "rem-appt", + userId: "ad-a", + label: "Praxis Nord", + origin: "ENCOUNTER", + location: "Hauptstr. 1, Berlin", + }, + }); + await prisma.measurementReminder.create({ + data: { + id: "rem-checkup", + userId: "ad-a", + label: "Blood panel", + location: "Labor Mitte", + }, + }); + // ad-a: an edit's audit row from before the contact fields were sealed, + // carrying the old address and phone number, and a current-shape row. + await prisma.auditLog.create({ + data: { + id: "audit-old", + userId: "ad-a", + action: "practitioner.contact.update", + details: JSON.stringify({ + practitionerId: "pr-x", + fields: ["location", "name", "phone"], + previous: { + location: "Hauptstr. 1, Berlin", + name: "Praxis Alt", + phone: "+49 30 1234567", + }, + }), + }, + }); + const currentDetails = JSON.stringify({ + practitionerId: "pr-x", + fields: ["location"], + previous: {}, + }); + await prisma.auditLog.create({ + data: { + id: "audit-current", + userId: "ad-a", + action: "practitioner.contact.update", + details: currentDetails, + }, + }); + // An account deleted since: the row lost its owner and no per-user job + // reaches it, so discovery scrubs it itself. + await prisma.auditLog.create({ + data: { + id: "audit-orphan", + userId: null, + action: "practitioner.contact.update", + details: JSON.stringify({ + practitionerId: "pr-y", + fields: ["phone"], + previous: { phone: "+49 40 7654321" }, + }), + }, + }); + // ad-b: a route whose readable column is a JSON null rather than SQL + // NULL. Nothing to seal, but discovery must not find it every night. + const startedAt = new Date("2026-09-01T06:00:00.000Z"); + await prisma.workout.create({ + data: { + id: "wk-jsonnull", + userId: "ad-b", + sportType: "running", + startedAt, + endedAt: new Date(startedAt.getTime() + 30 * 60_000), + durationSec: 1800, + route: { create: { id: "rt-jsonnull", geometry: Prisma.JsonNull } }, + }, + }); + + const first = makeCapturingBoss(); + setGlobalBoss(first.boss as never); + expect( + (await enqueueBootTimeFreeTextEncryptionBackfill()).error, + ).toBeNull(); + expect(first.sent.map((s) => s.userId).sort()).toEqual(["ad-a", "ad-b"]); + + const summaryA = await runFreeTextEncryptionBackfillForUser("ad-a"); + expect(summaryA).toMatchObject({ + appointmentAddressesCleared: 1, + contactAuditRowsScrubbed: 1, + }); + await runFreeTextEncryptionBackfillForUser("ad-b"); + + const reminders = await prisma.measurementReminder.findMany({ + where: { userId: "ad-a" }, + orderBy: { id: "asc" }, + }); + expect(reminders.map((r) => [r.id, r.location])).toEqual([ + ["rem-appt", null], + ["rem-checkup", "Labor Mitte"], + ]); + + const audits = await prisma.auditLog.findMany({ + where: { action: "practitioner.contact.update" }, + orderBy: { id: "asc" }, + }); + const byId = Object.fromEntries(audits.map((a) => [a.id, a.details])); + expect(JSON.parse(byId["audit-old"]!)).toEqual({ + practitionerId: "pr-x", + fields: ["location", "name", "phone"], + previous: { name: "Praxis Alt" }, + }); + expect(byId["audit-current"]).toBe(currentDetails); + expect(JSON.parse(byId["audit-orphan"]!)).toEqual({ + practitionerId: "pr-y", + fields: ["phone"], + previous: {}, + }); + for (const details of Object.values(byId)) { + expect(details).not.toContain("Hauptstr"); + expect(details).not.toContain("+49"); + } + + const onDisk = await prisma.$queryRaw<{ readable_is_null: boolean }[]>` + SELECT geometry IS NULL AS readable_is_null + FROM workout_routes WHERE id = 'rt-jsonnull'`; + expect(onDisk).toEqual([{ readable_is_null: true }]); + + // Converged: the next discovery finds nobody. + const second = makeCapturingBoss(); + setGlobalBoss(second.boss as never); + expect( + (await enqueueBootTimeFreeTextEncryptionBackfill()).error, + ).toBeNull(); + expect(second.sent).toEqual([]); + }); + it("has the partial indexes migration 0361 adds", async () => { const indexes = await getPrismaClient().$queryRaw< { indexname: string; indexdef: string }[]