From bb12d942c0e61d4b36b2c69955656bf5a9525f79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 26 Sep 2026 22:08:51 +0200 Subject: [PATCH] fix(backup): keep a wipe's purge request until the next off-host run A data wipe during the nightly upload could leave the pre-wipe copy in the bucket: the purge the wipe started removed its request at once, and the upload still in flight checked that request after finishing and found nothing. A wipe's request now stays until a nightly run has walked the account since the wipe. Until then each purge removes only the copies dated up to the wipe's day, so the copy of what the account holds after the wipe stays, and the admin card no longer counts the waiting request as copies still in the bucket. --- src/app/api/admin/backups/route.ts | 13 ++- src/lib/jobs/__tests__/offhost-purge.test.ts | 81 +++++++++++++++-- src/lib/jobs/offhost-purge.ts | 86 +++++++++++++++++-- .../offhost-purge-on-deletion.test.ts | 13 +++ 4 files changed, 179 insertions(+), 14 deletions(-) diff --git a/src/app/api/admin/backups/route.ts b/src/app/api/admin/backups/route.ts index 2a1939e08..b24339aa7 100644 --- a/src/app/api/admin/backups/route.ts +++ b/src/app/api/admin/backups/route.ts @@ -167,9 +167,20 @@ export const GET = apiHandler(async () => { a.username.localeCompare(b.username), ); + // A wipe whose copies are already gone waits for the next nightly run + // before its request is removed (see `offhost-purge.ts`); nothing of it is + // left in the bucket, so it is not counted as a pending deletion. + const stillInBucket = { + NOT: { + reason: "data_wiped", + lastFailure: null, + lastAttemptAt: { not: null }, + }, + }; const [purgeCount, oldestPurge, lifecycle] = await Promise.all([ - prisma.offhostPurgeRequest.count(), + prisma.offhostPurgeRequest.count({ where: stillInBucket }), prisma.offhostPurgeRequest.findFirst({ + where: stillInBucket, orderBy: { requestedAt: "asc" }, select: { requestedAt: true, lastFailure: true }, }), diff --git a/src/lib/jobs/__tests__/offhost-purge.test.ts b/src/lib/jobs/__tests__/offhost-purge.test.ts index eb6224df9..4a7b958e8 100644 --- a/src/lib/jobs/__tests__/offhost-purge.test.ts +++ b/src/lib/jobs/__tests__/offhost-purge.test.ts @@ -27,6 +27,7 @@ interface Row { function makeLedger(existingUsers: string[] = []) { const rows: Row[] = []; + const walked = new Map(); let seq = 0; const prisma = { offhostPurgeRequest: { @@ -60,11 +61,15 @@ function makeLedger(existingUsers: string[] = []) { data, }: { where: { id: { in: string[] } }; - data: { lastAttemptAt: Date; lastFailure: string }; + data: { + lastAttemptAt: Date; + lastFailure: string | null; + attempts?: unknown; + }; }) => { for (const row of rows) { if (!where.id.in.includes(row.id)) continue; - row.attempts += 1; + if (data.attempts) row.attempts += 1; row.lastAttemptAt = data.lastAttemptAt; row.lastFailure = data.lastFailure; } @@ -72,12 +77,23 @@ function makeLedger(existingUsers: string[] = []) { ), }, user: { - count: vi.fn(async ({ where }: { where: { id: string } }) => - existingUsers.includes(where.id) ? 1 : 0, + findMany: vi.fn(async ({ where }: { where: { id: { in: string[] } } }) => + existingUsers + .filter((id) => where.id.in.includes(id)) + .map((id) => ({ id })), + ), + }, + // When the nightly run last walked each account. + offhostBackupState: { + findMany: vi.fn(async () => + [...walked].map(([userId, lastAttemptAt]) => ({ + userId, + lastAttemptAt, + })), ), }, }; - return { rows, prisma }; + return { rows, prisma, walked }; } function makeBucket(keys: string[]) { @@ -149,7 +165,7 @@ describe("off-host purge", () => { }); it("keeps the request, with the bucket's answer, when a delete is refused", async () => { - const { rows, prisma } = makeLedger(["gone"]); + const { rows, prisma, walked } = makeLedger(["gone"]); await requestOffhostPurge(prisma as never, "gone", "data_wiped"); const bucket = makeBucket(BUCKET); bucket.deleteObject.mockRejectedValueOnce(new Error("AccessDenied")); @@ -160,8 +176,9 @@ describe("off-host purge", () => { expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ attempts: 1, lastFailure: "AccessDenied" }); - // The next run finishes it, and the account that still exists keeps the - // receipt. + // The next run finishes it once a nightly run has walked the account + // since the wipe, and the account that still exists keeps the receipt. + walked.set("gone", new Date(Date.now() + 60_000)); const again = await processOffhostPurges(prisma as never, bucket); expect(again.completed).toBe(1); expect(rows).toHaveLength(0); @@ -171,6 +188,54 @@ describe("off-host purge", () => { ); }); + it("keeps a wipe's request until a nightly run has walked the account since", async () => { + // The upload in flight during the wipe reads the request to throw its + // pre-wipe copy away; removing the request on the first purge let that + // copy land afterwards and stay. + const { rows, prisma, walked } = makeLedger(["gone"]); + walked.set("gone", new Date(Date.now() - 3_600_000)); + await requestOffhostPurge(prisma as never, "gone", "data_wiped"); + const wipeDay = rows[0]!.requestedAt.toISOString().slice(0, 10); + const bucket = makeBucket(BUCKET); + + const first = await processOffhostPurges(prisma as never, bucket); + expect(first).toMatchObject({ completed: 0, awaitingRun: 1, failed: 0 }); + expect(first.objectsDeleted).toBe(2); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ attempts: 0, lastFailure: null }); + expect(auditLog).toHaveBeenCalledTimes(1); + + // A pass before the run reached the account keeps waiting and writes no + // second receipt. + const waiting = await processOffhostPurges(prisma as never, bucket); + expect(waiting).toMatchObject({ completed: 0, awaitingRun: 1 }); + expect(auditLog).toHaveBeenCalledTimes(1); + + // The next night walks the account and writes the copy of what it holds + // after the wipe. The purge then closes the request and leaves that copy. + const nextDay = new Date(Date.parse(`${wipeDay}T00:00:00Z`) + 86_400_000) + .toISOString() + .slice(0, 10); + bucket.store.add(`${nextDay}/user-gone.json.enc`); + walked.set("gone", new Date(Date.now() + 60_000)); + const done = await processOffhostPurges(prisma as never, bucket); + expect(done).toMatchObject({ completed: 1, awaitingRun: 0 }); + expect(rows).toHaveLength(0); + expect(bucket.store.has(`${nextDay}/user-gone.json.enc`)).toBe(true); + }); + + it("removes every copy of an account deleted after a wipe", async () => { + const { rows, prisma } = makeLedger(); + await requestOffhostPurge(prisma as never, "gone", "data_wiped"); + await requestOffhostPurge(prisma as never, "gone", "account_deleted"); + const bucket = makeBucket([...BUCKET, "2999-01-01/user-gone.json.enc"]); + + const report = await processOffhostPurges(prisma as never, bucket); + expect(report).toMatchObject({ completed: 2, awaitingRun: 0 }); + expect(rows).toHaveLength(0); + expect([...bucket.store].some((k) => k.includes("user-gone."))).toBe(false); + }); + it("records nothing on a host without off-host backup", async () => { vi.stubEnv("BACKUP_S3_ENDPOINT", ""); const { rows, prisma } = makeLedger(); diff --git a/src/lib/jobs/offhost-purge.ts b/src/lib/jobs/offhost-purge.ts index 15585fb03..a42fcb78f 100644 --- a/src/lib/jobs/offhost-purge.ts +++ b/src/lib/jobs/offhost-purge.ts @@ -17,6 +17,13 @@ * the job runs again every night and after every nightly upload, so a late * copy the upload wrote for an account deleted while it ran is caught too. * + * A wiped account keeps its request until a nightly run has walked it since + * the wipe. The upload that was in flight when the data went reads the + * request to know its copy is from before the wipe; removing the request on + * the first purge let that copy land afterwards and stay. Until then each + * purge removes only the copies dated up to the wipe's day, so the copy of + * what the account holds after the wipe is left alone. + * * The admin wipe of every account writes no request, on purpose: it is the * operator clearing their own host, the bucket is theirs, and its copies are * the documented way back from a wipe pressed by mistake. The health-check @@ -57,6 +64,11 @@ const MAX_DELETES_PER_RUN = 50_000; type Db = Pick; +type PurgeDb = Pick< + PrismaClient, + "offhostPurgeRequest" | "offhostBackupState" | "user" +>; + /** * Record that `subjectId`'s off-host copies have to go. Call it with the * transaction that deletes the account or wipes its data, so the request and @@ -101,6 +113,8 @@ export interface OffhostPurgeReport { pending: number; completed: number; failed: number; + /** Wipe requests kept until a nightly run has walked the account since. */ + awaitingRun: number; objectsDeleted: number; } @@ -114,7 +128,7 @@ function trimError(err: unknown): string { * gone. Answers what it did. */ export async function processOffhostPurges( - prisma: PrismaClient, + prisma: PurgeDb, s3Override?: S3Like, ): Promise { const requests = await prisma.offhostPurgeRequest.findMany({ @@ -125,6 +139,7 @@ export async function processOffhostPurges( pending: requests.length, completed: 0, failed: 0, + awaitingRun: 0, objectsDeleted: 0, }; if (requests.length === 0) return report; @@ -163,12 +178,56 @@ export async function processOffhostPurges( } const subjects = new Set(requests.map((r) => r.subjectId)); + const [liveUsers, ledger] = await Promise.all([ + prisma.user.findMany({ + where: { id: { in: [...subjects] } }, + select: { id: true }, + }), + prisma.offhostBackupState.findMany({ + where: { userId: { in: [...subjects] } }, + select: { userId: true, lastAttemptAt: true }, + }), + ]); + const live = new Set(liveUsers.map((u) => u.id)); + const lastWalked = new Map(ledger.map((r) => [r.userId, r.lastAttemptAt])); + + // A wipe of an account that still exists: its request stays until a + // nightly run has walked the account since, because the upload in flight + // at the wipe reads the request to throw its pre-wipe copy away. + const awaitsRun = (request: (typeof requests)[number]) => + request.reason === "data_wiped" && + live.has(request.subjectId) && + !( + (lastWalked.get(request.subjectId)?.getTime() ?? 0) >= + request.requestedAt.getTime() + ); + + // Where every request of a subject is a wipe of a live account, only the + // copies dated up to the latest wipe's day go; a copy from a later night + // holds what the account has recorded since, and stays. An account that is + // gone loses every copy. + const lastDayToDelete = new Map(); + for (const request of requests) { + const subject = request.subjectId; + const day = + request.reason === "data_wiped" && live.has(subject) + ? request.requestedAt.toISOString().slice(0, 10) + : null; + const prior = lastDayToDelete.get(subject); + if (prior === undefined) lastDayToDelete.set(subject, day); + else if (prior !== null) { + lastDayToDelete.set(subject, day === null || day > prior ? day : prior); + } + } + const bySubject = new Map(); for (const { key } of objects) { const match = ACCOUNT_OBJECT.exec(key); if (!match) continue; const subject = match[1]; if (!subjects.has(subject)) continue; + const lastDay = lastDayToDelete.get(subject); + if (lastDay && key.slice(0, 10) > lastDay) continue; const list = bySubject.get(subject) ?? []; list.push(key); bySubject.set(subject, list); @@ -205,14 +264,29 @@ export async function processOffhostPurges( continue; } const deleted = bySubject.get(request.subjectId)?.length ?? 0; - await prisma.offhostPurgeRequest.delete({ where: { id: request.id } }); - report.completed++; + // A wipe that already waited a pass had its copies removed, and its + // receipt written, on that pass. + const alreadyReceipted = + request.lastAttemptAt !== null && request.lastFailure === null; + if (awaitsRun(request)) { + // Its copies up to the wipe are gone; what is left is the wait for + // the next run. Marked so the admin card stops counting it as copies + // still in the bucket. + await prisma.offhostPurgeRequest.updateMany({ + where: { id: { in: [request.id] } }, + data: { lastAttemptAt: now, lastFailure: null }, + }); + report.awaitingRun++; + } else { + await prisma.offhostPurgeRequest.delete({ where: { id: request.id } }); + report.completed++; + } + if (alreadyReceipted && deleted === 0) continue; // The account a deleted subject named is gone, and so is its audit // history; the receipt names the reason and the count, not the account. // A wiped account still exists and keeps the receipt as its own. const accountRemains = - request.reason === "data_wiped" && - (await prisma.user.count({ where: { id: request.subjectId } })) > 0; + request.reason === "data_wiped" && live.has(request.subjectId); await auditLog("offhost.backup.purged", { userId: accountRemains ? request.subjectId : null, actorUserId: null, @@ -236,6 +310,7 @@ export async function handleOffhostPurge( evt.addMeta("offhost_purge_pending", report.pending); evt.addMeta("offhost_purge_completed", report.completed); evt.addMeta("offhost_purge_failed", report.failed); + evt.addMeta("offhost_purge_awaiting_run", report.awaitingRun); evt.addMeta("offhost_purge_objects_deleted", report.objectsDeleted); if (report.failed > 0) { await reportWorkerError( @@ -249,6 +324,7 @@ export async function handleOffhostPurge( offhost_purge_pending: report.pending, offhost_purge_completed: report.completed, offhost_purge_failed: report.failed, + offhost_purge_awaiting_run: report.awaitingRun, offhost_purge_objects_deleted: report.objectsDeleted, }); } catch (err) { diff --git a/tests/integration/offhost-purge-on-deletion.test.ts b/tests/integration/offhost-purge-on-deletion.test.ts index 9c96deb75..3cf7ac76d 100644 --- a/tests/integration/offhost-purge-on-deletion.test.ts +++ b/tests/integration/offhost-purge-on-deletion.test.ts @@ -153,5 +153,18 @@ describe("off-host copies leave with the account", () => { where: { action: "offhost.backup.purged", userId }, }); expect(receipt).not.toBeNull(); + + // The request outlives the purge until a nightly run has walked the + // account since the wipe: the upload in flight at the wipe reads it to + // throw its pre-wipe copy away. + expect(await prisma.offhostPurgeRequest.count()).toBe(1); + await prisma.offhostBackupState.upsert({ + where: { userId }, + create: { userId, lastAttemptAt: new Date(Date.now() + 1000) }, + update: { lastAttemptAt: new Date(Date.now() + 1000) }, + }); + const report = await processOffhostPurges(prisma); + expect(report).toMatchObject({ completed: 1, awaitingRun: 0 }); + expect(await prisma.offhostPurgeRequest.count()).toBe(0); }); });