diff --git a/src/app/api/admin/backups/route.ts b/src/app/api/admin/backups/route.ts index 2a1939e08..b24339aa7 100644 --- a/src/app/api/admin/backups/route.ts +++ b/src/app/api/admin/backups/route.ts @@ -167,9 +167,20 @@ export const GET = apiHandler(async () => { a.username.localeCompare(b.username), ); + // A wipe whose copies are already gone waits for the next nightly run + // before its request is removed (see `offhost-purge.ts`); nothing of it is + // left in the bucket, so it is not counted as a pending deletion. + const stillInBucket = { + NOT: { + reason: "data_wiped", + lastFailure: null, + lastAttemptAt: { not: null }, + }, + }; const [purgeCount, oldestPurge, lifecycle] = await Promise.all([ - prisma.offhostPurgeRequest.count(), + prisma.offhostPurgeRequest.count({ where: stillInBucket }), prisma.offhostPurgeRequest.findFirst({ + where: stillInBucket, orderBy: { requestedAt: "asc" }, select: { requestedAt: true, lastFailure: true }, }), diff --git a/src/lib/jobs/__tests__/offhost-purge.test.ts b/src/lib/jobs/__tests__/offhost-purge.test.ts index eb6224df9..4a7b958e8 100644 --- a/src/lib/jobs/__tests__/offhost-purge.test.ts +++ b/src/lib/jobs/__tests__/offhost-purge.test.ts @@ -27,6 +27,7 @@ interface Row { function makeLedger(existingUsers: string[] = []) { const rows: Row[] = []; + const walked = new Map(); let seq = 0; const prisma = { offhostPurgeRequest: { @@ -60,11 +61,15 @@ function makeLedger(existingUsers: string[] = []) { data, }: { where: { id: { in: string[] } }; - data: { lastAttemptAt: Date; lastFailure: string }; + data: { + lastAttemptAt: Date; + lastFailure: string | null; + attempts?: unknown; + }; }) => { for (const row of rows) { if (!where.id.in.includes(row.id)) continue; - row.attempts += 1; + if (data.attempts) row.attempts += 1; row.lastAttemptAt = data.lastAttemptAt; row.lastFailure = data.lastFailure; } @@ -72,12 +77,23 @@ function makeLedger(existingUsers: string[] = []) { ), }, user: { - count: vi.fn(async ({ where }: { where: { id: string } }) => - existingUsers.includes(where.id) ? 1 : 0, + findMany: vi.fn(async ({ where }: { where: { id: { in: string[] } } }) => + existingUsers + .filter((id) => where.id.in.includes(id)) + .map((id) => ({ id })), + ), + }, + // When the nightly run last walked each account. + offhostBackupState: { + findMany: vi.fn(async () => + [...walked].map(([userId, lastAttemptAt]) => ({ + userId, + lastAttemptAt, + })), ), }, }; - return { rows, prisma }; + return { rows, prisma, walked }; } function makeBucket(keys: string[]) { @@ -149,7 +165,7 @@ describe("off-host purge", () => { }); it("keeps the request, with the bucket's answer, when a delete is refused", async () => { - const { rows, prisma } = makeLedger(["gone"]); + const { rows, prisma, walked } = makeLedger(["gone"]); await requestOffhostPurge(prisma as never, "gone", "data_wiped"); const bucket = makeBucket(BUCKET); bucket.deleteObject.mockRejectedValueOnce(new Error("AccessDenied")); @@ -160,8 +176,9 @@ describe("off-host purge", () => { expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ attempts: 1, lastFailure: "AccessDenied" }); - // The next run finishes it, and the account that still exists keeps the - // receipt. + // The next run finishes it once a nightly run has walked the account + // since the wipe, and the account that still exists keeps the receipt. + walked.set("gone", new Date(Date.now() + 60_000)); const again = await processOffhostPurges(prisma as never, bucket); expect(again.completed).toBe(1); expect(rows).toHaveLength(0); @@ -171,6 +188,54 @@ describe("off-host purge", () => { ); }); + it("keeps a wipe's request until a nightly run has walked the account since", async () => { + // The upload in flight during the wipe reads the request to throw its + // pre-wipe copy away; removing the request on the first purge let that + // copy land afterwards and stay. + const { rows, prisma, walked } = makeLedger(["gone"]); + walked.set("gone", new Date(Date.now() - 3_600_000)); + await requestOffhostPurge(prisma as never, "gone", "data_wiped"); + const wipeDay = rows[0]!.requestedAt.toISOString().slice(0, 10); + const bucket = makeBucket(BUCKET); + + const first = await processOffhostPurges(prisma as never, bucket); + expect(first).toMatchObject({ completed: 0, awaitingRun: 1, failed: 0 }); + expect(first.objectsDeleted).toBe(2); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ attempts: 0, lastFailure: null }); + expect(auditLog).toHaveBeenCalledTimes(1); + + // A pass before the run reached the account keeps waiting and writes no + // second receipt. + const waiting = await processOffhostPurges(prisma as never, bucket); + expect(waiting).toMatchObject({ completed: 0, awaitingRun: 1 }); + expect(auditLog).toHaveBeenCalledTimes(1); + + // The next night walks the account and writes the copy of what it holds + // after the wipe. The purge then closes the request and leaves that copy. + const nextDay = new Date(Date.parse(`${wipeDay}T00:00:00Z`) + 86_400_000) + .toISOString() + .slice(0, 10); + bucket.store.add(`${nextDay}/user-gone.json.enc`); + walked.set("gone", new Date(Date.now() + 60_000)); + const done = await processOffhostPurges(prisma as never, bucket); + expect(done).toMatchObject({ completed: 1, awaitingRun: 0 }); + expect(rows).toHaveLength(0); + expect(bucket.store.has(`${nextDay}/user-gone.json.enc`)).toBe(true); + }); + + it("removes every copy of an account deleted after a wipe", async () => { + const { rows, prisma } = makeLedger(); + await requestOffhostPurge(prisma as never, "gone", "data_wiped"); + await requestOffhostPurge(prisma as never, "gone", "account_deleted"); + const bucket = makeBucket([...BUCKET, "2999-01-01/user-gone.json.enc"]); + + const report = await processOffhostPurges(prisma as never, bucket); + expect(report).toMatchObject({ completed: 2, awaitingRun: 0 }); + expect(rows).toHaveLength(0); + expect([...bucket.store].some((k) => k.includes("user-gone."))).toBe(false); + }); + it("records nothing on a host without off-host backup", async () => { vi.stubEnv("BACKUP_S3_ENDPOINT", ""); const { rows, prisma } = makeLedger(); diff --git a/src/lib/jobs/offhost-purge.ts b/src/lib/jobs/offhost-purge.ts index 15585fb03..a42fcb78f 100644 --- a/src/lib/jobs/offhost-purge.ts +++ b/src/lib/jobs/offhost-purge.ts @@ -17,6 +17,13 @@ * the job runs again every night and after every nightly upload, so a late * copy the upload wrote for an account deleted while it ran is caught too. * + * A wiped account keeps its request until a nightly run has walked it since + * the wipe. The upload that was in flight when the data went reads the + * request to know its copy is from before the wipe; removing the request on + * the first purge let that copy land afterwards and stay. Until then each + * purge removes only the copies dated up to the wipe's day, so the copy of + * what the account holds after the wipe is left alone. + * * The admin wipe of every account writes no request, on purpose: it is the * operator clearing their own host, the bucket is theirs, and its copies are * the documented way back from a wipe pressed by mistake. The health-check @@ -57,6 +64,11 @@ const MAX_DELETES_PER_RUN = 50_000; type Db = Pick; +type PurgeDb = Pick< + PrismaClient, + "offhostPurgeRequest" | "offhostBackupState" | "user" +>; + /** * Record that `subjectId`'s off-host copies have to go. Call it with the * transaction that deletes the account or wipes its data, so the request and @@ -101,6 +113,8 @@ export interface OffhostPurgeReport { pending: number; completed: number; failed: number; + /** Wipe requests kept until a nightly run has walked the account since. */ + awaitingRun: number; objectsDeleted: number; } @@ -114,7 +128,7 @@ function trimError(err: unknown): string { * gone. Answers what it did. */ export async function processOffhostPurges( - prisma: PrismaClient, + prisma: PurgeDb, s3Override?: S3Like, ): Promise { const requests = await prisma.offhostPurgeRequest.findMany({ @@ -125,6 +139,7 @@ export async function processOffhostPurges( pending: requests.length, completed: 0, failed: 0, + awaitingRun: 0, objectsDeleted: 0, }; if (requests.length === 0) return report; @@ -163,12 +178,56 @@ export async function processOffhostPurges( } const subjects = new Set(requests.map((r) => r.subjectId)); + const [liveUsers, ledger] = await Promise.all([ + prisma.user.findMany({ + where: { id: { in: [...subjects] } }, + select: { id: true }, + }), + prisma.offhostBackupState.findMany({ + where: { userId: { in: [...subjects] } }, + select: { userId: true, lastAttemptAt: true }, + }), + ]); + const live = new Set(liveUsers.map((u) => u.id)); + const lastWalked = new Map(ledger.map((r) => [r.userId, r.lastAttemptAt])); + + // A wipe of an account that still exists: its request stays until a + // nightly run has walked the account since, because the upload in flight + // at the wipe reads the request to throw its pre-wipe copy away. + const awaitsRun = (request: (typeof requests)[number]) => + request.reason === "data_wiped" && + live.has(request.subjectId) && + !( + (lastWalked.get(request.subjectId)?.getTime() ?? 0) >= + request.requestedAt.getTime() + ); + + // Where every request of a subject is a wipe of a live account, only the + // copies dated up to the latest wipe's day go; a copy from a later night + // holds what the account has recorded since, and stays. An account that is + // gone loses every copy. + const lastDayToDelete = new Map(); + for (const request of requests) { + const subject = request.subjectId; + const day = + request.reason === "data_wiped" && live.has(subject) + ? request.requestedAt.toISOString().slice(0, 10) + : null; + const prior = lastDayToDelete.get(subject); + if (prior === undefined) lastDayToDelete.set(subject, day); + else if (prior !== null) { + lastDayToDelete.set(subject, day === null || day > prior ? day : prior); + } + } + const bySubject = new Map(); for (const { key } of objects) { const match = ACCOUNT_OBJECT.exec(key); if (!match) continue; const subject = match[1]; if (!subjects.has(subject)) continue; + const lastDay = lastDayToDelete.get(subject); + if (lastDay && key.slice(0, 10) > lastDay) continue; const list = bySubject.get(subject) ?? []; list.push(key); bySubject.set(subject, list); @@ -205,14 +264,29 @@ export async function processOffhostPurges( continue; } const deleted = bySubject.get(request.subjectId)?.length ?? 0; - await prisma.offhostPurgeRequest.delete({ where: { id: request.id } }); - report.completed++; + // A wipe that already waited a pass had its copies removed, and its + // receipt written, on that pass. + const alreadyReceipted = + request.lastAttemptAt !== null && request.lastFailure === null; + if (awaitsRun(request)) { + // Its copies up to the wipe are gone; what is left is the wait for + // the next run. Marked so the admin card stops counting it as copies + // still in the bucket. + await prisma.offhostPurgeRequest.updateMany({ + where: { id: { in: [request.id] } }, + data: { lastAttemptAt: now, lastFailure: null }, + }); + report.awaitingRun++; + } else { + await prisma.offhostPurgeRequest.delete({ where: { id: request.id } }); + report.completed++; + } + if (alreadyReceipted && deleted === 0) continue; // The account a deleted subject named is gone, and so is its audit // history; the receipt names the reason and the count, not the account. // A wiped account still exists and keeps the receipt as its own. const accountRemains = - request.reason === "data_wiped" && - (await prisma.user.count({ where: { id: request.subjectId } })) > 0; + request.reason === "data_wiped" && live.has(request.subjectId); await auditLog("offhost.backup.purged", { userId: accountRemains ? request.subjectId : null, actorUserId: null, @@ -236,6 +310,7 @@ export async function handleOffhostPurge( evt.addMeta("offhost_purge_pending", report.pending); evt.addMeta("offhost_purge_completed", report.completed); evt.addMeta("offhost_purge_failed", report.failed); + evt.addMeta("offhost_purge_awaiting_run", report.awaitingRun); evt.addMeta("offhost_purge_objects_deleted", report.objectsDeleted); if (report.failed > 0) { await reportWorkerError( @@ -249,6 +324,7 @@ export async function handleOffhostPurge( offhost_purge_pending: report.pending, offhost_purge_completed: report.completed, offhost_purge_failed: report.failed, + offhost_purge_awaiting_run: report.awaitingRun, offhost_purge_objects_deleted: report.objectsDeleted, }); } catch (err) { diff --git a/tests/integration/offhost-purge-on-deletion.test.ts b/tests/integration/offhost-purge-on-deletion.test.ts index 9c96deb75..3cf7ac76d 100644 --- a/tests/integration/offhost-purge-on-deletion.test.ts +++ b/tests/integration/offhost-purge-on-deletion.test.ts @@ -153,5 +153,18 @@ describe("off-host copies leave with the account", () => { where: { action: "offhost.backup.purged", userId }, }); expect(receipt).not.toBeNull(); + + // The request outlives the purge until a nightly run has walked the + // account since the wipe: the upload in flight at the wipe reads it to + // throw its pre-wipe copy away. + expect(await prisma.offhostPurgeRequest.count()).toBe(1); + await prisma.offhostBackupState.upsert({ + where: { userId }, + create: { userId, lastAttemptAt: new Date(Date.now() + 1000) }, + update: { lastAttemptAt: new Date(Date.now() + 1000) }, + }); + const report = await processOffhostPurges(prisma); + expect(report).toMatchObject({ completed: 1, awaitingRun: 0 }); + expect(await prisma.offhostPurgeRequest.count()).toBe(0); }); });