fix(backup): leave the admin wipe's off-host copies in place, and correct the backup and rotation runbooks

The admin wipe of every account is the operator clearing their own host,
and the bucket's copies are the documented way back from a wipe pressed
by mistake, so it writes no off-host deletion request. Deleting an
account, deleting a managed profile and "Delete all data" still do.

The backup runbook no longer says the worker never deletes an object: it
lists the grant the worker needs (DeleteObject included), when objects
are deleted, the version-4 envelope, the lifecycle check and how to
rotate the off-host key. The rotation runbook no longer says backups are
covered: rotation reaches a backup's envelope, not the content a restore
writes back, and a retired key stays needed until no backup lists it.
The privacy notice beside account deletion says the same about off-host
copies.

A failed import records its failure in a plain try block, and the
extraction test that inflates past the 64 MiB floor gets the time it
needs on a loaded runner.
This commit is contained in:
Marc-André Bombeck committed 2026-09-26 20:33:30 +02:00
1 parent 849748c4cd
commit 7da778760e
19 files changed
+223 -153

No files matched your search

-9
View File
@@ -1,8 +1,3 @@
import {
kickOffhostPurge,
PURGE_ALL_SUBJECTS,
requestOffhostPurge,
} from "@/lib/jobs/offhost-purge";
import { prisma } from "@/lib/db";
import { apiHandler, requireAdmin } from "@/lib/api-handler";
import { auditLog } from "@/lib/auth/audit";
@@ -113,9 +108,6 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
// columns does not. Same classification contract as the model list.
await tx.user.updateMany({ data: USER_RESET });
// Every account's nightly off-host copies go too.
await requestOffhostPurge(tx, PURGE_ALL_SUBJECTS, "admin_wipe");
return perModel;
},
{
@@ -146,7 +138,6 @@ export const DELETE = apiHandler(async (request: NextRequest) => {
// v1.16.9 — the wipe touched every user's rows; clear every cache
// bucket so no per-user payload survives the reset.
invalidateAllCaches();
await kickOffhostPurge();
return apiSuccess({ cleared: true, deletedRows, models: counts });
});
+3 -3
View File
@@ -223,9 +223,9 @@ export const WIPE_MODELS = [
// ended. About this account's record, so they go with it.
"BackupRestoreJob",
// What this host last put in the operator's bucket for this account. The
// objects themselves stay — the worker holds no DeleteObject grant and the
// bucket's lifecycle rule owns their retirement — but the row saying when
// they were written is about this account and goes with it.
// objects themselves are deleted by the purge request the wipe writes
// (`requestOffhostPurge`); this row, saying when they were written, is about
// this account and goes with it.
"OffhostBackupState",
"AuditLog",
"ConsentReceipt",
+1 -6
View File
@@ -30,12 +30,7 @@
*/
import { Buffer } from "node:buffer";
import {
decrypt,
decryptBytes,
extractKeyIdFromBytes,
getConfiguredKeyIds,
} from "@/lib/crypto";
import { decrypt, decryptBytes, getConfiguredKeyIds } from "@/lib/crypto";
/** The key id a value with no key id prefix was written under. */
export const LEGACY_INNER_KEY_ID = "v1";
@@ -193,7 +193,8 @@ describe("extractExportXml", () => {
await expect(extractExportXml(zipPath)).rejects.toThrow(/zip bomb/);
expect(staged().filter((n) => !before.has(n))).toEqual([]);
});
// Inflating past the 64 MiB floor takes a moment on a loaded runner.
}, 60_000);
it("throws when the export.xml member is missing", async () => {
const payload = Buffer.from("noop");
+1 -17
View File
@@ -13,11 +13,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() }));
import { auditLog } from "@/lib/auth/audit";
import {
processOffhostPurges,
PURGE_ALL_SUBJECTS,
requestOffhostPurge,
} from "../offhost-purge";
import { processOffhostPurges, requestOffhostPurge } from "../offhost-purge";
interface Row {
id: string;
@@ -175,18 +171,6 @@ describe("off-host purge", () => {
);
});
it("clears every account object for the admin wipe", async () => {
const { prisma } = makeLedger();
await requestOffhostPurge(
prisma as never,
PURGE_ALL_SUBJECTS,
"admin_wipe",
);
const bucket = makeBucket(BUCKET);
await processOffhostPurges(prisma as never, bucket);
expect([...bucket.store]).toEqual(["_healthcheck/1.bin"]);
});
it("records nothing on a host without off-host backup", async () => {
vi.stubEnv("BACKUP_S3_ENDPOINT", "");
const { rows, prisma } = makeLedger();
+8 -8
View File
@@ -592,21 +592,21 @@ export async function handleAppleHealthImport(
// left behind in that case either.
if (extractedXmlPath) safeUnlink(extractedXmlPath);
safeUnlink(uploadPath);
await prisma.importJob
.update({
try {
await prisma.importJob.update({
where: { id: importJobId },
data: {
status: "failed",
failureReason: reason.slice(0, 1000),
completedAt: new Date(),
},
})
.catch((updateErr: unknown) => {
console.warn(
`[apple-health-import] Could not record the failure on ImportJob=${importJobId}`,
updateErr,
);
});
} catch (updateErr) {
console.warn(
`[apple-health-import] Could not record the failure on ImportJob=${importJobId}`,
updateErr,
);
}
throw err;
}
}
+12 -12
View File
@@ -11,15 +11,15 @@
* Object key layout:
* <bucket>/<YYYY-MM-DD>/user-<userId>.json.enc
*
* Retention: the worker NEVER calls DeleteObject on backup keys. Operators
* MUST configure a bucket-level lifecycle rule (e.g. expire after
* `BACKUP_RETENTION_DAYS`, which this module reads nowhere — see
* `loadOffhostConfig`). This keeps the IAM grant for the worker
* limited to PutObject + GetObject + AbortMultipartUpload, so a compromised
* worker cannot wipe the backup history. The abort is what cleans up a run
* that failed partway rather than leaving billed, unlistable parts behind;
* `AbortMultipartUpload` can only touch an upload this worker started, never
* a finished object. See docs/ops/backup-restore.md.
* Retention: the worker does not expire objects by age. Operators configure a
* bucket-level lifecycle rule (e.g. expire after `BACKUP_RETENTION_DAYS`,
* which this module reads nowhere — see `loadOffhostConfig`); the admin
* off-host card reads that rule back (`probeOffhostLifecycle`) and says when
* there is none. Objects ARE deleted in one case: an account that is deleted
* or whose data is wiped (`offhost-purge.ts`), which needs DeleteObject in the
* grant. `AbortMultipartUpload` cleans up a run that failed partway rather
* than leaving billed, unlistable parts behind. See
* docs/ops/backup-restore.md.
*/
import { Buffer } from "node:buffer";
import {
@@ -108,8 +108,8 @@ export function loadOffhostConfig(): OffhostBackupConfig | null {
// `BACKUP_RETENTION_DAYS` is deliberately absent from this config. It used
// to be parsed and clamped here and then read by nobody, which read as an
// enforcer the worker is not: retention belongs to the bucket's lifecycle
// rule, and no consumer can ever appear here because DeleteObject is kept
// out of the worker's grant on purpose (see the header). The variable stays
// rule, which the worker leaves to the bucket rather than deleting by age
// (see the header). The variable stays
// documented and on the compose whitelist because it is the number the
// operator sets that rule to.
return {
@@ -964,7 +964,7 @@ export async function runOffhostBackup(
const purgedMeanwhile =
(await prisma.offhostPurgeRequest.count({
where: {
subjectId: { in: [user.id, "*"] },
subjectId: user.id,
requestedAt: { gte: accountStartedAt },
},
})) > 0 || (await prisma.user.count({ where: { id: user.id } })) === 0;
+9 -19
View File
@@ -17,9 +17,11 @@
* the job runs again every night and after every nightly upload, so a late
* copy the upload wrote for an account deleted while it ran is caught too.
*
* `subjectId` "*" is the admin wipe of every account: every account object
* goes. The health-check probes and anything else in the bucket that is not
* an account object is never touched.
* The admin wipe of every account writes no request, on purpose: it is the
* operator clearing their own host, the bucket is theirs, and its copies are
* the documented way back from a wipe pressed by mistake. The health-check
* probes and anything else in the bucket that is not an account object is
* never touched.
*/
import type { Job } from "pg-boss";
@@ -41,11 +43,8 @@ export const OFFHOST_PURGE_QUEUE = "offhost-backup-purge";
/** 03:40, after the nightly upload (02:30) has had its hour. */
export const OFFHOST_PURGE_CRON = "40 3 * * *";
/** Every subject, for the admin wipe of every account. */
export const PURGE_ALL_SUBJECTS = "*";
export type OffhostPurgeReason =
"account_deleted" | "data_wiped" | "managed_profile_deleted" | "admin_wipe";
"account_deleted" | "data_wiped" | "managed_profile_deleted";
/** An account object: `<date>/user-<id>.json.enc`. */
const ACCOUNT_OBJECT = /^\d{4}-\d{2}-\d{2}\/user-(.+)\.json\.enc$/;
@@ -163,14 +162,13 @@ export async function processOffhostPurges(
return report;
}
const everything = requests.some((r) => r.subjectId === PURGE_ALL_SUBJECTS);
const subjects = new Set(requests.map((r) => r.subjectId));
const bySubject = new Map<string, string[]>();
for (const { key } of objects) {
const match = ACCOUNT_OBJECT.exec(key);
if (!match) continue;
const subject = match[1];
if (!everything && !subjects.has(subject)) continue;
if (!subjects.has(subject)) continue;
const list = bySubject.get(subject) ?? [];
list.push(key);
bySubject.set(subject, list);
@@ -201,20 +199,12 @@ export async function processOffhostPurges(
}
for (const request of requests) {
const failure =
request.subjectId === PURGE_ALL_SUBJECTS
? failedSubjects.size > 0
? [...failedSubjects.values()][0]
: null
: failedSubjects.get(request.subjectId);
const failure = failedSubjects.get(request.subjectId);
if (failure) {
await markFailed([request.id], failure);
continue;
}
const deleted =
request.subjectId === PURGE_ALL_SUBJECTS
? [...bySubject.values()].reduce((sum, keys) => sum + keys.length, 0)
: (bySubject.get(request.subjectId)?.length ?? 0);
const deleted = bySubject.get(request.subjectId)?.length ?? 0;
await prisma.offhostPurgeRequest.delete({ where: { id: request.id } });
report.completed++;
// The account a deleted subject named is gone, and so is its audit