fix(backup): take off-host copies out with the account, and say which keys backups still need

Deleting an account, deleting a managed profile, "Delete all data" and the
admin wipe now write an off-host deletion request in the same transaction
as the deletion. A new job (offhost-backup-purge, kicked straight away and
run nightly as a backstop) lists the bucket once, deletes every
<date>/user-<id>.json.enc object of each pending account and removes the
request only when nothing of the account is left. A refused delete keeps
the request with the bucket's answer and is retried; the off-host card
shows how many deletions are outstanding and why. A copy the nightly run
was writing while the account was deleted is taken back after the upload.

The off-host card also reads the bucket's lifecycle rule and says plainly
when there is none (every nightly copy kept forever) or when the
credential may not read it.

Every stored backup now records the key ids its content was written
under, and every off-host upload records the same per key. The admin
encryption view lists, per key, the stored copies that need it with the
oldest date, and when the last off-host copy needing it expires. The
"safe to drop" badge now requires that no backup still needs a retired
key, instead of only that every row was rotated.

Migration 0358 adds data_backups.inner_key_ids, the key-use table and the
deletion requests.

src/lib/data-wipe/wipe-plan.ts

@@ -293,6 +293,10 @@ export const INSTANCE_SCOPED: Readonly<Record<string, string>> = {
   AppSettings: "instance-wide configuration singleton",
   RateLimit: "instance-wide rate-limit buckets, keyed by string, self-expiring",
   HostMetric: "instance-wide host telemetry, not attributable to an account",
+  OffhostBackupKeyUse:
+    "per encryption key, when the nightly off-host run last wrote an object needing it; says nothing about any account",
+  OffhostPurgeRequest:
+    "the deletions an account deletion or wipe asked of the off-host bucket; written by the wipe itself and removed once the bucket holds nothing of the account",
   CycleSymptomCategory:
     "seeded catalogue shared by every account; a user's custom symptoms live on CycleSymptom",
   IllnessSymptom: "seeded symptom catalogue shared by every account",

src/lib/export/store-backup-blob.ts

@@ -34,6 +34,7 @@ import {
   type BackupJsonProducer,
   type PackBackupOptions,
 } from "@/lib/export/backup-blob";
+import { createBackupKeyIdTextScanner } from "@/lib/export/backup-key-ids";
 import {
   BackupIntegrityError,
   newChunkStreamId,
@@ -112,6 +113,11 @@ export async function storeBackupBlob(
       await tx.dataBackupChunk.deleteMany({ where: { backupId: row.id } });
 
       const streamId = newChunkStreamId();
+      // The key ids the copy's content was written under, read from the JSON
+      // on its way into the pieces. Rotation re-seals the pieces but never
+      // what is inside them, so this is what tells an operator how long a
+      // retired key is still needed (`GET /api/admin/encryption/status`).
+      const keyScanner = createBackupKeyIdTextScanner();
       const { chunks, bytes } = await packBackupChunks(
         async (sealed, seq) => {
           await tx.dataBackupChunk.create({
@@ -120,7 +126,11 @@ export async function storeBackupBlob(
           });
         },
         streamId,
-        producer,
+        (write) =>
+          producer(async (chunk) => {
+            keyScanner.feed(chunk);
+            await write(chunk);
+          }),
         options,
       );
 
@@ -135,6 +145,8 @@ export async function storeBackupBlob(
           ...(input.ownerAfterRead ? { userId: input.ownerAfterRead() } : {}),
           chunkCount: chunks,
           chunkStreamId: streamId,
+          innerKeyIds: keyScanner.keyIds(),
+          innerKeyIdsRecorded: true,
           createdAt: new Date(),
         },
       });

src/lib/jobs/__tests__/offhost-backup.test.ts

@@ -32,6 +32,14 @@ import {
 const ENC_KEY =
   "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
 
+/** The purge ledger and key-use rows the run touches after each upload. */
+function purgeLedgerMocks() {
+  return {
+    offhostPurgeRequest: { count: vi.fn().mockResolvedValue(0) },
+    offhostBackupKeyUse: { upsert: vi.fn() },
+  };
+}
+
 describe("offhost-backup envelope", () => {
   it("encrypts and decrypts JSON round-trip", () => {
     const key = Buffer.from(ENC_KEY, "hex");
@@ -184,8 +192,10 @@ describe("runOffhostBackup", () => {
     s3.store.set("2020-01-01/user-old.json.enc", Buffer.from([0]));
 
     const prisma = {
+      ...purgeLedgerMocks(),
       offhostBackupState: { upsert: vi.fn() },
       user: {
+        count: vi.fn().mockResolvedValue(1),
         findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]),
       },
       measurement: { findMany: vi.fn().mockResolvedValue([]) },
@@ -248,11 +258,43 @@ describe("runOffhostBackup", () => {
     expect(first.create.lastSuccessAt).toBeInstanceOf(Date);
   });
 
+  it("takes back the copy of an account deleted while it was written", async () => {
+    const s3 = makeS3Mock();
+    const ledger = purgeLedgerMocks();
+    const prisma = {
+      ...ledger,
+      offhostBackupState: { upsert: vi.fn() },
+      user: {
+        // u1 is gone by the time its upload lands; u2 is not.
+        count: vi.fn(async ({ where }: { where: { id: string } }) =>
+          where.id === "u1" ? 0 : 1,
+        ),
+        findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]),
+      },
+    };
+    const report = await runOffhostBackup(
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      prisma as any,
+      s3,
+      new Date("2026-05-08T03:00:00Z"),
+    );
+    expect(s3.store.has("2026-05-08/user-u1.json.enc")).toBe(false);
+    expect(s3.store.has("2026-05-08/user-u2.json.enc")).toBe(true);
+    expect(s3.deleteObject).toHaveBeenCalledWith("2026-05-08/user-u1.json.enc");
+    expect(report.uploaded).toBe(1);
+    // No ledger row for an account that no longer exists.
+    expect(prisma.offhostBackupState.upsert).toHaveBeenCalledTimes(1);
+  });
+
   it("uploads the canonical builder output without reshaping it", async () => {
     const s3 = makeS3Mock();
     const prisma = {
+      ...purgeLedgerMocks(),
       offhostBackupState: { upsert: vi.fn() },
-      user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
+      user: {
+        count: vi.fn().mockResolvedValue(1),
+        findMany: vi.fn().mockResolvedValue([{ id: "u1" }]),
+      },
     };
     const canonicalPayload = {
       schemaVersion: BACKUP_SCHEMA_VERSION,
@@ -310,8 +352,10 @@ describe("runOffhostBackup", () => {
     const s3 = makeS3Mock();
     const upsert = vi.fn();
     const prisma = {
+      ...purgeLedgerMocks(),
       offhostBackupState: { upsert },
       user: {
+        count: vi.fn().mockResolvedValue(1),
         findMany: vi.fn().mockResolvedValue([{ id: "u1" }, { id: "u2" }]),
       },
     };
@@ -357,10 +401,14 @@ describe("runOffhostBackup", () => {
   it("still reports a landed object as uploaded when the ledger write fails", async () => {
     const s3 = makeS3Mock();
     const prisma = {
+      ...purgeLedgerMocks(),
       offhostBackupState: {
         upsert: vi.fn().mockRejectedValue(new Error("pool timeout")),
       },
-      user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
+      user: {
+        count: vi.fn().mockResolvedValue(1),
+        findMany: vi.fn().mockResolvedValue([{ id: "u1" }]),
+      },
     };
 
     const report = await runOffhostBackup(
@@ -382,8 +430,12 @@ describe("runOffhostBackup", () => {
   it("refuses an account whose object outgrows one multipart upload", async () => {
     const s3 = makeS3Mock();
     const prisma = {
+      ...purgeLedgerMocks(),
       offhostBackupState: { upsert: vi.fn() },
-      user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
+      user: {
+        count: vi.fn().mockResolvedValue(1),
+        findMany: vi.fn().mockResolvedValue([{ id: "u1" }]),
+      },
     };
     mocks.buildFullBackupPayload.mockResolvedValue({
       payload: {
@@ -419,8 +471,12 @@ describe("runOffhostBackup", () => {
   it("reports the largest object it wrote", async () => {
     const s3 = makeS3Mock();
     const prisma = {
+      ...purgeLedgerMocks(),
       offhostBackupState: { upsert: vi.fn() },
-      user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) },
+      user: {
+        count: vi.fn().mockResolvedValue(1),
+        findMany: vi.fn().mockResolvedValue([{ id: "u1" }]),
+      },
     };
     const report = await runOffhostBackup(
       prisma as never,

src/lib/jobs/__tests__/offhost-purge.test.ts

@@ -0,0 +1,205 @@
+/**
+ * Off-host copies leave the bucket with the account.
+ *
+ * The bucket is a map behind the same `S3Like` boundary the real client
+ * implements; the request ledger is an in-memory table. Mutations that must
+ * turn this red: drop the `deleteObject` call (the objects stay), remove the
+ * request on failure too (the refusal case loses the request), or match on the
+ * key prefix instead of the full account pattern (another account's objects,
+ * or the health-check probe, go too).
+ */
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() }));
+
+import { auditLog } from "@/lib/auth/audit";
+import {
+  processOffhostPurges,
+  PURGE_ALL_SUBJECTS,
+  requestOffhostPurge,
+} from "../offhost-purge";
+
+interface Row {
+  id: string;
+  subjectId: string;
+  reason: string;
+  requestedAt: Date;
+  attempts: number;
+  lastAttemptAt: Date | null;
+  lastFailure: string | null;
+}
+
+function makeLedger(existingUsers: string[] = []) {
+  const rows: Row[] = [];
+  let seq = 0;
+  const prisma = {
+    offhostPurgeRequest: {
+      create: vi.fn(
+        async ({ data }: { data: { subjectId: string; reason: string } }) => {
+          const row: Row = {
+            id: `r${++seq}`,
+            subjectId: data.subjectId,
+            reason: data.reason,
+            requestedAt: new Date(Date.now() + seq),
+            attempts: 0,
+            lastAttemptAt: null,
+            lastFailure: null,
+          };
+          rows.push(row);
+          return { id: row.id };
+        },
+      ),
+      findMany: vi.fn(async () =>
+        [...rows].sort((a, b) => +a.requestedAt - +b.requestedAt),
+      ),
+      delete: vi.fn(async ({ where }: { where: { id: string } }) => {
+        rows.splice(
+          rows.findIndex((r) => r.id === where.id),
+          1,
+        );
+      }),
+      updateMany: vi.fn(
+        async ({
+          where,
+          data,
+        }: {
+          where: { id: { in: string[] } };
+          data: { lastAttemptAt: Date; lastFailure: string };
+        }) => {
+          for (const row of rows) {
+            if (!where.id.in.includes(row.id)) continue;
+            row.attempts += 1;
+            row.lastAttemptAt = data.lastAttemptAt;
+            row.lastFailure = data.lastFailure;
+          }
+        },
+      ),
+    },
+    user: {
+      count: vi.fn(async ({ where }: { where: { id: string } }) =>
+        existingUsers.includes(where.id) ? 1 : 0,
+      ),
+    },
+  };
+  return { rows, prisma };
+}
+
+function makeBucket(keys: string[]) {
+  const store = new Set(keys);
+  return {
+    store,
+    putObject: vi.fn(),
+    putStream: vi.fn(),
+    getObject: vi.fn(),
+    headObject: vi.fn(),
+    listObjects: vi.fn(async (prefix: string) =>
+      [...store].filter((k) => k.startsWith(prefix)).map((key) => ({ key })),
+    ),
+    deleteObject: vi.fn(async (key: string) => {
+      store.delete(key);
+    }),
+  };
+}
+
+const BUCKET = [
+  "2026-09-01/user-gone.json.enc",
+  "2026-09-02/user-gone.json.enc",
+  "2026-09-02/user-gone2.json.enc",
+  "2026-09-02/user-stays.json.enc",
+  "_healthcheck/1.bin",
+];
+
+beforeEach(() => {
+  vi.unstubAllEnvs();
+  vi.clearAllMocks();
+  vi.stubEnv("BACKUP_S3_ENDPOINT", "https://r2.example");
+  vi.stubEnv("BACKUP_S3_BUCKET", "hl-backups");
+  vi.stubEnv("BACKUP_S3_ACCESS_KEY", "AKIA");
+  vi.stubEnv("BACKUP_S3_SECRET_KEY", "secret");
+  vi.stubEnv("BACKUP_ENCRYPTION_KEY", "ab".repeat(32));
+});
+
+describe("off-host purge", () => {
+  it("deletes every copy of the account, and nothing else", async () => {
+    const { rows, prisma } = makeLedger();
+    await requestOffhostPurge(prisma as never, "gone", "account_deleted");
+    const bucket = makeBucket(BUCKET);
+
+    const report = await processOffhostPurges(prisma as never, bucket);
+
+    expect([...bucket.store].sort()).toEqual([
+      "2026-09-02/user-gone2.json.enc",
+      "2026-09-02/user-stays.json.enc",
+      "_healthcheck/1.bin",
+    ]);
+    expect(report).toMatchObject({
+      completed: 1,
+      failed: 0,
+      objectsDeleted: 2,
+    });
+    expect(rows).toHaveLength(0);
+    // The account is gone, and so is its audit trail: the receipt carries the
+    // reason and the count, not the account.
+    expect(auditLog).toHaveBeenCalledWith(
+      "offhost.backup.purged",
+      expect.objectContaining({
+        userId: null,
+        details: expect.objectContaining({
+          reason: "account_deleted",
+          objectsDeleted: 2,
+        }),
+      }),
+    );
+  });
+
+  it("keeps the request, with the bucket's answer, when a delete is refused", async () => {
+    const { rows, prisma } = makeLedger(["gone"]);
+    await requestOffhostPurge(prisma as never, "gone", "data_wiped");
+    const bucket = makeBucket(BUCKET);
+    bucket.deleteObject.mockRejectedValueOnce(new Error("AccessDenied"));
+
+    const report = await processOffhostPurges(prisma as never, bucket);
+
+    expect(report).toMatchObject({ completed: 0, failed: 1 });
+    expect(rows).toHaveLength(1);
+    expect(rows[0]).toMatchObject({ attempts: 1, lastFailure: "AccessDenied" });
+
+    // The next run finishes it, and the account that still exists keeps the
+    // receipt.
+    const again = await processOffhostPurges(prisma as never, bucket);
+    expect(again.completed).toBe(1);
+    expect(rows).toHaveLength(0);
+    expect(auditLog).toHaveBeenCalledWith(
+      "offhost.backup.purged",
+      expect.objectContaining({ userId: "gone" }),
+    );
+  });
+
+  it("clears every account object for the admin wipe", async () => {
+    const { prisma } = makeLedger();
+    await requestOffhostPurge(
+      prisma as never,
+      PURGE_ALL_SUBJECTS,
+      "admin_wipe",
+    );
+    const bucket = makeBucket(BUCKET);
+    await processOffhostPurges(prisma as never, bucket);
+    expect([...bucket.store]).toEqual(["_healthcheck/1.bin"]);
+  });
+
+  it("records nothing on a host without off-host backup", async () => {
+    vi.stubEnv("BACKUP_S3_ENDPOINT", "");
+    const { rows, prisma } = makeLedger();
+    expect(
+      await requestOffhostPurge(prisma as never, "gone", "account_deleted"),
+    ).toBe(false);
+    expect(rows).toHaveLength(0);
+  });
+
+  it("does not list the bucket when nothing is pending", async () => {
+    const { prisma } = makeLedger();
+    const bucket = makeBucket(BUCKET);
+    await processOffhostPurges(prisma as never, bucket);
+    expect(bucket.listObjects).not.toHaveBeenCalled();
+  });
+});

src/lib/jobs/job-outcome.ts

@@ -144,6 +144,10 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
   "offhost_backup_oversized",
   "offhost_backup_total_users",
   "offhost_backup_uploaded",
+  "offhost_purge_completed",
+  "offhost_purge_failed",
+  "offhost_purge_objects_deleted",
+  "offhost_purge_pending",
   "outcome",
   "passes_failed",
   "per_sample_rows_deleted",

src/lib/jobs/offhost-backup.ts

@@ -27,6 +27,7 @@ import { Readable, Transform } from "node:stream";
 import { createGunzip, createGzip, gunzipSync, gzipSync } from "node:zlib";
 import type { PrismaClient } from "@/generated/prisma/client";
 import { createRawStreamEncryptor, decryptRawStream } from "@/lib/crypto";
+import { createBackupKeyIdTextScanner } from "@/lib/export/backup-key-ids";
 import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
 import { annotate, getEvent } from "@/lib/logging/context";
 
@@ -489,6 +490,24 @@ export interface S3Like {
     prefix: string,
   ): Promise<Array<{ key: string; lastModified?: Date }>>;
   deleteObject(key: string): Promise<void>;
+  /**
+   * The bucket's lifecycle rule, as far as the credential may read it.
+   * Optional so a test double that has no opinion leaves it out.
+   */
+  getLifecycle?(): Promise<OffhostLifecycle>;
+}
+
+/**
+ * Whether the bucket retires old copies by itself.
+ *
+ * `configured` names the shortest expiry of an enabled rule; `missing` means
+ * the bucket answered that it has no rule; `unknown` means it would not say
+ * (a credential without `s3:GetLifecycleConfiguration`, a target that does not
+ * implement the call, a network failure) and says nothing either way.
+ */
+export interface OffhostLifecycle {
+  state: "configured" | "missing" | "unknown";
+  expirationDays: number | null;
 }
 
 export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
@@ -601,9 +620,78 @@ export async function getS3Client(cfg: OffhostBackupConfig): Promise<S3Like> {
         new mod.DeleteObjectCommand({ Bucket: cfg.bucket, Key: key }),
       );
     },
+    getLifecycle: async () => {
+      try {
+        const out = await client.send(
+          new mod.GetBucketLifecycleConfigurationCommand({
+            Bucket: cfg.bucket,
+          }),
+          { abortSignal: AbortSignal.timeout(5_000) },
+        );
+        const days = (out.Rules ?? [])
+          .filter((rule) => rule.Status === "Enabled")
+          .map((rule) => rule.Expiration?.Days)
+          .filter((d): d is number => typeof d === "number" && d > 0);
+        return days.length > 0
+          ? { state: "configured", expirationDays: Math.min(...days) }
+          : { state: "missing", expirationDays: null };
+      } catch (err) {
+        const name = (err as { name?: string; Code?: string }).name;
+        const code = (err as { Code?: string }).Code;
+        if (
+          name === "NoSuchLifecycleConfiguration" ||
+          code === "NoSuchLifecycleConfiguration"
+        ) {
+          return { state: "missing", expirationDays: null };
+        }
+        return { state: "unknown", expirationDays: null };
+      }
+    },
   };
 }
 
+const LIFECYCLE_CACHE_MS = 10 * 60 * 1000;
+let lifecycleCache: { at: number; value: OffhostLifecycle } | null = null;
+
+/**
+ * The bucket's lifecycle rule for the admin card, cached for ten minutes so a
+ * page render does not become a bucket call. Never throws.
+ */
+export async function probeOffhostLifecycle(
+  s3Override?: S3Like,
+  now: number = Date.now(),
+): Promise<OffhostLifecycle> {
+  if (
+    !s3Override &&
+    lifecycleCache &&
+    now - lifecycleCache.at < LIFECYCLE_CACHE_MS
+  ) {
+    return lifecycleCache.value;
+  }
+  const cfg = loadOffhostConfigSafe();
+  if (!cfg) return { state: "unknown", expirationDays: null };
+  let value: OffhostLifecycle;
+  try {
+    const s3 = s3Override ?? (await getS3Client(cfg));
+    value = s3.getLifecycle
+      ? await s3.getLifecycle()
+      : { state: "unknown", expirationDays: null };
+  } catch {
+    value = { state: "unknown", expirationDays: null };
+  }
+  if (!s3Override) lifecycleCache = { at: now, value };
+  return value;
+}
+
+/** `loadOffhostConfig`, answering null for a malformed key instead of throwing. */
+function loadOffhostConfigSafe(): OffhostBackupConfig | null {
+  try {
+    return loadOffhostConfig();
+  } catch {
+    return null;
+  }
+}
+
 interface BackupRunReport {
   config: { endpoint: string; bucket: string; region: string };
   uploaded: number;
@@ -642,6 +730,30 @@ export interface RunOffhostBackupOptions extends UploadBackupOptions {
   shouldStop?: () => boolean;
 }
 
+/**
+ * Note, per key id, that an object needing it went into the bucket now. Never
+ * fails the account: the object is in the bucket whatever this row says.
+ */
+async function recordOffhostKeyUse(
+  prisma: PrismaClient,
+  keyIds: readonly string[],
+  at: Date,
+): Promise<void> {
+  for (const keyId of keyIds) {
+    try {
+      await prisma.offhostBackupKeyUse.upsert({
+        where: { keyId },
+        create: { keyId, firstWrittenAt: at, lastWrittenAt: at },
+        update: { lastWrittenAt: at },
+      });
+    } catch (err) {
+      getEvent()?.addWarning(
+        `offhost-backup key-use write failed: ${(err as Error).message?.slice(0, 200)}`,
+      );
+    }
+  }
+}
+
 export async function runOffhostBackup(
   prisma: PrismaClient,
   s3Override?: S3Like,
@@ -689,24 +801,54 @@ export async function runOffhostBackup(
       break;
     }
     let objectBytes: number | null = null;
+    const objectKey = `${dateKey}/user-${user.id}.json.enc`;
+    const accountStartedAt = new Date();
+    // The key ids the object's content needs, read as it is written: the
+    // admin encryption view uses them to say how long a retired key is still
+    // needed for what is in the bucket.
+    const keyScanner = createBackupKeyIdTextScanner();
     try {
       objectBytes = await uploadEncryptedBackup(
         s3,
-        `${dateKey}/user-${user.id}.json.enc`,
+        objectKey,
         cfg.encryptionKey,
         // The same writer the weekly in-database pass uses. The payload
         // builder was always shared; everything after it was not, which is why
         // this job kept dying on a record the weekly one had learned to
         // survive.
         (write) =>
-          streamFullBackupJson(prisma, user.id, write, {
-            purpose: "disaster-recovery",
-            exportedAt: now,
-          }),
+          streamFullBackupJson(
+            prisma,
+            user.id,
+            async (chunk) => {
+              keyScanner.feed(chunk);
+              await write(chunk);
+            },
+            {
+              purpose: "disaster-recovery",
+              exportedAt: now,
+            },
+          ),
         options,
       );
+      // The account was deleted or wiped while its copy was being written.
+      // The purge that request started may already have run, so the copy
+      // this run just put there is removed here rather than left behind.
+      const purgedMeanwhile =
+        (await prisma.offhostPurgeRequest.count({
+          where: {
+            subjectId: { in: [user.id, "*"] },
+            requestedAt: { gte: accountStartedAt },
+          },
+        })) > 0 || (await prisma.user.count({ where: { id: user.id } })) === 0;
+      if (purgedMeanwhile) {
+        await s3.deleteObject(objectKey);
+        objectBytes = null;
+        continue;
+      }
       largestObjectBytes = Math.max(largestObjectBytes, objectBytes);
       uploaded++;
+      await recordOffhostKeyUse(prisma, keyScanner.keyIds(), new Date());
     } catch (err) {
       failed++;
       if (err instanceof OffhostBackupTooLargeError) oversized++;

src/lib/jobs/offhost-purge.ts

@@ -0,0 +1,270 @@
+/**
+ * Off-host copies leave the bucket with the account.
+ *
+ * The nightly job writes `<YYYY-MM-DD>/user-<id>.json.enc` for every account,
+ * and until v1.39.3 nothing ever removed one: deleting an account, or wiping
+ * its data, took the rows in this database and left every nightly copy of the
+ * record in the operator's bucket for as long as its lifecycle rule kept them,
+ * or forever where no rule was set. A confirmation that says "permanently
+ * delete" has to mean the off-host copies too.
+ *
+ * How. The deletion writes an `OffhostPurgeRequest` in its own transaction, so
+ * the request exists exactly when the deletion committed. This job then lists
+ * the bucket once, deletes every object of every pending account, and removes
+ * the request when nothing of the account is left. A request the bucket
+ * refuses (a credential without DeleteObject, an outage) stays, counts its
+ * attempts and keeps the bucket's answer, which the admin off-host card shows;
+ * the job runs again every night and after every nightly upload, so a late
+ * copy the upload wrote for an account deleted while it ran is caught too.
+ *
+ * `subjectId` "*" is the admin wipe of every account: every account object
+ * goes. The health-check probes and anything else in the bucket that is not
+ * an account object is never touched.
+ */
+import type { Job } from "pg-boss";
+
+import type { Prisma, PrismaClient } from "@/generated/prisma/client";
+import { auditLog } from "@/lib/auth/audit";
+import { getGlobalBoss } from "@/lib/jobs/boss-instance";
+import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
+import {
+  getS3Client,
+  loadOffhostConfig,
+  offhostBackupConfigured,
+  type S3Like,
+} from "@/lib/jobs/offhost-backup";
+import { reportWorkerError } from "@/lib/jobs/report-worker-error";
+import { getWorkerPrisma } from "@/lib/jobs/reminder/shared";
+import { withBackgroundEvent } from "@/lib/logging/background";
+
+export const OFFHOST_PURGE_QUEUE = "offhost-backup-purge";
+/** 03:40, after the nightly upload (02:30) has had its hour. */
+export const OFFHOST_PURGE_CRON = "40 3 * * *";
+
+/** Every subject, for the admin wipe of every account. */
+export const PURGE_ALL_SUBJECTS = "*";
+
+export type OffhostPurgeReason =
+  "account_deleted" | "data_wiped" | "managed_profile_deleted" | "admin_wipe";
+
+/** An account object: `<date>/user-<id>.json.enc`. */
+const ACCOUNT_OBJECT = /^\d{4}-\d{2}-\d{2}\/user-(.+)\.json\.enc$/;
+
+/** Requests one run works through; the next run takes the rest. */
+const REQUESTS_PER_RUN = 200;
+
+/** Deletes one run may issue, so a huge bucket cannot run the job forever. */
+const MAX_DELETES_PER_RUN = 50_000;
+
+type Db = Pick<Prisma.TransactionClient, "offhostPurgeRequest">;
+
+/**
+ * Record that `subjectId`'s off-host copies have to go. Call it with the
+ * transaction that deletes the account or wipes its data, so the request and
+ * the deletion commit together. A host without off-host backup configured has
+ * nothing in any bucket this could reach, and records nothing.
+ */
+export async function requestOffhostPurge(
+  db: Db,
+  subjectId: string,
+  reason: OffhostPurgeReason,
+): Promise<boolean> {
+  if (!offhostBackupConfigured()) return false;
+  await db.offhostPurgeRequest.create({
+    data: { subjectId, reason },
+    select: { id: true },
+  });
+  return true;
+}
+
+/**
+ * Wake the purge job now rather than at its nightly slot. Best effort: the
+ * request row is what guarantees the purge, and the cron picks it up if the
+ * queue is not reachable from here.
+ */
+export async function kickOffhostPurge(): Promise<void> {
+  try {
+    await getGlobalBoss()?.send(
+      OFFHOST_PURGE_QUEUE,
+      {},
+      {
+        singletonKey: OFFHOST_PURGE_QUEUE,
+        retryLimit: 3,
+        retryDelay: 300,
+      },
+    );
+  } catch {
+    // The cron is the backstop.
+  }
+}
+
+export interface OffhostPurgeReport {
+  pending: number;
+  completed: number;
+  failed: number;
+  objectsDeleted: number;
+}
+
+function trimError(err: unknown): string {
+  return (err instanceof Error ? err.message : String(err)).slice(0, 300);
+}
+
+/**
+ * Work through the pending requests: list the bucket once, delete every
+ * object of every pending subject, remove each request whose objects are all
+ * gone. Answers what it did.
+ */
+export async function processOffhostPurges(
+  prisma: PrismaClient,
+  s3Override?: S3Like,
+): Promise<OffhostPurgeReport> {
+  const requests = await prisma.offhostPurgeRequest.findMany({
+    orderBy: { requestedAt: "asc" },
+    take: REQUESTS_PER_RUN,
+  });
+  const report: OffhostPurgeReport = {
+    pending: requests.length,
+    completed: 0,
+    failed: 0,
+    objectsDeleted: 0,
+  };
+  if (requests.length === 0) return report;
+
+  const cfg = loadOffhostConfig();
+  if (!cfg) {
+    // Configuration removed after the requests were written: leave them, an
+    // operator who puts it back gets the purge.
+    return report;
+  }
+
+  const now = new Date();
+  const markFailed = async (ids: string[], err: unknown) => {
+    await prisma.offhostPurgeRequest.updateMany({
+      where: { id: { in: ids } },
+      data: {
+        attempts: { increment: 1 },
+        lastAttemptAt: now,
+        lastFailure: trimError(err),
+      },
+    });
+    report.failed += ids.length;
+  };
+
+  let objects: Array<{ key: string }>;
+  let s3: S3Like;
+  try {
+    s3 = s3Override ?? (await getS3Client(cfg));
+    objects = await s3.listObjects("");
+  } catch (err) {
+    await markFailed(
+      requests.map((r) => r.id),
+      err,
+    );
+    return report;
+  }
+
+  const everything = requests.some((r) => r.subjectId === PURGE_ALL_SUBJECTS);
+  const subjects = new Set(requests.map((r) => r.subjectId));
+  const bySubject = new Map<string, string[]>();
+  for (const { key } of objects) {
+    const match = ACCOUNT_OBJECT.exec(key);
+    if (!match) continue;
+    const subject = match[1];
+    if (!everything && !subjects.has(subject)) continue;
+    const list = bySubject.get(subject) ?? [];
+    list.push(key);
+    bySubject.set(subject, list);
+  }
+
+  // One subject at a time, so a refusal part-way fails only the subjects it
+  // reached, and a request is removed only when its own objects are gone.
+  const failedSubjects = new Map<string, unknown>();
+  let deletes = 0;
+  for (const [subject, keys] of bySubject) {
+    for (const key of keys) {
+      if (deletes >= MAX_DELETES_PER_RUN) {
+        failedSubjects.set(
+          subject,
+          new Error("Deletion budget for one run reached; continues next run"),
+        );
+        break;
+      }
+      try {
+        await s3.deleteObject(key);
+        deletes++;
+        report.objectsDeleted++;
+      } catch (err) {
+        failedSubjects.set(subject, err);
+        break;
+      }
+    }
+  }
+
+  for (const request of requests) {
+    const failure =
+      request.subjectId === PURGE_ALL_SUBJECTS
+        ? failedSubjects.size > 0
+          ? [...failedSubjects.values()][0]
+          : null
+        : failedSubjects.get(request.subjectId);
+    if (failure) {
+      await markFailed([request.id], failure);
+      continue;
+    }
+    const deleted =
+      request.subjectId === PURGE_ALL_SUBJECTS
+        ? [...bySubject.values()].reduce((sum, keys) => sum + keys.length, 0)
+        : (bySubject.get(request.subjectId)?.length ?? 0);
+    await prisma.offhostPurgeRequest.delete({ where: { id: request.id } });
+    report.completed++;
+    // The account a deleted subject named is gone, and so is its audit
+    // history; the receipt names the reason and the count, not the account.
+    // A wiped account still exists and keeps the receipt as its own.
+    const accountRemains =
+      request.reason === "data_wiped" &&
+      (await prisma.user.count({ where: { id: request.subjectId } })) > 0;
+    await auditLog("offhost.backup.purged", {
+      userId: accountRemains ? request.subjectId : null,
+      actorUserId: null,
+      details: {
+        reason: request.reason,
+        objectsDeleted: deleted,
+        attempts: request.attempts + 1,
+      },
+    });
+  }
+  return report;
+}
+
+export async function handleOffhostPurge(
+  jobs: Job<object>[],
+): Promise<JobOutcome> {
+  void jobs;
+  return withBackgroundEvent("job.offhost_purge", async (evt) => {
+    try {
+      const report = await processOffhostPurges(getWorkerPrisma());
+      evt.addMeta("offhost_purge_pending", report.pending);
+      evt.addMeta("offhost_purge_completed", report.completed);
+      evt.addMeta("offhost_purge_failed", report.failed);
+      evt.addMeta("offhost_purge_objects_deleted", report.objectsDeleted);
+      if (report.failed > 0) {
+        await reportWorkerError(
+          OFFHOST_PURGE_QUEUE,
+          new Error(
+            `${report.failed} off-host deletion request(s) could not be completed; they stay pending and are retried nightly. Check that the bucket credential allows DeleteObject.`,
+          ),
+        );
+      }
+      return jobDone({
+        offhost_purge_pending: report.pending,
+        offhost_purge_completed: report.completed,
+        offhost_purge_failed: report.failed,
+        offhost_purge_objects_deleted: report.objectsDeleted,
+      });
+    } catch (err) {
+      evt.addWarning(`offhost-purge failed: ${err}`);
+      await reportWorkerError(OFFHOST_PURGE_QUEUE, err);
+      return jobFailed("offhost purge failed", err);
+    }
+  });
+}

src/lib/jobs/reminder/register-maintenance.ts

@@ -67,6 +67,11 @@ import {
   enqueueBootTimeIntakeSlotDedup,
   type IntakeSlotDedupPayload,
 } from "@/lib/medications/intake-slot-dedup";
+import {
+  handleOffhostPurge,
+  OFFHOST_PURGE_CRON,
+  OFFHOST_PURGE_QUEUE,
+} from "@/lib/jobs/offhost-purge";
 import {
   handleRestoreDrill,
   RESTORE_DRILL_CRON,
@@ -353,6 +358,10 @@ const allQueues = [
   STEP_UP_ELEVATION_CLEANUP_QUEUE,
   OFFHOST_BACKUP_QUEUE,
   RESTORE_DRILL_QUEUE,
+  // v1.39.3 — deleting an account or wiping its data takes its off-host
+  // copies out of the bucket. Without this entry the requests would be
+  // written and never worked.
+  OFFHOST_PURGE_QUEUE,
   // v1.39.1 — an admin's restore of a stored backup, run off the request, and
   // the boot sweep that re-queues one a stopped worker left running. Without
   // this entry every restore request would queue a job nobody works.
@@ -505,6 +514,9 @@ const schedules: ScheduleEntry[] = [
   ],
   [OFFHOST_BACKUP_QUEUE, OFFHOST_BACKUP_CRON],
   [RESTORE_DRILL_QUEUE, RESTORE_DRILL_CRON],
+  // Nightly backstop for the purge the deletion kicks straight away: a
+  // request the bucket refused, or one written while the queue was down.
+  [OFFHOST_PURGE_QUEUE, OFFHOST_PURGE_CRON, cronIsTheRetry],
   [HOST_METRIC_QUEUE, HOST_METRIC_CRON, cronIsTheRetry],
   [FEEDBACK_AGGREGATOR_QUEUE, FEEDBACK_AGGREGATOR_CRON, cronIsTheRetry],
   // v1.4.37 — hourly geo backfill. The helper is idempotent + capped
@@ -747,6 +759,12 @@ export async function registerMaintenanceQueues(
     { localConcurrency: 1 },
     handleRestoreDrill,
   );
+  await createAndWork(
+    boss,
+    OFFHOST_PURGE_QUEUE,
+    { localConcurrency: 1 },
+    handleOffhostPurge,
+  );
   // One restore at a time per worker: each holds a transaction over a whole
   // account and a batch of readings in memory, and two at once on a small host
   // is how a restore that fits alone would run out of heap.

src/lib/jobs/subsystem-surface.ts

@@ -101,6 +101,7 @@ const subsystemSurface = {
   "step-up-elevation-cleanup": { audience: "system" },
   "data-backup-offhost": { audience: "system" },
   "data-restore-drill": { audience: "system" },
+  "offhost-backup-purge": { audience: "system" },
   "backup-restore": { audience: "system" },
   "host-metric-sample": { audience: "system" },
   "feedback-aggregator": { audience: "system" },

src/lib/managed-profiles/lifecycle.ts

@@ -1,3 +1,7 @@
+import {
+  kickOffhostPurge,
+  requestOffhostPurge,
+} from "@/lib/jobs/offhost-purge";
 import { prisma } from "@/lib/db";
 import { auditLog } from "@/lib/auth/audit";
 import type { Prisma } from "@/generated/prisma/client";
@@ -209,9 +213,11 @@ export async function deleteManagedProfile(input: {
         details: { profileId: profile.id },
         client: tx,
       });
+      await requestOffhostPurge(tx, profile.id, "managed_profile_deleted");
       await tx.user.delete({ where: { id: profile.id } });
     });
   });
+  await kickOffhostPurge();
 }
 
 /**

src/lib/openapi/error-codes.ts

@@ -66,6 +66,8 @@ export const ERROR_CODE_CATALOGUE: Readonly<Record<string, readonly string[]>> =
       "auth.token.invalid",
     ],
     backup: [
+      "backup.foreign_reference",
+      "backup.key.missing",
       "backup.payload.undecryptable",
       "backup.restore.active",
       "backup.section.missing",