Merge branch 'fix/backup-chunked-storage' into release/v1.39.2

src/__tests__/helpers/legacy-backup-blob.ts

@@ -1,9 +1,9 @@
 /**
- * The single-value backup form v1.39.1 wrote into `data_backups.data`: the
+ * The single-value backup form v1.38.6 to v1.39.1 wrote into `data_backups.data`: the
  * JSON gzipped, then sealed as one AES-256-GCM stream (`~hlgcm1.…`).
  *
  * Nothing in the app writes it any more; copies stored in pieces replaced it
- * in v1.39.2. Copies in this form still exist on every host that ran v1.39.x,
+ * in v1.39.2. Copies in this form still exist on every host that ran those releases,
  * and they still have to restore, so the tests build them with this.
  */
 import { Buffer } from "node:buffer";

src/app/api/admin/backups/[id]/download/route.ts

@@ -22,11 +22,9 @@ import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
 import { apiError, getClientIp } from "@/lib/api-response";
 import { auditLog } from "@/lib/auth/audit";
 import {
-  BACKUP_UNDECRYPTABLE_CODE,
-  BACKUP_UNDECRYPTABLE_ERROR,
-} from "@/lib/export/backup-blob";
-import {
+  isStoredBackupReadError,
   openStoredBackup,
+  storedBackupRefusal,
   STORED_BACKUP_SELECT,
 } from "@/lib/export/stored-backup";
 import {
@@ -89,8 +87,9 @@ export const GET = apiHandler(
           reason: err instanceof Error ? err.message : "decrypt_failed",
         },
       });
-      return apiError(BACKUP_UNDECRYPTABLE_ERROR, 422, {
-        errorCode: BACKUP_UNDECRYPTABLE_CODE,
+      const refusal = storedBackupRefusal(err);
+      return apiError(refusal.message, refusal.status, {
+        errorCode: refusal.code,
       });
     }
 
@@ -103,16 +102,25 @@ export const GET = apiHandler(
       streamed = await readStreamedBackup(source);
       parseBackupPayload(streamed.raw);
     } catch (err) {
+      const readFailure = isStoredBackupReadError(err);
       await auditLog("admin.backups.download.failed", {
         userId: admin.id,
         ipAddress: getClientIp(request),
         details: {
           backupId: id,
           ownerId: backup.userId,
-          reason: "schema_invalid",
+          reason: readFailure ? "read_failed" : "schema_invalid",
           message: err instanceof Error ? err.message : String(err),
         },
       });
+      if (readFailure) {
+        // The copy changed between opening and reading (replaced by the
+        // weekly run, or altered): the same answer the open gives.
+        const refusal = storedBackupRefusal(err);
+        return apiError(refusal.message, refusal.status, {
+          errorCode: refusal.code,
+        });
+      }
       return apiError("Backup payload failed schema validation", 500);
     }
 

src/app/api/admin/backups/[id]/restore/route.ts

@@ -24,12 +24,9 @@ import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
 import { apiError, apiSuccess, getClientIp } from "@/lib/api-response";
 import { auditLog } from "@/lib/auth/audit";
 import { prisma } from "@/lib/db";
-import {
-  BACKUP_UNDECRYPTABLE_CODE,
-  BACKUP_UNDECRYPTABLE_ERROR,
-} from "@/lib/export/backup-blob";
 import {
   openStoredBackup,
+  storedBackupRefusal,
   STORED_BACKUP_SELECT,
   storedBackupIdentity,
 } from "@/lib/export/stored-backup";
@@ -114,8 +111,9 @@ const handler = apiHandler(
           reason: err instanceof Error ? err.message : "decrypt_failed",
         },
       });
-      return apiError(BACKUP_UNDECRYPTABLE_ERROR, 422, {
-        errorCode: BACKUP_UNDECRYPTABLE_CODE,
+      const refusal = storedBackupRefusal(err);
+      return apiError(refusal.message, refusal.status, {
+        errorCode: refusal.code,
       });
     }
 

src/app/api/admin/backups/[id]/summary/route.ts

@@ -17,11 +17,9 @@ import { prisma } from "@/lib/db";
 import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
 import { apiError, apiSuccess } from "@/lib/api-response";
 import {
-  BACKUP_UNDECRYPTABLE_CODE,
-  BACKUP_UNDECRYPTABLE_ERROR,
-} from "@/lib/export/backup-blob";
-import {
+  isStoredBackupReadError,
   openStoredBackup,
+  storedBackupRefusal,
   STORED_BACKUP_SELECT,
 } from "@/lib/export/stored-backup";
 import {
@@ -62,12 +60,13 @@ export const GET = apiHandler(
     let source: BackupSource;
     try {
       source = await openStoredBackup(prisma, backup);
-    } catch {
+    } catch (err) {
       // Same refusal the restore and the download give, for the same reason:
       // a copy this instance cannot open is bad stored input, and the preview
       // is the first place an operator meets it.
-      return apiError(BACKUP_UNDECRYPTABLE_ERROR, 422, {
-        errorCode: BACKUP_UNDECRYPTABLE_CODE,
+      const refusal = storedBackupRefusal(err);
+      return apiError(refusal.message, refusal.status, {
+        errorCode: refusal.code,
       });
     }
 
@@ -77,7 +76,13 @@ export const GET = apiHandler(
       const streamed = await readStreamedBackup(source);
       measurementCount = streamed.measurementCount;
       payload = parseBackupPayload(streamed.raw);
-    } catch {
+    } catch (err) {
+      if (isStoredBackupReadError(err)) {
+        const refusal = storedBackupRefusal(err);
+        return apiError(refusal.message, refusal.status, {
+          errorCode: refusal.code,
+        });
+      }
       return apiError("Backup payload failed schema validation", 422);
     }
 

src/lib/crypto.ts

@@ -283,8 +283,14 @@ export function reencryptToActive(encoded: string): string {
 /** Version byte identifying the binary2 layout. */
 const BYTES_CODEC_VERSION = 0x02;
 
-/** Encrypt raw bytes with the active key into the binary2 layout. */
-export function encryptBytes(plaintext: Buffer): Buffer {
+/**
+ * Encrypt raw bytes with the active key into the binary2 layout.
+ *
+ * `aad` binds the value to a purpose: it is authenticated but not stored, so
+ * the value opens only through `decryptBytes` with the same label. Omitted,
+ * the layout and the behaviour are exactly what they always were.
+ */
+export function encryptBytes(plaintext: Buffer, aad?: string): Buffer {
   const { id, key } = getActiveKey();
   const keyId = Buffer.from(id, "ascii");
   if (keyId.byteLength < 1 || keyId.byteLength > 32) {
@@ -293,6 +299,7 @@ export function encryptBytes(plaintext: Buffer): Buffer {
   }
   const iv = randomBytes(IV_LENGTH);
   const cipher = createCipheriv(ALGORITHM, key, iv);
+  if (aad !== undefined) cipher.setAAD(Buffer.from(aad, "utf8"));
   const ct = Buffer.concat([cipher.update(plaintext), cipher.final()]);
   const tag = cipher.getAuthTag();
   const header = Buffer.from([BYTES_CODEC_VERSION, keyId.byteLength]);
@@ -337,7 +344,7 @@ function parseBytesHeader(payload: Buffer): {
  * version, a malformed header, or an unconfigured key id throws — the caller
  * must treat a throw as "cannot serve", never fall back to the ciphertext.
  */
-export function decryptBytes(payload: Buffer): Buffer {
+export function decryptBytes(payload: Buffer, aad?: string): Buffer {
   const { keyId, iv, tag, ct } = parseBytesHeader(payload);
   const key = getKeyById(keyId);
   if (!key) {
@@ -348,6 +355,7 @@ export function decryptBytes(payload: Buffer): Buffer {
   }
   const dec = createDecipheriv(ALGORITHM, key, iv);
   dec.setAuthTag(tag);
+  if (aad !== undefined) dec.setAAD(Buffer.from(aad, "utf8"));
   return Buffer.concat([dec.update(ct), dec.final()]);
 }
 
@@ -360,9 +368,12 @@ export function extractKeyIdFromBytes(payload: Buffer): string | null {
   }
 }
 
-/** Re-encrypt a binary2 payload with the active key. Used by rotation. */
-export function reencryptBytesToActive(payload: Buffer): Buffer {
-  return encryptBytes(decryptBytes(payload));
+/**
+ * Re-encrypt a binary2 payload with the active key, under the same `aad`
+ * label it was sealed with. Used by rotation.
+ */
+export function reencryptBytesToActive(payload: Buffer, aad?: string): Buffer {
+  return encryptBytes(decryptBytes(payload, aad), aad);
 }
 
 // ─── HKDF subkey derivation ──────────────────────────────────────────────────
@@ -578,15 +589,56 @@ export function decryptStream(stored: string): Buffer {
   return decryptRawStream(Buffer.from(rest.slice(dot + 1), "base64"), key);
 }
 
+/** Incremental reader of a streamed ciphertext's body. */
+export interface StreamDecryptor {
+  /** Characters of the stored value taken by the header, before the body. */
+  readonly headerLength: number;
+  /** Decrypt one piece of ciphertext. The output is NOT yet authenticated. */
+  update(ciphertext: Buffer): Buffer;
+  /** Check the tag over everything passed to `update`; throws when it fails. */
+  final(tag: Buffer): Buffer;
+}
+
 /**
- * Re-seal a streamed ciphertext under the active key, as a streamed
- * ciphertext. Used by rotation for the backups v1.39.1 stored in this form;
- * the plaintext is the backup's gzip bytes and is never inspected.
+ * Open a streamed ciphertext for reading a piece at a time, from the first
+ * characters of the stored value (at least the header).
+ *
+ * Unlike `decryptStream`, this releases plaintext before the tag has been
+ * checked, so it is only for a caller that writes what it reads somewhere it
+ * can take back: the conversion of a stored backup into pieces does it inside
+ * one transaction, which rolls back when `final` throws.
  */
-export function reencryptStreamToActive(stored: string): string {
-  const plaintext = decryptStream(stored);
-  const encryptor = createStreamEncryptor();
-  return `${encryptor.header}${encryptor.update(plaintext)}${encryptor.final()}`;
+export function openStreamDecryptor(head: string): StreamDecryptor {
+  if (!isStreamCiphertext(head)) {
+    throw new Error("Not a streamed ciphertext");
+  }
+  const rest = head.slice(STREAM_CODEC_PREFIX.length);
+  const dot = rest.indexOf(".");
+  const keyId = dot > 0 ? rest.slice(0, dot) : "";
+  if (!/^[A-Za-z0-9_-]{1,32}$/.test(keyId)) {
+    throw new Error("Streamed ciphertext carries a malformed key id");
+  }
+  // The 12-byte IV is exactly sixteen base64 characters.
+  const iv = Buffer.from(rest.slice(dot + 1, dot + 17), "base64");
+  if (iv.byteLength !== IV_LENGTH) {
+    throw new Error("Streamed ciphertext has a malformed header");
+  }
+  const key = getKeyById(keyId);
+  if (!key) {
+    throw new Error(
+      `Encryption key id '${keyId}' is not configured. Add it to ` +
+        `ENCRYPTION_KEYS before decrypting rows written under that key.`,
+    );
+  }
+  const dec = createDecipheriv(ALGORITHM, key, iv);
+  return {
+    headerLength: STREAM_CODEC_PREFIX.length + dot + 17,
+    update: (ciphertext) => dec.update(ciphertext),
+    final: (tag) => {
+      dec.setAuthTag(tag);
+      return dec.final();
+    },
+  };
 }
 
 /** The key id a streamed ciphertext was written under, or null when unparsable. */

src/lib/crypto/__tests__/encryption-corpus.test.ts

@@ -52,7 +52,7 @@ function makeClient(seed: Record<string, Row[]>): {
         let rows = [...store[model]].sort((a, b) =>
           String(a.id).localeCompare(String(b.id)),
         );
-        const after = where?.id?.gt;
+        const after = (where?.id as { gt?: string } | undefined)?.gt;
         if (after !== undefined)
           rows = rows.filter((r) => String(r.id) > after);
         if (take !== undefined) rows = rows.slice(0, take);
@@ -256,4 +256,30 @@ describe("encryption-corpus scan + rotate", () => {
       expect(take!).toBeLessThanOrEqual(5_000);
     }
   });
+
+  it("treats a row deleted between the read and the write as gone, not as an error", async () => {
+    // A weekly backup replaces its pieces while a rotation walks them: the
+    // piece the walk read no longer exists when it writes it back.
+    const { client } = makeClient({
+      NotificationChannel: [
+        { id: "c1", config: activeUnder("v1", "a") },
+        { id: "c2", config: activeUnder("v1", "b") },
+      ],
+    });
+    const delegate = client.notificationChannel!;
+    const update = delegate.update;
+    delegate.update = async (args) => {
+      if (args.where.id === "c1") {
+        throw Object.assign(new Error("Record to update not found."), {
+          code: "P2025",
+        });
+      }
+      return update(args);
+    };
+    const rotation = await rotateCorpus(client);
+    const result = rotation.results.find(
+      (r) => r.model === "NotificationChannel",
+    )!;
+    expect(result).toMatchObject({ rotated: 1, errors: 0 });
+  });
 });

src/lib/crypto/encrypted-columns.ts

@@ -39,6 +39,13 @@
 
 export type EncryptedColumnKind = "string" | "bytes";
 
+/**
+ * The associated-data label a stored backup's pieces are sealed with. The same
+ * key seals other binary values (a stored document, for one); the label keeps
+ * a value sealed for another purpose from ever opening as a backup piece.
+ */
+export const BACKUP_CHUNK_AAD = "healthlog/data-backup-chunk/v1";
+
 export interface EncryptedColumn {
   /** Prisma model name (PascalCase, as declared in schema.prisma). */
   readonly model: string;
@@ -60,6 +67,19 @@ export interface EncryptedColumn {
    * layout. Like `codecField`, it implies batching.
    */
   readonly codec?: "binary2";
+  /**
+   * The associated-data label every value of the column is sealed with
+   * (`encryptBytes(value, aad)`). Rotation re-seals under the same label.
+   */
+  readonly aad?: string;
+  /**
+   * The column holds single-value backups, which rotation does not re-seal in
+   * place but converts into pieces in `DataBackupChunk` under the active key,
+   * reading the value a slice at a time. Re-sealing in place needs the whole
+   * value in memory several times over, and one value can be a hundred
+   * megabytes.
+   */
+  readonly convertsToPieces?: true;
   /**
    * Walk this column in bounded id-cursor batches instead of one `findMany`.
    * Set it on any column whose rows are blobs rather than short strings — a
@@ -387,9 +407,15 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [
   //
   // Rotation re-encrypts the ciphertext WITHOUT touching the plaintext, so it
   // is blind to the envelope inside: a row is the plain backup JSON, the
-  // `HLZ1:`-prefixed gzip form, or the single `~hlgcm1.` stream v1.39.1 wrote
+  // `HLZ1:`-prefixed gzip form, or the single `~hlgcm1.` stream v1.38.6 to v1.39.1 wrote
   // (re-sealed as a stream). Batched, because a single row is megabytes.
-  { model: "DataBackup", field: "data", kind: "string", batched: true },
+  {
+    model: "DataBackup",
+    field: "data",
+    kind: "string",
+    batched: true,
+    convertsToPieces: true,
+  },
 
   // ───── Whole-account backup, in pieces (Bytes, binary2, batched) ─────
   // How every backup is stored from v1.39.2: ordered pieces of about a
@@ -402,6 +428,7 @@ export const ENCRYPTED_COLUMNS: readonly EncryptedColumn[] = [
     field: "data",
     kind: "bytes",
     codec: "binary2",
+    aad: BACKUP_CHUNK_AAD,
   },
 
   // ───── Idempotent-replay response cache (String, disposable) ─────

src/lib/crypto/encryption-corpus.ts

@@ -39,8 +39,12 @@ import {
   getActiveKeyId,
   isStreamCiphertext,
   reencryptBytesToActive,
-  reencryptStreamToActive,
 } from "@/lib/crypto";
+import type { PrismaClient } from "@/generated/prisma/client";
+import {
+  convertSingleValueBackup,
+  singleValueBackupHead,
+} from "@/lib/export/store-backup-blob";
 import {
   ENCRYPTED_COLUMNS,
   type EncryptedColumn,
@@ -72,7 +76,7 @@ interface ColumnDelegate {
     select: Record<string, true>;
     orderBy?: Record<string, "asc" | "desc">;
     take?: number;
-    where?: Record<string, { gt: string }>;
+    where?: Record<string, unknown>;
   }) => Promise<Array<Record<string, unknown>>>;
   update: (args: {
     where: Record<string, string>;
@@ -139,10 +143,14 @@ function blobKeyId(value: Uint8Array, codec: string): string | null {
 }
 
 /** Re-encrypt one codec-dispatched blob under its OWN codec (never converts). */
-function reencryptBlob(value: Uint8Array, codec: string): Uint8Array {
+function reencryptBlob(
+  value: Uint8Array,
+  codec: string,
+  aad: string | undefined,
+): Uint8Array {
   const buf = Buffer.from(value);
   if (codec === "binary2") {
-    const rotated = reencryptBytesToActive(buf);
+    const rotated = reencryptBytesToActive(buf, aad);
     const next = new Uint8Array(new ArrayBuffer(rotated.byteLength));
     next.set(rotated);
     return next;
@@ -232,17 +240,44 @@ function walkedKeyId(
   if (codec !== null) return blobKeyId(value as Uint8Array, codec);
   const ciphertext = toCiphertext(value, kind);
   if (ciphertext == null) return null;
-  // The single-stream backup form v1.39.1 wrote carries its key id behind a
+  // The single-stream backup form v1.38.6 to v1.39.1 wrote carries its key id behind a
   // `~hlgcm1.` marker, where the string codec's parser does not look.
   if (isStreamCiphertext(ciphertext)) return extractStreamKeyId(ciphertext);
   return extractKeyId(ciphertext);
 }
 
-/** Re-seal a string-codec value under the active key, keeping its form. */
-function reencryptString(ciphertext: string): string {
-  return isStreamCiphertext(ciphertext)
-    ? reencryptStreamToActive(ciphertext)
-    : encrypt(decrypt(ciphertext));
+/**
+ * The ids of the rows of a `convertsToPieces` column that still hold a single
+ * value, a page of ids at a time. Only the id is read: the value can be a
+ * hundred megabytes.
+ */
+async function* singleValueIds(
+  delegate: ColumnDelegate,
+): AsyncGenerator<string> {
+  let cursor: string | null = null;
+  for (;;) {
+    const rows = await delegate.findMany({
+      select: { id: true },
+      orderBy: { id: "asc" },
+      take: ROW_ROTATION_BATCH_SIZE,
+      where: {
+        data: { not: null },
+        ...(cursor ? { id: { gt: cursor } } : {}),
+      },
+    });
+    for (const row of rows) yield row.id as string;
+    if (rows.length < ROW_ROTATION_BATCH_SIZE) return;
+    cursor = rows[rows.length - 1]!.id as string;
+  }
+}
+
+/** True for a Prisma "record to update not found" (P2025). */
+function isRowGone(err: unknown): boolean {
+  return (
+    typeof err === "object" &&
+    err !== null &&
+    (err as { code?: unknown }).code === "P2025"
+  );
 }
 
 export interface ColumnScan {
@@ -266,6 +301,29 @@ export async function scanColumn(
   const byKeyId: Record<string, number> = {};
   let total = 0;
 
+  if (col.convertsToPieces) {
+    // One value can be a hundred megabytes: read only the first characters,
+    // which carry the key id.
+    for await (const id of singleValueIds(delegate)) {
+      const head = await singleValueBackupHead(
+        client as unknown as PrismaClient,
+        id,
+      );
+      if (head === null) continue;
+      total += 1;
+      const key = walkedKeyId(head, null, col.kind) ?? LEGACY_BUCKET;
+      byKeyId[key] = (byKeyId[key] ?? 0) + 1;
+    }
+    return {
+      model: col.model,
+      field: col.field,
+      kind: col.kind,
+      total,
+      byKeyId,
+      legacy: byKeyId[LEGACY_BUCKET] ?? 0,
+    };
+  }
+
   // Bounded batches for every column, per-row codec where the column has one.
   await walkColumn(delegate, col, ({ value, codec }) => {
     total += 1;
@@ -367,6 +425,28 @@ export async function rotateColumn(
     }
   };
 
+  if (col.convertsToPieces) {
+    // Converted into pieces under the active key, one row at a time and a
+    // slice of the value at a time, whatever key it is under now: re-sealing
+    // it in place would hold the whole value several times over
+    // (`convertSingleValueBackup`). A row converted, replaced or deleted since
+    // it was listed is gone, not an error.
+    for await (const id of singleValueIds(delegate)) {
+      result.scanned += 1;
+      try {
+        const outcome = await convertSingleValueBackup(
+          client as unknown as PrismaClient,
+          id,
+        );
+        if (outcome === "converted") result.rotated += 1;
+        else result.scanned -= 1;
+      } catch {
+        result.errors += 1;
+      }
+    }
+    return result;
+  }
+
   // Bounded id-cursor batches for every column (never an unbounded
   // findMany), re-encrypted under each row's OWN codec where it has one.
   // Idempotent — rows already on the active key are skipped, so an
@@ -380,9 +460,9 @@ export async function rotateColumn(
       // a stored value can legitimately carry survives rotation untouched.
       const next =
         codec !== null
-          ? reencryptBlob(value as Uint8Array, codec)
+          ? reencryptBlob(value as Uint8Array, codec, col.aad)
           : fromCiphertext(
-              reencryptString(toCiphertext(value, col.kind)!),
+              encrypt(decrypt(toCiphertext(value, col.kind)!)),
               col.kind,
             );
       await delegate.update({
@@ -390,7 +470,10 @@ export async function rotateColumn(
         data: { [col.field]: next },
       });
       result.rotated += 1;
-    } catch {
+    } catch (err) {
+      // Deleted between the read and the write (a backup replacing its
+      // pieces, an account going): nothing is left to rotate.
+      if (isRowGone(err)) return;
       await onUnreadable(id);
     }
   });

src/lib/export/__tests__/backup-chunks.test.ts

@@ -13,6 +13,8 @@ import { Buffer } from "node:buffer";
 
 import { describe, expect, it } from "vitest";
 
+import { decryptBytes, encryptBytes } from "@/lib/crypto";
+import { BACKUP_CHUNK_AAD } from "@/lib/crypto/encrypted-columns";
 import {
   BackupIntegrityError,
   newChunkStreamId,
@@ -91,4 +93,18 @@ describe("sealed backup pieces", () => {
       openBackupChunk(relabelled, { streamId: id, seq: 0, last: true }),
     ).toThrow(BackupIntegrityError);
   });
+
+  it("refuses a piece sealed without the backup-piece label", () => {
+    // The same key seals other binary values too (documents, for one). A
+    // value sealed for another purpose must not open as a backup piece, even
+    // if its plaintext happens to look like one.
+    const id = newChunkStreamId();
+    const sealed = sealBackupChunk(id, 0, true, PAYLOAD);
+    const plain = decryptBytes(sealed, BACKUP_CHUNK_AAD);
+    const unlabelled = encryptBytes(plain);
+    expect(() =>
+      openBackupChunk(unlabelled, { streamId: id, seq: 0, last: true }),
+    ).toThrow(BackupIntegrityError);
+    expect(() => decryptBytes(sealed)).toThrow();
+  });
 });

src/lib/export/backup-blob.ts

@@ -111,6 +111,22 @@ export class BackupBlobTooLargeError extends Error {
   }
 }
 
+/**
+ * Thrown when the copy a store would replace is being restored: a restore of
+ * it is queued or running. Replacing it then would pull the pieces out from
+ * under the restore. Nothing was written; the weekly run tries again next
+ * time.
+ */
+export class BackupBusyError extends Error {
+  constructor(readonly backupId: string) {
+    super(
+      "Backup not replaced: a restore of the current copy is queued or " +
+        "running. The previous copy is unchanged; the next run replaces it.",
+    );
+    this.name = "BackupBusyError";
+  }
+}
+
 /**
  * Produces the backup JSON in pieces. Every piece is written in order.
  *
@@ -142,44 +158,23 @@ export interface PackBackupOptions {
 export type BackupChunkSink = (sealed: Buffer, seq: number) => Promise<void>;
 
 /**
- * Serialised backup JSON, produced in pieces → sealed pieces of the stored
- * copy, handed to `sink` as they fill.
- *
- * The pipeline is JSON piece → gzip → a piece of about `BACKUP_CHUNK_BYTES`
- * → AES-256-GCM (`sealBackupChunk`). Nothing is held but the piece being
- * filled: the gzip stream applies backpressure through the promise `write`
- * hands back, and a full piece is sealed and sunk before the producer may
- * write again. The last piece is always sealed at the end, marked as the last,
- * even when it carries nothing, because that mark is what tells a reader the
- * copy is complete.
+ * Collects gzip output and seals it into pieces of about `chunkBytes`, in
+ * order, counting the stored bytes against the limit. `finish` seals what is
+ * left as the last piece, even when that is nothing, because the last-piece
+ * mark is what tells a reader the copy is complete.
  */
-export async function packBackupChunks(
+function chunkSealer(
   sink: BackupChunkSink,
   streamId: string,
-  producer: BackupJsonProducer,
-  options: PackBackupOptions = {},
-): Promise<{ chunks: number; bytes: number }> {
+  options: PackBackupOptions,
+) {
   const limitBytes = options.maxBytes ?? defaultBackupStoreLimit();
   const chunkBytes = options.chunkBytes ?? BACKUP_CHUNK_BYTES;
-  const gzip = createGzip();
   let pending: Buffer[] = [];
   let pendingBytes = 0;
   let seq = 0;
   let stored = 0;
 
-  gzip.on("data", (chunk: Buffer) => {
-    pending.push(chunk);
-    pendingBytes += chunk.byteLength;
-  });
-  const finished = new Promise<void>((resolve, reject) => {
-    gzip.on("end", resolve);
-    gzip.on("error", reject);
-  });
-  // The producer's own failure is what gets reported; this only keeps the
-  // same rejection from also surfacing as unhandled. `await finished` below
-  // still sees it.
-  finished.catch(() => {});
-
   const seal = async (last: boolean): Promise<void> => {
     const payload = pending.length === 1 ? pending[0]! : Buffer.concat(pending);
     pending = [];
@@ -193,8 +188,52 @@ export async function packBackupChunks(
     seq += 1;
   };
 
+  return {
+    push(gz: Buffer): void {
+      pending.push(gz);
+      pendingBytes += gz.byteLength;
+    },
+    async flushFull(): Promise<void> {
+      while (pendingBytes >= chunkBytes) await seal(false);
+    },
+    async finish(): Promise<{ chunks: number; bytes: number }> {
+      while (pendingBytes >= chunkBytes) await seal(false);
+      await seal(true);
+      return { chunks: seq, bytes: stored };
+    },
+  };
+}
+
+/**
+ * Serialised backup JSON, produced in pieces → sealed pieces of the stored
+ * copy, handed to `sink` as they fill.
+ *
+ * The pipeline is JSON piece → gzip → a piece of about `BACKUP_CHUNK_BYTES`
+ * → AES-256-GCM (`sealBackupChunk`). Nothing is held but the piece being
+ * filled: the gzip stream applies backpressure through the promise `write`
+ * hands back, and a full piece is sealed and sunk before the producer may
+ * write again.
+ */
+export async function packBackupChunks(
+  sink: BackupChunkSink,
+  streamId: string,
+  producer: BackupJsonProducer,
+  options: PackBackupOptions = {},
+): Promise<{ chunks: number; bytes: number }> {
+  const sealer = chunkSealer(sink, streamId, options);
+  const gzip = createGzip();
+  gzip.on("data", (chunk: Buffer) => sealer.push(chunk));
+  const finished = new Promise<void>((resolve, reject) => {
+    gzip.on("end", resolve);
+    gzip.on("error", reject);
+  });
+  // The producer's own failure is what gets reported; this only keeps the
+  // same rejection from also surfacing as unhandled. `await finished` below
+  // still sees it.
+  finished.catch(() => {});
+
   const write = async (chunk: string | Buffer): Promise<void> => {
-    while (pendingBytes >= chunkBytes) await seal(false);
+    await sealer.flushFull();
     if (gzip.write(chunk, "utf8")) return;
     await new Promise<void>((resolve, reject) => {
       const onDrain = () => {
@@ -218,9 +257,26 @@ export async function packBackupChunks(
   }
   gzip.end();
   await finished;
-  while (pendingBytes >= chunkBytes) await seal(false);
-  await seal(true);
-  return { chunks: seq, bytes: stored };
+  return sealer.finish();
+}
+
+/**
+ * Bytes that are already one gzip stream of the backup JSON → sealed pieces.
+ * For converting a copy stored before v1.39.2, whose content is gzip output
+ * already and needs no second compression.
+ */
+export async function packGzipChunks(
+  sink: BackupChunkSink,
+  streamId: string,
+  gz: AsyncIterable<Buffer> | Iterable<Buffer>,
+  options: PackBackupOptions = {},
+): Promise<{ chunks: number; bytes: number }> {
+  const sealer = chunkSealer(sink, streamId, options);
+  for await (const piece of gz) {
+    sealer.push(piece);
+    await sealer.flushFull();
+  }
+  return sealer.finish();
 }
 
 /**
@@ -267,6 +323,19 @@ export function unpackBackupBlob(stored: string): string {
   ).toString("utf8");
 }
 
+/**
+ * A single-value copy in one of the two forms written from one string (plain
+ * `encrypt(json)` or `HLZ1:` gzip) → its content as gzip bytes, for the
+ * conversion into pieces. The `~hlgcm1.` stream form is read in slices instead.
+ */
+export function singleValueToGzip(stored: string): Buffer {
+  const plaintext = decrypt(stored);
+  if (plaintext.startsWith(GZIP_MARKER)) {
+    return Buffer.from(plaintext.slice(GZIP_MARKER.length), "base64");
+  }
+  return gzipSync(plaintext);
+}
+
 /**
  * A stored `DataBackup.data` string → a source of its JSON as byte chunks,
  * which can be opened as many times as the caller needs to read it.

src/lib/export/backup-chunks.ts

@@ -28,11 +28,15 @@
  * with the position and the flag carried inside the authenticated plaintext
  * rather than in the nonce. Every piece has its own random 96-bit IV.
  *
+ * Every piece is sealed with a fixed associated-data label
+ * (`BACKUP_CHUNK_AAD`), so a value the same key sealed for another purpose
+ * never opens as a backup piece.
+ *
  * The header sits inside the ciphertext rather than beside it, deliberately:
  * key rotation re-seals a piece without knowing what it holds
- * (`reencryptBytesToActive`), and the binding survives that untouched. A
- * binding held as associated data outside the ciphertext would need a rotation
- * path of its own for this one column.
+ * (`reencryptBytesToActive`, given the same fixed label), and the binding
+ * survives that untouched. A position or stream id held as associated data
+ * would differ per piece, and rotation would need to know it for each one.
  *
  * The payload is gzip output. The pieces of one copy, opened in order and
  * concatenated, are one gzip stream of the backup JSON.
@@ -41,6 +45,7 @@ import { Buffer } from "node:buffer";
 import { randomBytes } from "node:crypto";
 
 import { decryptBytes, encryptBytes } from "@/lib/crypto";
+import { BACKUP_CHUNK_AAD } from "@/lib/crypto/encrypted-columns";
 
 const MAGIC = Buffer.from("HLBC", "ascii");
 const VERSION = 0x01;
@@ -96,7 +101,7 @@ export function sealBackupChunk(
   header.writeUInt32BE(seq, at);
   at += 4;
   header[at] = last ? FLAG_LAST : 0;
-  return encryptBytes(Buffer.concat([header, payload]));
+  return encryptBytes(Buffer.concat([header, payload]), BACKUP_CHUNK_AAD);
 }
 
 export interface ExpectedChunk {
@@ -117,7 +122,7 @@ export function openBackupChunk(
 ): Buffer {
   let plain: Buffer;
   try {
-    plain = decryptBytes(Buffer.from(sealed));
+    plain = decryptBytes(Buffer.from(sealed), BACKUP_CHUNK_AAD);
   } catch (err) {
     throw new BackupIntegrityError(
       `Piece ${expected.seq} of the stored copy could not be decrypted: ${

src/lib/export/restore-backup.ts

@@ -32,11 +32,9 @@ import { Buffer } from "node:buffer";
 import { prisma, toJson } from "@/lib/db";
 import { auditLog } from "@/lib/auth/audit";
 import {
-  BACKUP_UNDECRYPTABLE_CODE,
-  BACKUP_UNDECRYPTABLE_ERROR,
-} from "@/lib/export/backup-blob";
-import {
+  isStoredBackupReadError,
   openStoredBackup,
+  storedBackupRefusal,
   type StoredBackupRef,
 } from "@/lib/export/stored-backup";
 import {
@@ -354,8 +352,9 @@ export async function restoreBackup(
     // Bad stored input, not a broken server: a copy written under a key the
     // operator has since dropped, or one whose bytes have changed. Refused
     // above the transaction, so nothing was touched.
-    return refused(422, BACKUP_UNDECRYPTABLE_CODE, BACKUP_UNDECRYPTABLE_ERROR, {
-      errorCode: BACKUP_UNDECRYPTABLE_CODE,
+    const refusal = storedBackupRefusal(err);
+    return refused(refusal.status, refusal.code, refusal.message, {
+      errorCode: refusal.code,
     });
   }
 
@@ -381,16 +380,25 @@ export async function restoreBackup(
     raw = streamed.raw;
     payload = parseBackupPayload(raw);
   } catch (err) {
+    const readFailure = isStoredBackupReadError(err);
     await auditLog("admin.backups.restore.failed", {
       userId: input.actorUserId,
       ipAddress: input.ipAddress,
       details: {
         backupId: backup.id,
         ownerId: backup.userId,
-        reason: "schema_invalid",
+        reason: readFailure ? "read_failed" : "schema_invalid",
         message: err instanceof Error ? err.message : String(err),
       },
     });
+    if (readFailure) {
+      // The copy changed after it was opened: replaced by the weekly run, or
+      // altered. Nothing has been deleted yet.
+      const refusal = storedBackupRefusal(err);
+      return refused(refusal.status, refusal.code, refusal.message, {
+        errorCode: refusal.code,
+      });
+    }
     return refused(
       422,
       "schema_invalid",
@@ -2299,6 +2307,14 @@ export async function restoreBackup(
       },
     });
     annotate({ meta: { restoreFailReason: verbose } });
+    if (isStoredBackupReadError(err)) {
+      // The second read, inside the transaction, found the copy replaced or
+      // altered. The transaction rolled back, so nothing was changed.
+      const refusal = storedBackupRefusal(err);
+      return refused(refusal.status, refusal.code, refusal.message, {
+        errorCode: refusal.code,
+      });
+    }
     return refused(
       500,
       "transaction_failed",

src/lib/export/store-backup-blob.ts

@@ -25,12 +25,20 @@
  */
 import type { PrismaClient } from "@/generated/prisma/client";
 
+import { isStreamCiphertext, openStreamDecryptor } from "@/lib/crypto";
 import {
+  BackupBusyError,
   packBackupChunks,
+  packGzipChunks,
+  singleValueToGzip,
   type BackupJsonProducer,
   type PackBackupOptions,
 } from "@/lib/export/backup-blob";
-import { newChunkStreamId } from "@/lib/export/backup-chunks";
+import {
+  BackupIntegrityError,
+  newChunkStreamId,
+} from "@/lib/export/backup-chunks";
+import { BackupFormsConflictError } from "@/lib/export/stored-backup";
 
 /**
  * How long the storing transaction may stay open. The job around it expires
@@ -64,6 +72,17 @@ export interface StoreBackupBlobInput {
   ownerAfterRead?: () => string;
 }
 
+type Tx = Parameters<Parameters<PrismaClient["$transaction"]>[0]>[0];
+
+/** Refuse to touch a copy an unfinished restore is reading. */
+async function assertNotBeingRestored(tx: Tx, backupId: string) {
+  const active = await tx.backupRestoreJob.findFirst({
+    where: { backupId, status: { in: ["queued", "running"] } },
+    select: { id: true },
+  });
+  if (active) throw new BackupBusyError(backupId);
+}
+
 /**
  * Pack `producer`'s JSON into sealed pieces and store them as the
  * `(userId, type)` backup, replacing that backup's previous copy. Resolves to
@@ -89,6 +108,7 @@ export async function storeBackupBlob(
         create: { userId: input.userId, type: input.type, data: null },
         select: { id: true },
       });
+      await assertNotBeingRestored(tx, row.id);
       await tx.dataBackupChunk.deleteMany({ where: { backupId: row.id } });
 
       const streamId = newChunkStreamId();
@@ -104,6 +124,11 @@ export async function storeBackupBlob(
         options,
       );
 
+      // Again at the end: a restore may have been queued while this ran. The
+      // window left is the few milliseconds to the commit, and a restore
+      // started in it finds the copy changed (`backup_changed`) or, if it is
+      // already reading, is told the copy was replaced while being read.
+      await assertNotBeingRestored(tx, row.id);
       await tx.dataBackup.update({
         where: { id: row.id },
         data: {
@@ -118,3 +143,129 @@ export async function storeBackupBlob(
     { timeout: STORE_TRANSACTION_TIMEOUT_MS, maxWait: 60_000 },
   );
 }
+
+/**
+ * The first characters of a single stored value, which carry its key id, or
+ * null when the row no longer holds one. For the key-rotation scan, which must
+ * not read a value that can be a hundred megabytes whole.
+ */
+export async function singleValueBackupHead(
+  prisma: PrismaClient,
+  id: string,
+): Promise<string | null> {
+  const [row] = await prisma.$queryRaw<Array<{ head: string | null }>>`
+    SELECT left(data, 64) AS head FROM data_backups WHERE id = ${id}
+  `;
+  return row?.head ?? null;
+}
+
+/**
+ * How much of a single stored value the conversion reads per query, in
+ * characters. A multiple of four, so every slice of base64 decodes on its own.
+ */
+const CONVERT_SLICE_CHARS = 1024 * 1024;
+
+/**
+ * Turn a copy stored as one value (before v1.39.2) into sealed pieces under
+ * the active key, in place: same row, same date. Resolves to `"gone"` when the
+ * row no longer holds a single value (converted, replaced or deleted since it
+ * was listed).
+ *
+ * Why convert rather than re-seal. Re-sealing a single value under the new key
+ * needs the stored string, its decoded bytes, the plaintext and the new
+ * ciphertext at once, and one value can be a hundred megabytes; key rotation
+ * runs in the app container. Converting reads the value a slice at a time and
+ * writes pieces as it goes.
+ *
+ * The single stream of v1.38.6 to v1.39.1 is read in slices and its tag is checked only at
+ * the end, so the pieces are written before the check has passed. That is why
+ * all of it is one transaction: a copy that fails the check rolls back, and
+ * the row is left exactly as it was. The older forms were written from one
+ * string and fit in one, so they are read whole.
+ */
+export async function convertSingleValueBackup(
+  prisma: PrismaClient,
+  id: string,
+  /** Characters per read; a multiple of four. Tests lower it. */
+  sliceChars: number = CONVERT_SLICE_CHARS,
+): Promise<"converted" | "gone"> {
+  if (sliceChars % 4 !== 0) throw new Error("sliceChars must divide by 4");
+  return prisma.$transaction(
+    async (tx) => {
+      await tx.$executeRawUnsafe(
+        `SET LOCAL idle_in_transaction_session_timeout = '${STORE_IDLE_TIMEOUT}'`,
+      );
+      const [row] = await tx.$queryRaw<
+        Array<{ head: string; len: number; stream_id: string | null }>
+      >`
+        SELECT left(data, 128) AS head, length(data)::int AS len,
+               chunk_stream_id AS stream_id
+        FROM data_backups
+        WHERE id = ${id} AND data IS NOT NULL
+        FOR UPDATE
+      `;
+      if (!row) return "gone";
+      if (row.stream_id !== null) {
+        throw new BackupFormsConflictError();
+      }
+
+      let gz: AsyncIterable<Buffer> | Iterable<Buffer>;
+      if (isStreamCiphertext(row.head)) {
+        const decryptor = openStreamDecryptor(row.head);
+        const slice = async (from: number): Promise<Buffer> => {
+          const [part] = await tx.$queryRaw<Array<{ s: string }>>`
+            SELECT substr(data, ${from + 1}::int, ${sliceChars}::int) AS s
+            FROM data_backups WHERE id = ${id}
+          `;
+          return Buffer.from(part?.s ?? "", "base64");
+        };
+        gz = (async function* () {
+          // The last 16 bytes of the body are the tag, so they are held back
+          // from every slice until the next one shows they were not the end.
+          let held = Buffer.alloc(0);
+          for (let at = decryptor.headerLength; at < row.len;) {
+            const bytes = Buffer.concat([held, await slice(at)]);
+            at += sliceChars;
+            const cut = Math.max(0, bytes.byteLength - 16);
+            held = Buffer.from(bytes.subarray(cut));
+            const out = decryptor.update(bytes.subarray(0, cut));
+            if (out.byteLength > 0) yield out;
+          }
+          if (held.byteLength !== 16) {
+            throw new BackupIntegrityError("The stored copy is truncated.");
+          }
+          const tail = decryptor.final(held);
+          if (tail.byteLength > 0) yield tail;
+        })();
+      } else {
+        const whole = await tx.dataBackup.findUniqueOrThrow({
+          where: { id },
+          select: { data: true },
+        });
+        gz = [singleValueToGzip(whole.data!)];
+      }
+
+      await tx.dataBackupChunk.deleteMany({ where: { backupId: id } });
+      const streamId = newChunkStreamId();
+      const { chunks } = await packGzipChunks(
+        async (sealed, seq) => {
+          await tx.dataBackupChunk.create({
+            data: { backupId: id, seq, data: new Uint8Array(sealed) },
+            select: { id: true },
+          });
+        },
+        streamId,
+        gz,
+        // The copy exists already; converting it must not be refused by the
+        // limit on writing a new one.
+        { maxBytes: Number.MAX_SAFE_INTEGER },
+      );
+      await tx.dataBackup.update({
+        where: { id },
+        data: { data: null, chunkCount: chunks, chunkStreamId: streamId },
+      });
+      return "converted";
+    },
+    { timeout: STORE_TRANSACTION_TIMEOUT_MS, maxWait: 60_000 },
+  );
+}

src/lib/export/stored-backup.ts

@@ -23,7 +23,11 @@ import { Readable, pipeline } from "node:stream";
 import { createGunzip } from "node:zlib";
 
 import type { PrismaClient } from "@/generated/prisma/client";
-import { openBackupBlob } from "@/lib/export/backup-blob";
+import {
+  BACKUP_UNDECRYPTABLE_CODE,
+  BACKUP_UNDECRYPTABLE_ERROR,
+  openBackupBlob,
+} from "@/lib/export/backup-blob";
 import {
   BackupIntegrityError,
   openBackupChunk,
@@ -47,7 +51,43 @@ export interface StoredBackupRef {
   chunkStreamId: string | null;
 }
 
-type ChunkReader = Pick<PrismaClient, "dataBackupChunk">;
+type ChunkReader = Pick<PrismaClient, "dataBackupChunk" | "dataBackup">;
+
+/**
+ * Why a row holding both forms is refused. Only an older release writes a
+ * single value into a row that already has pieces (after a downgrade), and
+ * nothing on the row says which of the two is the newer copy.
+ */
+const BOTH_FORMS_MESSAGE_TEXT =
+  "This backup holds both a single stored value and pieces, which happens " +
+  "when an older HealthLog version wrote to it after an upgrade. There is no " +
+  "way to tell which of the two is current, so it is not read. The next " +
+  "weekly backup replaces it with a new copy.";
+
+export const BOTH_FORMS_MESSAGE = BOTH_FORMS_MESSAGE_TEXT;
+
+/** A row holding both forms (see {@link BOTH_FORMS_MESSAGE}). */
+export class BackupFormsConflictError extends BackupIntegrityError {
+  constructor() {
+    super(BOTH_FORMS_MESSAGE);
+    this.name = "BackupFormsConflictError";
+  }
+}
+
+/**
+ * The copy was replaced by a newer one (the weekly backup ran) while a
+ * restore, preview or download was reading it. Nothing is wrong with either
+ * copy; the reader has to start again on the new one.
+ */
+export class BackupReplacedError extends Error {
+  constructor() {
+    super(
+      "The backup was replaced by a newer one while it was being read. " +
+        "Nothing was changed; start again to use the new copy.",
+    );
+    this.name = "BackupReplacedError";
+  }
+}
 
 /**
  * A value that changes whenever the stored copy is replaced, for the restore
@@ -70,20 +110,34 @@ async function* openedChunks(
   count: number,
 ): AsyncGenerator<Buffer> {
   for (let seq = 0; seq < count; seq++) {
-    const row = await prisma.dataBackupChunk.findUnique({
-      where: { backupId_seq: { backupId, seq } },
-      select: { data: true },
-    });
-    if (!row) {
-      throw new BackupIntegrityError(
-        `Piece ${seq} of ${count} of the stored copy is missing.`,
-      );
+    let piece: Buffer;
+    try {
+      const row = await prisma.dataBackupChunk.findUnique({
+        where: { backupId_seq: { backupId, seq } },
+        select: { data: true },
+      });
+      if (!row) {
+        throw new BackupIntegrityError(
+          `Piece ${seq} of ${count} of the stored copy is missing.`,
+        );
+      }
+      piece = openBackupChunk(row.data, {
+        streamId,
+        seq,
+        last: seq === count - 1,
+      });
+    } catch (err) {
+      // A piece that no longer fits is either tampering or the copy having
+      // been replaced since it was opened. The row tells them apart: a new
+      // copy always has a new stream id.
+      const now = await prisma.dataBackup.findUnique({
+        where: { id: backupId },
+        select: { chunkStreamId: true },
+      });
+      if (now?.chunkStreamId !== streamId) throw new BackupReplacedError();
+      throw err;
     }
-    yield openBackupChunk(row.data, {
-      streamId,
-      seq,
-      last: seq === count - 1,
-    });
+    yield piece;
   }
 }
 
@@ -97,6 +151,14 @@ export async function openStoredBackup(
   prisma: ChunkReader,
   backup: StoredBackupRef,
 ): Promise<BackupSource> {
+  if (backup.data != null && backup.chunkStreamId != null) {
+    throw new BackupFormsConflictError();
+  }
+  if ((backup.chunkStreamId == null) !== (backup.chunkCount == null)) {
+    throw new BackupIntegrityError(
+      "The stored copy lists pieces without saying how many, or the reverse.",
+    );
+  }
   if (backup.chunkStreamId == null || backup.chunkCount == null) {
     if (backup.data == null) {
       throw new BackupIntegrityError("The stored copy has no content.");
@@ -113,6 +175,11 @@ export async function openStoredBackup(
     where: { backupId: id },
   });
   if (stored !== count) {
+    const now = await prisma.dataBackup.findUnique({
+      where: { id },
+      select: { chunkStreamId: true },
+    });
+    if (now?.chunkStreamId !== streamId) throw new BackupReplacedError();
     throw new BackupIntegrityError(
       `The stored copy lists ${count} pieces but ${stored} are stored.`,
     );
@@ -135,3 +202,38 @@ export async function openStoredBackup(
     return gunzip;
   };
 }
+
+/** How a route or the restore job answers a stored copy it could not read. */
+export interface StoredBackupRefusal {
+  status: 409 | 422;
+  /** `backup_changed` or `backup.payload.undecryptable`. */
+  code: "backup_changed" | typeof BACKUP_UNDECRYPTABLE_CODE;
+  message: string;
+}
+
+/**
+ * The answer for a failure to open or read a stored copy. A copy replaced
+ * while it was read is not damage and says so (409, `backup_changed`); a row
+ * holding both forms says why it is not read; everything else is the
+ * undecryptable refusal the routes have always given.
+ */
+export function storedBackupRefusal(err: unknown): StoredBackupRefusal {
+  if (err instanceof BackupReplacedError) {
+    return { status: 409, code: "backup_changed", message: err.message };
+  }
+  return {
+    status: 422,
+    code: BACKUP_UNDECRYPTABLE_CODE,
+    message:
+      err instanceof BackupFormsConflictError
+        ? `${err.message} Nothing was changed.`
+        : BACKUP_UNDECRYPTABLE_ERROR,
+  };
+}
+
+/** True for the failures `storedBackupRefusal` describes, rather than a bug. */
+export function isStoredBackupReadError(err: unknown): boolean {
+  return (
+    err instanceof BackupReplacedError || err instanceof BackupIntegrityError
+  );
+}

src/lib/jobs/job-outcome.ts

@@ -234,6 +234,9 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
   "users_retryable",
   "users_scanned",
   "users_skipped",
+  // v1.39.2 — accounts the weekly backup left alone because a restore of their
+  // current copy was queued or running.
+  "users_skipped_restoring",
   "users_synced",
   "users_useful",
   "wedge_skipped",

src/lib/jobs/reminder/__tests__/backup-handlers.test.ts

@@ -54,7 +54,10 @@ vi.mock("../shared", () => ({
   getWorkerPrisma: mocks.getWorkerPrisma,
 }));
 
-import { BackupBlobTooLargeError } from "@/lib/export/backup-blob";
+import {
+  BackupBlobTooLargeError,
+  BackupBusyError,
+} from "@/lib/export/backup-blob";
 
 import { handleDataBackup } from "../backup-handlers";
 
@@ -188,6 +191,21 @@ describe("handleDataBackup outcome", () => {
     });
   });
 
+  it("skips an account whose copy is being restored, without failing the pass", async () => {
+    // Replacing a copy while a restore reads it would pull the pieces out from
+    // under the restore. The account keeps its copy and the next run tries
+    // again; that is not a failure of this pass.
+    mocks.getWorkerPrisma.mockReturnValue(buildPrismaMock());
+    mocks.store.mockRejectedValue(new BackupBusyError("backup-row"));
+
+    const outcome = await handleDataBackup([]);
+
+    expect(outcome).toMatchObject({
+      ok: true,
+      did: { backed: 0, total: 1, users_failed: 0, users_skipped_restoring: 1 },
+    });
+  });
+
   it("still passes when one account failed and another was written", async () => {
     // The fan-out rule: a pass is judged on the pass. Failing the queue over
     // one account's record would re-run the whole cohort on every retry, and

src/lib/jobs/reminder/backup-handlers.ts

@@ -7,7 +7,10 @@
 import { type Job, type JobWithMetadata } from "pg-boss";
 import { jobBudget } from "@/lib/jobs/job-budget";
 import { recordError } from "@/lib/jobs/worker-status";
-import { BackupBlobTooLargeError } from "@/lib/export/backup-blob";
+import {
+  BackupBlobTooLargeError,
+  BackupBusyError,
+} from "@/lib/export/backup-blob";
 import { storeBackupBlob } from "@/lib/export/store-backup-blob";
 import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
 import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
@@ -132,6 +135,7 @@ export async function handleDataBackup(
       let backed = 0;
       let usersFailed = 0;
       let oversized = 0;
+      let skippedRestoring = 0;
       let largestBlobBytes = 0;
       // Kept for the failure report: without it the run that failed for
       // everybody names no reason at all, which is the shape this pass spent
@@ -140,7 +144,7 @@ export async function handleDataBackup(
       for (const user of users) {
         reportJobProgress({
           backup_user: user.id,
-          backup_users_done: backed + usersFailed,
+          backup_users_done: backed + usersFailed + skippedRestoring,
           backup_users_total: users.length,
         });
         try {
@@ -162,6 +166,16 @@ export async function handleDataBackup(
           largestBlobBytes = Math.max(largestBlobBytes, storedBytes);
           backed++;
         } catch (err) {
+          // A restore of this account's current copy is queued or running.
+          // Replacing the copy would pull it out from under the restore, so
+          // the account keeps it and the next run tries again. Not a failure.
+          if (err instanceof BackupBusyError) {
+            skippedRestoring++;
+            evt.addWarning(
+              `Skipped user ${user.id}: a restore of the current copy is queued or running`,
+            );
+            continue;
+          }
           // One user's payload failing is that user's problem, not the pass's:
           // it rides out as a count so the weekly run is not retried for the
           // whole cohort.
@@ -192,6 +206,7 @@ export async function handleDataBackup(
         total: users.length,
         users_failed: usersFailed,
         records_oversized: oversized,
+        users_skipped_restoring: skippedRestoring,
       };
 
       // A pass that wrote nothing for anybody protected nobody, and saying
@@ -201,8 +216,9 @@ export async function handleDataBackup(
       // fan-out rule, and that account's own row ages on the backups page —
       // but zero out of a non-empty cohort is the pass failing rather than a
       // leg of it, so pg-boss records a failed run and the backups page reads
-      // it back through `readLastQueueRun`.
-      if (users.length > 0 && backed === 0) {
+      // it back through `readLastQueueRun`. An account skipped because its
+      // copy is being restored still has that copy, so it does not count.
+      if (users.length > 0 && backed === 0 && usersFailed > 0) {
         return jobFailed("no account could be backed up", lastError, did);
       }
 

src/lib/openapi/routes/admin-backups.ts

@@ -139,7 +139,17 @@ const restoreJob = z
  */
 const undecryptableResponse = {
   description:
-    "The stored copy could not be opened: either the key that wrote it is no longer in `ENCRYPTION_KEYS` (a rotation that dropped the legacy entry too early), or the stored bytes are not the ones that were written (a stored piece missing, moved, altered, taken from another copy, or the copy cut short). Every piece is checked before any of the copy is read. `meta.errorCode` = `backup.payload.undecryptable`. Nothing was changed.",
+    "The stored copy could not be opened: either the key that wrote it is no longer in `ENCRYPTION_KEYS` (a rotation that dropped the legacy entry too early), or the stored bytes are not the ones that were written (a stored piece missing, moved, altered, taken from another copy, or the copy cut short), or the row holds both a single value and pieces, which only an older release writing to it after an upgrade produces. Every piece is checked before any of the copy is read. `meta.errorCode` = `backup.payload.undecryptable`. Nothing was changed.",
+  content: { "application/json": { schema: errorEnvelope } },
+};
+
+/**
+ * A copy replaced by a newer one (the weekly backup) while it was being
+ * opened or read. Not damage: the reader starts again on the new copy.
+ */
+const replacedResponse = {
+  description:
+    "The stored copy was replaced by a newer one while it was being read. `meta.errorCode` = `backup_changed`. Nothing was changed; start again to use the new copy.",
   content: { "application/json": { schema: errorEnvelope } },
 };
 
@@ -181,6 +191,7 @@ export const adminBackupPaths: NonNullable<ZodOpenApiObject["paths"]> = {
         },
         "403": adminOnlyResponse,
         "404": notFoundResponse,
+        "409": replacedResponse,
         "422": undecryptableResponse,
         "500": {
           description:
@@ -230,7 +241,7 @@ export const adminBackupPaths: NonNullable<ZodOpenApiObject["paths"]> = {
         "404": notFoundResponse,
         "409": {
           description:
-            "A restore of the same account is already queued or running (`meta.errorCode` = `backup.restore.active`, `meta.jobId` names it), or a request under the same `Idempotency-Key` is still in flight. Nothing was changed.",
+            "A restore of the same account is already queued or running (`meta.errorCode` = `backup.restore.active`, `meta.jobId` names it), the stored copy was replaced by a newer one while it was being checked (`meta.errorCode` = `backup_changed`), or a request under the same `Idempotency-Key` is still in flight. Nothing was changed.",
           content: { "application/json": { schema: errorEnvelope } },
         },
         "413": {