Merge branch 'fix/backup-chunked-storage' into release/v1.39.2

src/lib/export/__tests__/backup-chunks.test.ts

@@ -13,6 +13,8 @@ import { Buffer } from "node:buffer";
 
 import { describe, expect, it } from "vitest";
 
+import { decryptBytes, encryptBytes } from "@/lib/crypto";
+import { BACKUP_CHUNK_AAD } from "@/lib/crypto/encrypted-columns";
 import {
   BackupIntegrityError,
   newChunkStreamId,
@@ -91,4 +93,18 @@ describe("sealed backup pieces", () => {
       openBackupChunk(relabelled, { streamId: id, seq: 0, last: true }),
     ).toThrow(BackupIntegrityError);
   });
+
+  it("refuses a piece sealed without the backup-piece label", () => {
+    // The same key seals other binary values too (documents, for one). A
+    // value sealed for another purpose must not open as a backup piece, even
+    // if its plaintext happens to look like one.
+    const id = newChunkStreamId();
+    const sealed = sealBackupChunk(id, 0, true, PAYLOAD);
+    const plain = decryptBytes(sealed, BACKUP_CHUNK_AAD);
+    const unlabelled = encryptBytes(plain);
+    expect(() =>
+      openBackupChunk(unlabelled, { streamId: id, seq: 0, last: true }),
+    ).toThrow(BackupIntegrityError);
+    expect(() => decryptBytes(sealed)).toThrow();
+  });
 });

src/lib/export/backup-blob.ts

@@ -111,6 +111,22 @@ export class BackupBlobTooLargeError extends Error {
   }
 }
 
+/**
+ * Thrown when the copy a store would replace is being restored: a restore of
+ * it is queued or running. Replacing it then would pull the pieces out from
+ * under the restore. Nothing was written; the weekly run tries again next
+ * time.
+ */
+export class BackupBusyError extends Error {
+  constructor(readonly backupId: string) {
+    super(
+      "Backup not replaced: a restore of the current copy is queued or " +
+        "running. The previous copy is unchanged; the next run replaces it.",
+    );
+    this.name = "BackupBusyError";
+  }
+}
+
 /**
  * Produces the backup JSON in pieces. Every piece is written in order.
  *
@@ -142,44 +158,23 @@ export interface PackBackupOptions {
 export type BackupChunkSink = (sealed: Buffer, seq: number) => Promise<void>;
 
 /**
- * Serialised backup JSON, produced in pieces → sealed pieces of the stored
- * copy, handed to `sink` as they fill.
- *
- * The pipeline is JSON piece → gzip → a piece of about `BACKUP_CHUNK_BYTES`
- * → AES-256-GCM (`sealBackupChunk`). Nothing is held but the piece being
- * filled: the gzip stream applies backpressure through the promise `write`
- * hands back, and a full piece is sealed and sunk before the producer may
- * write again. The last piece is always sealed at the end, marked as the last,
- * even when it carries nothing, because that mark is what tells a reader the
- * copy is complete.
+ * Collects gzip output and seals it into pieces of about `chunkBytes`, in
+ * order, counting the stored bytes against the limit. `finish` seals what is
+ * left as the last piece, even when that is nothing, because the last-piece
+ * mark is what tells a reader the copy is complete.
  */
-export async function packBackupChunks(
+function chunkSealer(
   sink: BackupChunkSink,
   streamId: string,
-  producer: BackupJsonProducer,
-  options: PackBackupOptions = {},
-): Promise<{ chunks: number; bytes: number }> {
+  options: PackBackupOptions,
+) {
   const limitBytes = options.maxBytes ?? defaultBackupStoreLimit();
   const chunkBytes = options.chunkBytes ?? BACKUP_CHUNK_BYTES;
-  const gzip = createGzip();
   let pending: Buffer[] = [];
   let pendingBytes = 0;
   let seq = 0;
   let stored = 0;
 
-  gzip.on("data", (chunk: Buffer) => {
-    pending.push(chunk);
-    pendingBytes += chunk.byteLength;
-  });
-  const finished = new Promise<void>((resolve, reject) => {
-    gzip.on("end", resolve);
-    gzip.on("error", reject);
-  });
-  // The producer's own failure is what gets reported; this only keeps the
-  // same rejection from also surfacing as unhandled. `await finished` below
-  // still sees it.
-  finished.catch(() => {});
-
   const seal = async (last: boolean): Promise<void> => {
     const payload = pending.length === 1 ? pending[0]! : Buffer.concat(pending);
     pending = [];
@@ -193,8 +188,52 @@ export async function packBackupChunks(
     seq += 1;
   };
 
+  return {
+    push(gz: Buffer): void {
+      pending.push(gz);
+      pendingBytes += gz.byteLength;
+    },
+    async flushFull(): Promise<void> {
+      while (pendingBytes >= chunkBytes) await seal(false);
+    },
+    async finish(): Promise<{ chunks: number; bytes: number }> {
+      while (pendingBytes >= chunkBytes) await seal(false);
+      await seal(true);
+      return { chunks: seq, bytes: stored };
+    },
+  };
+}
+
+/**
+ * Serialised backup JSON, produced in pieces → sealed pieces of the stored
+ * copy, handed to `sink` as they fill.
+ *
+ * The pipeline is JSON piece → gzip → a piece of about `BACKUP_CHUNK_BYTES`
+ * → AES-256-GCM (`sealBackupChunk`). Nothing is held but the piece being
+ * filled: the gzip stream applies backpressure through the promise `write`
+ * hands back, and a full piece is sealed and sunk before the producer may
+ * write again.
+ */
+export async function packBackupChunks(
+  sink: BackupChunkSink,
+  streamId: string,
+  producer: BackupJsonProducer,
+  options: PackBackupOptions = {},
+): Promise<{ chunks: number; bytes: number }> {
+  const sealer = chunkSealer(sink, streamId, options);
+  const gzip = createGzip();
+  gzip.on("data", (chunk: Buffer) => sealer.push(chunk));
+  const finished = new Promise<void>((resolve, reject) => {
+    gzip.on("end", resolve);
+    gzip.on("error", reject);
+  });
+  // The producer's own failure is what gets reported; this only keeps the
+  // same rejection from also surfacing as unhandled. `await finished` below
+  // still sees it.
+  finished.catch(() => {});
+
   const write = async (chunk: string | Buffer): Promise<void> => {
-    while (pendingBytes >= chunkBytes) await seal(false);
+    await sealer.flushFull();
     if (gzip.write(chunk, "utf8")) return;
     await new Promise<void>((resolve, reject) => {
       const onDrain = () => {
@@ -218,9 +257,26 @@ export async function packBackupChunks(
   }
   gzip.end();
   await finished;
-  while (pendingBytes >= chunkBytes) await seal(false);
-  await seal(true);
-  return { chunks: seq, bytes: stored };
+  return sealer.finish();
+}
+
+/**
+ * Bytes that are already one gzip stream of the backup JSON → sealed pieces.
+ * For converting a copy stored before v1.39.2, whose content is gzip output
+ * already and needs no second compression.
+ */
+export async function packGzipChunks(
+  sink: BackupChunkSink,
+  streamId: string,
+  gz: AsyncIterable<Buffer> | Iterable<Buffer>,
+  options: PackBackupOptions = {},
+): Promise<{ chunks: number; bytes: number }> {
+  const sealer = chunkSealer(sink, streamId, options);
+  for await (const piece of gz) {
+    sealer.push(piece);
+    await sealer.flushFull();
+  }
+  return sealer.finish();
 }
 
 /**
@@ -267,6 +323,19 @@ export function unpackBackupBlob(stored: string): string {
   ).toString("utf8");
 }
 
+/**
+ * A single-value copy in one of the two forms written from one string (plain
+ * `encrypt(json)` or `HLZ1:` gzip) → its content as gzip bytes, for the
+ * conversion into pieces. The `~hlgcm1.` stream form is read in slices instead.
+ */
+export function singleValueToGzip(stored: string): Buffer {
+  const plaintext = decrypt(stored);
+  if (plaintext.startsWith(GZIP_MARKER)) {
+    return Buffer.from(plaintext.slice(GZIP_MARKER.length), "base64");
+  }
+  return gzipSync(plaintext);
+}
+
 /**
  * A stored `DataBackup.data` string → a source of its JSON as byte chunks,
  * which can be opened as many times as the caller needs to read it.

src/lib/export/backup-chunks.ts

@@ -28,11 +28,15 @@
  * with the position and the flag carried inside the authenticated plaintext
  * rather than in the nonce. Every piece has its own random 96-bit IV.
  *
+ * Every piece is sealed with a fixed associated-data label
+ * (`BACKUP_CHUNK_AAD`), so a value the same key sealed for another purpose
+ * never opens as a backup piece.
+ *
  * The header sits inside the ciphertext rather than beside it, deliberately:
  * key rotation re-seals a piece without knowing what it holds
- * (`reencryptBytesToActive`), and the binding survives that untouched. A
- * binding held as associated data outside the ciphertext would need a rotation
- * path of its own for this one column.
+ * (`reencryptBytesToActive`, given the same fixed label), and the binding
+ * survives that untouched. A position or stream id held as associated data
+ * would differ per piece, and rotation would need to know it for each one.
  *
  * The payload is gzip output. The pieces of one copy, opened in order and
  * concatenated, are one gzip stream of the backup JSON.
@@ -41,6 +45,7 @@ import { Buffer } from "node:buffer";
 import { randomBytes } from "node:crypto";
 
 import { decryptBytes, encryptBytes } from "@/lib/crypto";
+import { BACKUP_CHUNK_AAD } from "@/lib/crypto/encrypted-columns";
 
 const MAGIC = Buffer.from("HLBC", "ascii");
 const VERSION = 0x01;
@@ -96,7 +101,7 @@ export function sealBackupChunk(
   header.writeUInt32BE(seq, at);
   at += 4;
   header[at] = last ? FLAG_LAST : 0;
-  return encryptBytes(Buffer.concat([header, payload]));
+  return encryptBytes(Buffer.concat([header, payload]), BACKUP_CHUNK_AAD);
 }
 
 export interface ExpectedChunk {
@@ -117,7 +122,7 @@ export function openBackupChunk(
 ): Buffer {
   let plain: Buffer;
   try {
-    plain = decryptBytes(Buffer.from(sealed));
+    plain = decryptBytes(Buffer.from(sealed), BACKUP_CHUNK_AAD);
   } catch (err) {
     throw new BackupIntegrityError(
       `Piece ${expected.seq} of the stored copy could not be decrypted: ${

src/lib/export/restore-backup.ts

@@ -32,11 +32,9 @@ import { Buffer } from "node:buffer";
 import { prisma, toJson } from "@/lib/db";
 import { auditLog } from "@/lib/auth/audit";
 import {
-  BACKUP_UNDECRYPTABLE_CODE,
-  BACKUP_UNDECRYPTABLE_ERROR,
-} from "@/lib/export/backup-blob";
-import {
+  isStoredBackupReadError,
   openStoredBackup,
+  storedBackupRefusal,
   type StoredBackupRef,
 } from "@/lib/export/stored-backup";
 import {
@@ -354,8 +352,9 @@ export async function restoreBackup(
     // Bad stored input, not a broken server: a copy written under a key the
     // operator has since dropped, or one whose bytes have changed. Refused
     // above the transaction, so nothing was touched.
-    return refused(422, BACKUP_UNDECRYPTABLE_CODE, BACKUP_UNDECRYPTABLE_ERROR, {
-      errorCode: BACKUP_UNDECRYPTABLE_CODE,
+    const refusal = storedBackupRefusal(err);
+    return refused(refusal.status, refusal.code, refusal.message, {
+      errorCode: refusal.code,
     });
   }
 
@@ -381,16 +380,25 @@ export async function restoreBackup(
     raw = streamed.raw;
     payload = parseBackupPayload(raw);
   } catch (err) {
+    const readFailure = isStoredBackupReadError(err);
     await auditLog("admin.backups.restore.failed", {
       userId: input.actorUserId,
       ipAddress: input.ipAddress,
       details: {
         backupId: backup.id,
         ownerId: backup.userId,
-        reason: "schema_invalid",
+        reason: readFailure ? "read_failed" : "schema_invalid",
         message: err instanceof Error ? err.message : String(err),
       },
     });
+    if (readFailure) {
+      // The copy changed after it was opened: replaced by the weekly run, or
+      // altered. Nothing has been deleted yet.
+      const refusal = storedBackupRefusal(err);
+      return refused(refusal.status, refusal.code, refusal.message, {
+        errorCode: refusal.code,
+      });
+    }
     return refused(
       422,
       "schema_invalid",
@@ -2299,6 +2307,14 @@ export async function restoreBackup(
       },
     });
     annotate({ meta: { restoreFailReason: verbose } });
+    if (isStoredBackupReadError(err)) {
+      // The second read, inside the transaction, found the copy replaced or
+      // altered. The transaction rolled back, so nothing was changed.
+      const refusal = storedBackupRefusal(err);
+      return refused(refusal.status, refusal.code, refusal.message, {
+        errorCode: refusal.code,
+      });
+    }
     return refused(
       500,
       "transaction_failed",

src/lib/export/store-backup-blob.ts

@@ -25,12 +25,20 @@
  */
 import type { PrismaClient } from "@/generated/prisma/client";
 
+import { isStreamCiphertext, openStreamDecryptor } from "@/lib/crypto";
 import {
+  BackupBusyError,
   packBackupChunks,
+  packGzipChunks,
+  singleValueToGzip,
   type BackupJsonProducer,
   type PackBackupOptions,
 } from "@/lib/export/backup-blob";
-import { newChunkStreamId } from "@/lib/export/backup-chunks";
+import {
+  BackupIntegrityError,
+  newChunkStreamId,
+} from "@/lib/export/backup-chunks";
+import { BackupFormsConflictError } from "@/lib/export/stored-backup";
 
 /**
  * How long the storing transaction may stay open. The job around it expires
@@ -64,6 +72,17 @@ export interface StoreBackupBlobInput {
   ownerAfterRead?: () => string;
 }
 
+type Tx = Parameters<Parameters<PrismaClient["$transaction"]>[0]>[0];
+
+/** Refuse to touch a copy an unfinished restore is reading. */
+async function assertNotBeingRestored(tx: Tx, backupId: string) {
+  const active = await tx.backupRestoreJob.findFirst({
+    where: { backupId, status: { in: ["queued", "running"] } },
+    select: { id: true },
+  });
+  if (active) throw new BackupBusyError(backupId);
+}
+
 /**
  * Pack `producer`'s JSON into sealed pieces and store them as the
  * `(userId, type)` backup, replacing that backup's previous copy. Resolves to
@@ -89,6 +108,7 @@ export async function storeBackupBlob(
         create: { userId: input.userId, type: input.type, data: null },
         select: { id: true },
       });
+      await assertNotBeingRestored(tx, row.id);
       await tx.dataBackupChunk.deleteMany({ where: { backupId: row.id } });
 
       const streamId = newChunkStreamId();
@@ -104,6 +124,11 @@ export async function storeBackupBlob(
         options,
       );
 
+      // Again at the end: a restore may have been queued while this ran. The
+      // window left is the few milliseconds to the commit, and a restore
+      // started in it finds the copy changed (`backup_changed`) or, if it is
+      // already reading, is told the copy was replaced while being read.
+      await assertNotBeingRestored(tx, row.id);
       await tx.dataBackup.update({
         where: { id: row.id },
         data: {
@@ -118,3 +143,129 @@ export async function storeBackupBlob(
     { timeout: STORE_TRANSACTION_TIMEOUT_MS, maxWait: 60_000 },
   );
 }
+
+/**
+ * The first characters of a single stored value, which carry its key id, or
+ * null when the row no longer holds one. For the key-rotation scan, which must
+ * not read a value that can be a hundred megabytes whole.
+ */
+export async function singleValueBackupHead(
+  prisma: PrismaClient,
+  id: string,
+): Promise<string | null> {
+  const [row] = await prisma.$queryRaw<Array<{ head: string | null }>>`
+    SELECT left(data, 64) AS head FROM data_backups WHERE id = ${id}
+  `;
+  return row?.head ?? null;
+}
+
+/**
+ * How much of a single stored value the conversion reads per query, in
+ * characters. A multiple of four, so every slice of base64 decodes on its own.
+ */
+const CONVERT_SLICE_CHARS = 1024 * 1024;
+
+/**
+ * Turn a copy stored as one value (before v1.39.2) into sealed pieces under
+ * the active key, in place: same row, same date. Resolves to `"gone"` when the
+ * row no longer holds a single value (converted, replaced or deleted since it
+ * was listed).
+ *
+ * Why convert rather than re-seal. Re-sealing a single value under the new key
+ * needs the stored string, its decoded bytes, the plaintext and the new
+ * ciphertext at once, and one value can be a hundred megabytes; key rotation
+ * runs in the app container. Converting reads the value a slice at a time and
+ * writes pieces as it goes.
+ *
+ * The single stream of v1.38.6 to v1.39.1 is read in slices and its tag is checked only at
+ * the end, so the pieces are written before the check has passed. That is why
+ * all of it is one transaction: a copy that fails the check rolls back, and
+ * the row is left exactly as it was. The older forms were written from one
+ * string and fit in one, so they are read whole.
+ */
+export async function convertSingleValueBackup(
+  prisma: PrismaClient,
+  id: string,
+  /** Characters per read; a multiple of four. Tests lower it. */
+  sliceChars: number = CONVERT_SLICE_CHARS,
+): Promise<"converted" | "gone"> {
+  if (sliceChars % 4 !== 0) throw new Error("sliceChars must divide by 4");
+  return prisma.$transaction(
+    async (tx) => {
+      await tx.$executeRawUnsafe(
+        `SET LOCAL idle_in_transaction_session_timeout = '${STORE_IDLE_TIMEOUT}'`,
+      );
+      const [row] = await tx.$queryRaw<
+        Array<{ head: string; len: number; stream_id: string | null }>
+      >`
+        SELECT left(data, 128) AS head, length(data)::int AS len,
+               chunk_stream_id AS stream_id
+        FROM data_backups
+        WHERE id = ${id} AND data IS NOT NULL
+        FOR UPDATE
+      `;
+      if (!row) return "gone";
+      if (row.stream_id !== null) {
+        throw new BackupFormsConflictError();
+      }
+
+      let gz: AsyncIterable<Buffer> | Iterable<Buffer>;
+      if (isStreamCiphertext(row.head)) {
+        const decryptor = openStreamDecryptor(row.head);
+        const slice = async (from: number): Promise<Buffer> => {
+          const [part] = await tx.$queryRaw<Array<{ s: string }>>`
+            SELECT substr(data, ${from + 1}::int, ${sliceChars}::int) AS s
+            FROM data_backups WHERE id = ${id}
+          `;
+          return Buffer.from(part?.s ?? "", "base64");
+        };
+        gz = (async function* () {
+          // The last 16 bytes of the body are the tag, so they are held back
+          // from every slice until the next one shows they were not the end.
+          let held = Buffer.alloc(0);
+          for (let at = decryptor.headerLength; at < row.len;) {
+            const bytes = Buffer.concat([held, await slice(at)]);
+            at += sliceChars;
+            const cut = Math.max(0, bytes.byteLength - 16);
+            held = Buffer.from(bytes.subarray(cut));
+            const out = decryptor.update(bytes.subarray(0, cut));
+            if (out.byteLength > 0) yield out;
+          }
+          if (held.byteLength !== 16) {
+            throw new BackupIntegrityError("The stored copy is truncated.");
+          }
+          const tail = decryptor.final(held);
+          if (tail.byteLength > 0) yield tail;
+        })();
+      } else {
+        const whole = await tx.dataBackup.findUniqueOrThrow({
+          where: { id },
+          select: { data: true },
+        });
+        gz = [singleValueToGzip(whole.data!)];
+      }
+
+      await tx.dataBackupChunk.deleteMany({ where: { backupId: id } });
+      const streamId = newChunkStreamId();
+      const { chunks } = await packGzipChunks(
+        async (sealed, seq) => {
+          await tx.dataBackupChunk.create({
+            data: { backupId: id, seq, data: new Uint8Array(sealed) },
+            select: { id: true },
+          });
+        },
+        streamId,
+        gz,
+        // The copy exists already; converting it must not be refused by the
+        // limit on writing a new one.
+        { maxBytes: Number.MAX_SAFE_INTEGER },
+      );
+      await tx.dataBackup.update({
+        where: { id },
+        data: { data: null, chunkCount: chunks, chunkStreamId: streamId },
+      });
+      return "converted";
+    },
+    { timeout: STORE_TRANSACTION_TIMEOUT_MS, maxWait: 60_000 },
+  );
+}

src/lib/export/stored-backup.ts

@@ -23,7 +23,11 @@ import { Readable, pipeline } from "node:stream";
 import { createGunzip } from "node:zlib";
 
 import type { PrismaClient } from "@/generated/prisma/client";
-import { openBackupBlob } from "@/lib/export/backup-blob";
+import {
+  BACKUP_UNDECRYPTABLE_CODE,
+  BACKUP_UNDECRYPTABLE_ERROR,
+  openBackupBlob,
+} from "@/lib/export/backup-blob";
 import {
   BackupIntegrityError,
   openBackupChunk,
@@ -47,7 +51,43 @@ export interface StoredBackupRef {
   chunkStreamId: string | null;
 }
 
-type ChunkReader = Pick<PrismaClient, "dataBackupChunk">;
+type ChunkReader = Pick<PrismaClient, "dataBackupChunk" | "dataBackup">;
+
+/**
+ * Why a row holding both forms is refused. Only an older release writes a
+ * single value into a row that already has pieces (after a downgrade), and
+ * nothing on the row says which of the two is the newer copy.
+ */
+const BOTH_FORMS_MESSAGE_TEXT =
+  "This backup holds both a single stored value and pieces, which happens " +
+  "when an older HealthLog version wrote to it after an upgrade. There is no " +
+  "way to tell which of the two is current, so it is not read. The next " +
+  "weekly backup replaces it with a new copy.";
+
+export const BOTH_FORMS_MESSAGE = BOTH_FORMS_MESSAGE_TEXT;
+
+/** A row holding both forms (see {@link BOTH_FORMS_MESSAGE}). */
+export class BackupFormsConflictError extends BackupIntegrityError {
+  constructor() {
+    super(BOTH_FORMS_MESSAGE);
+    this.name = "BackupFormsConflictError";
+  }
+}
+
+/**
+ * The copy was replaced by a newer one (the weekly backup ran) while a
+ * restore, preview or download was reading it. Nothing is wrong with either
+ * copy; the reader has to start again on the new one.
+ */
+export class BackupReplacedError extends Error {
+  constructor() {
+    super(
+      "The backup was replaced by a newer one while it was being read. " +
+        "Nothing was changed; start again to use the new copy.",
+    );
+    this.name = "BackupReplacedError";
+  }
+}
 
 /**
  * A value that changes whenever the stored copy is replaced, for the restore
@@ -70,20 +110,34 @@ async function* openedChunks(
   count: number,
 ): AsyncGenerator<Buffer> {
   for (let seq = 0; seq < count; seq++) {
-    const row = await prisma.dataBackupChunk.findUnique({
-      where: { backupId_seq: { backupId, seq } },
-      select: { data: true },
-    });
-    if (!row) {
-      throw new BackupIntegrityError(
-        `Piece ${seq} of ${count} of the stored copy is missing.`,
-      );
+    let piece: Buffer;
+    try {
+      const row = await prisma.dataBackupChunk.findUnique({
+        where: { backupId_seq: { backupId, seq } },
+        select: { data: true },
+      });
+      if (!row) {
+        throw new BackupIntegrityError(
+          `Piece ${seq} of ${count} of the stored copy is missing.`,
+        );
+      }
+      piece = openBackupChunk(row.data, {
+        streamId,
+        seq,
+        last: seq === count - 1,
+      });
+    } catch (err) {
+      // A piece that no longer fits is either tampering or the copy having
+      // been replaced since it was opened. The row tells them apart: a new
+      // copy always has a new stream id.
+      const now = await prisma.dataBackup.findUnique({
+        where: { id: backupId },
+        select: { chunkStreamId: true },
+      });
+      if (now?.chunkStreamId !== streamId) throw new BackupReplacedError();
+      throw err;
     }
-    yield openBackupChunk(row.data, {
-      streamId,
-      seq,
-      last: seq === count - 1,
-    });
+    yield piece;
   }
 }
 
@@ -97,6 +151,14 @@ export async function openStoredBackup(
   prisma: ChunkReader,
   backup: StoredBackupRef,
 ): Promise<BackupSource> {
+  if (backup.data != null && backup.chunkStreamId != null) {
+    throw new BackupFormsConflictError();
+  }
+  if ((backup.chunkStreamId == null) !== (backup.chunkCount == null)) {
+    throw new BackupIntegrityError(
+      "The stored copy lists pieces without saying how many, or the reverse.",
+    );
+  }
   if (backup.chunkStreamId == null || backup.chunkCount == null) {
     if (backup.data == null) {
       throw new BackupIntegrityError("The stored copy has no content.");
@@ -113,6 +175,11 @@ export async function openStoredBackup(
     where: { backupId: id },
   });
   if (stored !== count) {
+    const now = await prisma.dataBackup.findUnique({
+      where: { id },
+      select: { chunkStreamId: true },
+    });
+    if (now?.chunkStreamId !== streamId) throw new BackupReplacedError();
     throw new BackupIntegrityError(
       `The stored copy lists ${count} pieces but ${stored} are stored.`,
     );
@@ -135,3 +202,38 @@ export async function openStoredBackup(
     return gunzip;
   };
 }
+
+/** How a route or the restore job answers a stored copy it could not read. */
+export interface StoredBackupRefusal {
+  status: 409 | 422;
+  /** `backup_changed` or `backup.payload.undecryptable`. */
+  code: "backup_changed" | typeof BACKUP_UNDECRYPTABLE_CODE;
+  message: string;
+}
+
+/**
+ * The answer for a failure to open or read a stored copy. A copy replaced
+ * while it was read is not damage and says so (409, `backup_changed`); a row
+ * holding both forms says why it is not read; everything else is the
+ * undecryptable refusal the routes have always given.
+ */
+export function storedBackupRefusal(err: unknown): StoredBackupRefusal {
+  if (err instanceof BackupReplacedError) {
+    return { status: 409, code: "backup_changed", message: err.message };
+  }
+  return {
+    status: 422,
+    code: BACKUP_UNDECRYPTABLE_CODE,
+    message:
+      err instanceof BackupFormsConflictError
+        ? `${err.message} Nothing was changed.`
+        : BACKUP_UNDECRYPTABLE_ERROR,
+  };
+}
+
+/** True for the failures `storedBackupRefusal` describes, rather than a bug. */
+export function isStoredBackupReadError(err: unknown): boolean {
+  return (
+    err instanceof BackupReplacedError || err instanceof BackupIntegrityError
+  );
+}