Merge pull request #1064 from MBombeck/fix/backup-summary-manifest

Answer the restore preview from counts kept with the backup

src/app/api/admin/backups/[id]/summary/__tests__/route.test.ts

@@ -1,11 +1,14 @@
 /**
- * v1.37.20 — restore preview: the summary read decrypts and validates the
- * stored file through the SAME helpers the restore uses, and only ever
- * reads. Watched red: written against the route before its annotate landed;
- * the not-found and decrypt-failure arms were asserted first.
+ * Restore preview. A copy written with a preview is answered from it without
+ * opening the copy (#1031); a copy from before previews is read whole once,
+ * through the same helpers the restore uses, and what that read found is kept
+ * on the row. The verdicts (schema version, keys) are taken again on every
+ * read. Only a file that fails the schema is a 422 with that message; any
+ * other failure is the server's, not the file's.
  */
 import { describe, it, expect, vi, beforeEach } from "vitest";
 import { NextRequest } from "next/server";
+import { ZodError } from "zod/v4";
 
 vi.mock("@/lib/api-handler", () => ({
   apiHandler: <T extends (...args: unknown[]) => unknown>(fn: T) => fn,
@@ -23,9 +26,15 @@ vi.mock("@/lib/api-handler", () => ({
 vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
 
 const findUniqueMock = vi.fn();
+const updateManyMock = vi.fn();
+const chunkCountMock = vi.fn();
 vi.mock("@/lib/db", () => ({
   prisma: {
-    dataBackup: { findUnique: (...a: unknown[]) => findUniqueMock(...a) },
+    dataBackup: {
+      findUnique: (...a: unknown[]) => findUniqueMock(...a),
+      updateMany: (...a: unknown[]) => updateManyMock(...a),
+    },
+    dataBackupChunk: { count: (...a: unknown[]) => chunkCountMock(...a) },
   },
 }));
 
@@ -34,16 +43,32 @@ const decryptMock = vi.fn();
 // decrypts anything, so a stub of this module has to answer that too.
 vi.mock("@/lib/crypto", () => ({
   decrypt: (...a: unknown[]) => decryptMock(...a),
+  decryptBytes: vi.fn(),
+  getConfiguredKeyIds: () => ["v1"],
   isStreamCiphertext: () => false,
   decryptStream: vi.fn(),
 }));
 
+const openSpy = vi.fn();
+vi.mock("@/lib/export/stored-backup", async (importOriginal) => {
+  const actual =
+    await importOriginal<typeof import("@/lib/export/stored-backup")>();
+  return {
+    ...actual,
+    openStoredBackup: (...a: Parameters<typeof actual.openStoredBackup>) => {
+      openSpy(...a);
+      return actual.openStoredBackup(...a);
+    },
+  };
+});
+
 const parseMock = vi.fn();
 const summarizeMock = vi.fn();
+const compatibleMock = vi.fn((version: string) => version !== "99");
 vi.mock("@/lib/validations/backup", () => ({
   parseBackupPayload: (...a: unknown[]) => parseMock(...a),
   summarizeBackup: (...a: unknown[]) => summarizeMock(...a),
-  isCompatibleSchemaVersion: vi.fn(() => true),
+  isCompatibleSchemaVersion: (version: string) => compatibleMock(version),
   // The measurements are read as a stream and counted one by one.
   BACKUP_SCHEMA_VERSION: "2",
   backupMeasurementSchema: {
@@ -52,52 +77,201 @@ vi.mock("@/lib/validations/backup", () => ({
 }));
 
 import { GET } from "../route";
+import { storedCopyIdentity } from "@/lib/export/backup-preview";
 
 const request = new NextRequest(
   "http://localhost/api/admin/backups/b1/summary",
 );
-const params = { params: Promise.resolve({ id: "b1" }) };
+const params = () => ({ params: Promise.resolve({ id: "b1" }) });
+
+const ROW = {
+  id: "b1",
+  userId: "u1",
+  data: "ciphertext",
+  chunkCount: null,
+  chunkStreamId: null,
+  createdAt: new Date("2026-08-01T00:00:00Z"),
+  user: { id: "u1", username: "self-hoster" },
+};
+
+function storedPreview(overrides: Record<string, unknown> = {}) {
+  return {
+    version: 1,
+    copy: storedCopyIdentity(ROW),
+    summary: {
+      schemaVersion: "2",
+      userId: "u1",
+      exportedAt: "2026-08-01T00:00:00.000Z",
+      measurements: 1_800_000,
+      moodEntries: 3,
+    },
+    keys: [],
+    ...overrides,
+  };
+}
 
 beforeEach(() => {
   vi.clearAllMocks();
-  findUniqueMock.mockResolvedValue({
-    id: "b1",
-    userId: "u1",
-    data: "ciphertext",
-    chunkCount: null,
-    chunkStreamId: null,
-    createdAt: new Date("2026-08-01T00:00:00Z"),
-    user: { id: "u1", username: "self-hoster" },
-  });
+  findUniqueMock.mockResolvedValue({ ...ROW, preview: null });
+  updateManyMock.mockResolvedValue({ count: 1 });
   // Twelve measurements in the file: the preview counts them from the stream.
   decryptMock.mockReturnValue(
     JSON.stringify({
       measurements: Array.from({ length: 12 }, (_, i) => ({ id: `m${i}` })),
     }),
   );
-  parseMock.mockReturnValue({ schemaVersion: 3 });
-  summarizeMock.mockReturnValue({ measurements: 0, moodEntries: 3 });
+  parseMock.mockReturnValue({ schemaVersion: "2" });
+  summarizeMock.mockReturnValue({
+    schemaVersion: "2",
+    userId: "u1",
+    exportedAt: "2026-08-01T00:00:00.000Z",
+    measurements: 0,
+    moodEntries: 3,
+  });
 });
 
-describe("GET /api/admin/backups/[id]/summary", () => {
-  it("answers the file's counts without writing anything", async () => {
-    const res = await GET(request, params);
+describe("GET /api/admin/backups/[id]/summary, from the stored preview", () => {
+  it("answers from the preview without opening the copy", async () => {
+    findUniqueMock.mockResolvedValue({ ...ROW, preview: storedPreview() });
+    const res = await GET(request, params());
     expect(res.status).toBe(200);
     const body = await res.json();
-    expect(body.data.summary).toEqual({ measurements: 12, moodEntries: 3 });
+    expect(body.data.summary).toMatchObject({
+      measurements: 1_800_000,
+      moodEntries: 3,
+    });
     expect(body.data.owner).toBe("self-hoster");
+    expect(openSpy).not.toHaveBeenCalled();
+    expect(decryptMock).not.toHaveBeenCalled();
+    expect(updateManyMock).not.toHaveBeenCalled();
+  });
+
+  it("takes the schema verdict again against this release", async () => {
+    findUniqueMock.mockResolvedValue({
+      ...ROW,
+      preview: storedPreview({
+        summary: {
+          schemaVersion: "99",
+          userId: "u1",
+          exportedAt: "2026-08-01T00:00:00.000Z",
+          measurements: 1,
+        },
+      }),
+    });
+    const res = await GET(request, params());
+    expect(res.status).toBe(422);
+    expect(openSpy).not.toHaveBeenCalled();
+  });
+
+  it("takes the key verdict again against this server's keys", async () => {
+    findUniqueMock.mockResolvedValue({
+      ...ROW,
+      preview: storedPreview({
+        keys: [
+          {
+            keyId: "v7",
+            count: 4,
+            sections: ["moodEntries"],
+            sample: { value: "v7.AAAA", form: "string" },
+          },
+          // Only in the instance settings, which the restore leaves out.
+          {
+            keyId: "v8",
+            count: 1,
+            sections: ["appSettings"],
+            sample: null,
+          },
+        ],
+      }),
+    });
+    const res = await GET(request, params());
+    expect(res.status).toBe(422);
+    expect(await res.json()).toMatchObject({
+      meta: { errorCode: "backup.key.missing", keyIds: ["v7"] },
+    });
+    expect(openSpy).not.toHaveBeenCalled();
+  });
+
+  it("still counts the pieces, and refuses a copy that lost one", async () => {
+    const chunked = {
+      ...ROW,
+      data: null,
+      chunkCount: 3,
+      chunkStreamId: "s1",
+    };
+    findUniqueMock.mockResolvedValue({
+      ...chunked,
+      preview: storedPreview({ copy: storedCopyIdentity(chunked) }),
+    });
+    chunkCountMock.mockResolvedValue(3);
+    expect((await GET(request, params())).status).toBe(200);
+
+    chunkCountMock.mockResolvedValue(2);
+    const res = await GET(request, params());
+    expect(res.status).toBe(422);
+    expect(await res.json()).toMatchObject({
+      meta: { errorCode: "backup.payload.undecryptable" },
+    });
+    expect(openSpy).not.toHaveBeenCalled();
+  });
+
+  it("ignores a preview that describes another copy", async () => {
+    findUniqueMock.mockResolvedValue({
+      ...ROW,
+      preview: storedPreview({ copy: "chunks:other:3" }),
+    });
+    const res = await GET(request, params());
+    expect(res.status).toBe(200);
+    expect((await res.json()).data.summary.measurements).toBe(12);
+    expect(openSpy).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe("GET /api/admin/backups/[id]/summary, a copy without a preview", () => {
+  it("reads the copy once and keeps what it found on the row", async () => {
+    const res = await GET(request, params());
+    expect(res.status).toBe(200);
+    const body = await res.json();
+    expect(body.data.summary).toMatchObject({
+      measurements: 12,
+      moodEntries: 3,
+    });
+    expect(openSpy).toHaveBeenCalledTimes(1);
+    await vi.waitFor(() => expect(updateManyMock).toHaveBeenCalledTimes(1));
+    const [call] = updateManyMock.mock.calls[0] as [
+      { where: Record<string, unknown>; data: { preview: { copy: string } } },
+    ];
+    // Only onto the copy it was read from.
+    expect(call.where).toEqual({
+      id: "b1",
+      chunkStreamId: null,
+      chunkCount: null,
+    });
+    expect(call.data.preview.copy).toBe(storedCopyIdentity(ROW));
+  });
+
+  it("shares one read between requests for the same copy", async () => {
+    const [a, b] = await Promise.all([
+      GET(request, params()),
+      GET(request, params()),
+    ]);
+    expect(a.status).toBe(200);
+    expect(b.status).toBe(200);
+    expect(openSpy).toHaveBeenCalledTimes(1);
   });
 
   it("404s an unknown backup id", async () => {
     findUniqueMock.mockResolvedValue(null);
-    await expect(GET(request, params)).rejects.toMatchObject({ status: 404 });
+    await expect(GET(request, params())).rejects.toMatchObject({
+      status: 404,
+    });
   });
 
   it("refuses a file it cannot decrypt with the documented 422", async () => {
     decryptMock.mockImplementation(() => {
       throw new Error("bad key");
     });
-    const res = await GET(request, params);
+    const res = await GET(request, params());
     // Bad stored input — a key dropped from `ENCRYPTION_KEYS`, or bytes that
     // are not the ones written — and not a fault in this process, so it does
     // not answer 500 and does not reach the error reporter as one.
@@ -105,13 +279,42 @@ describe("GET /api/admin/backups/[id]/summary", () => {
     expect(await res.json()).toMatchObject({
       meta: { errorCode: "backup.payload.undecryptable" },
     });
+    expect(updateManyMock).not.toHaveBeenCalled();
   });
 
   it("422s a file that fails schema validation", async () => {
     parseMock.mockImplementation(() => {
-      throw new Error("nope");
+      throw new ZodError([]);
     });
-    const res = await GET(request, params);
+    const res = await GET(request, params());
     expect(res.status).toBe(422);
+    expect((await res.json()).error).toBe(
+      "Backup payload failed schema validation",
+    );
+    expect(updateManyMock).not.toHaveBeenCalled();
+  });
+
+  it("422s a file that is not JSON", async () => {
+    decryptMock.mockReturnValue("{ not json");
+    const res = await GET(request, params());
+    expect(res.status).toBe(422);
+    expect((await res.json()).error).toBe(
+      "Backup payload failed schema validation",
+    );
+  });
+
+  it("does not call a server failure a schema failure", async () => {
+    parseMock.mockImplementation(() => {
+      throw new Error("out of memory");
+    });
+    // Thrown to the handler, which answers 500 and reports it.
+    await expect(GET(request, params())).rejects.toThrow("out of memory");
+    expect(updateManyMock).not.toHaveBeenCalled();
+  });
+
+  it("still answers when keeping the preview fails", async () => {
+    updateManyMock.mockRejectedValue(new Error("db gone"));
+    const res = await GET(request, params());
+    expect(res.status).toBe(200);
   });
 });

src/app/api/admin/backups/[id]/summary/route.ts

@@ -4,33 +4,57 @@
  *
  * The restore dialog used to ask for the typed confirmation on nothing but
  * a filename and a date; the counts only appeared in the audit row after
- * the rows were already replaced. This read decrypts and schema-validates
- * the stored file exactly the way the restore itself does (same decrypt,
- * same `parseBackupPayload`, same compatibility gate) and answers with the
- * `summarizeBackup` counts — so what the operator confirms is what the
- * restore will write, derived from the same code path. Read-only; nothing
- * is touched.
+ * the rows were already replaced. This answers with the `summarizeBackup`
+ * counts of the stored copy, and refuses the way the restore would refuse:
+ * a schema version this release cannot restore, or a key the copy's inner
+ * ciphertext needs and this server does not have.
+ *
+ * The counts are worked out while the copy is written and kept on its row
+ * (`DataBackup.preview`, `src/lib/export/backup-preview.ts`), so this answers
+ * without opening the copy. Reading the copy here took two minutes for an
+ * account of 1.8 million readings on slow hardware, long after the dialog had
+ * given up (#1031). Both verdicts are taken again on every read, from what
+ * the preview keeps, because the release and the server's keys can both
+ * change after the copy was written. The copy's pieces are still counted
+ * against the row (no piece is read), so a copy that lost one since is
+ * refused here, as the restore and the download refuse it.
+ *
+ * A copy written before previews existed has none. That one is read whole,
+ * once: the preview worked out from it is stored on the row, so the next
+ * request answers at once, and concurrent requests for the same copy share
+ * the one read.
  */
 import { NextRequest } from "next/server";
+import { ZodError } from "zod/v4";
 
+import { Prisma } from "@/generated/prisma/client";
 import { prisma } from "@/lib/db";
 import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
 import { apiError, apiSuccess } from "@/lib/api-response";
 import {
+  checkStoredBackupShape,
   isStoredBackupReadError,
   openStoredBackup,
   storedBackupRefusal,
   STORED_BACKUP_SELECT,
+  type StoredBackupRef,
 } from "@/lib/export/stored-backup";
 import {
   assessBackupKeys,
   BACKUP_KEY_MISSING_CODE,
+  BackupKeyIdCollector,
   describeBackupKeyProblem,
 } from "@/lib/export/backup-key-ids";
+import { BackupJsonError } from "@/lib/export/backup-json-scan";
+import {
+  buildBackupPreview,
+  storedCopyIdentity,
+  storedPreviewFor,
+  type BackupPreview,
+} from "@/lib/export/backup-preview";
 import {
   readStreamedBackup,
-  type BackupSource,
-  type StreamedBackup,
+  StreamedBackupInvalidError,
 } from "@/lib/export/streamed-backup";
 import { annotate } from "@/lib/logging/context";
 import {
@@ -43,6 +67,114 @@ export const dynamic = "force-dynamic";
 
 type RouteParams = { params: Promise<{ id: string }> };
 
+/** Why a copy could not be summarised, as the route answers it. */
+type Refusal = { status: number; message: string; code?: string };
+
+type ReadOutcome = { preview: BackupPreview } | { refusal: Refusal };
+
+/**
+ * Reads of a copy without a preview that are running now, by copy. A second
+ * request for the same copy (the dialog opened again while the first read is
+ * still going) waits for the first instead of reading it all again.
+ */
+const inFlight = new Map<string, Promise<ReadOutcome>>();
+
+/** Whether a failure is the file itself not being a backup this reads. */
+function isPayloadInvalid(err: unknown): boolean {
+  return (
+    err instanceof BackupJsonError ||
+    err instanceof StreamedBackupInvalidError ||
+    err instanceof ZodError
+  );
+}
+
+/**
+ * Read a copy whole and work out its preview: the way every preview was
+ * taken before previews were stored. Throws for anything that is neither a
+ * read refusal nor a file that fails the schema, which the handler answers
+ * as a server fault.
+ */
+async function readPreview(
+  backup: StoredBackupRef,
+  copy: string,
+): Promise<ReadOutcome> {
+  const refused = (err: unknown): ReadOutcome => {
+    const refusal = storedBackupRefusal(err);
+    return {
+      refusal: {
+        status: refusal.status,
+        message: refusal.message,
+        code: refusal.code,
+      },
+    };
+  };
+
+  // Opened and read as a stream: a large record's JSON is longer than any
+  // string V8 can hold (#1031).
+  let source;
+  try {
+    source = await openStoredBackup(prisma, backup);
+  } catch (err) {
+    // Same refusal the restore and the download give, for the same reason:
+    // a copy this instance cannot open is bad stored input, and the preview
+    // is the first place an operator meets it.
+    return refused(err);
+  }
+
+  try {
+    const streamed = await readStreamedBackup(source);
+    const payload = parseBackupPayload(streamed.raw);
+    const summary = {
+      ...summarizeBackup(payload),
+      measurements: streamed.measurementCount,
+    };
+    return { preview: buildBackupPreview(copy, summary, streamed.keys) };
+  } catch (err) {
+    if (isStoredBackupReadError(err)) return refused(err);
+    // Only a file that is not a backup this release reads is a 422. Anything
+    // else is this server failing, not the file, and is answered as such.
+    if (isPayloadInvalid(err)) {
+      return {
+        refusal: {
+          status: 422,
+          message: "Backup payload failed schema validation",
+        },
+      };
+    }
+    throw err;
+  }
+}
+
+/**
+ * Keep a preview worked out by reading the copy, so the next request answers
+ * without reading it. Written only while the row still holds the copy it was
+ * read from; a failure here costs the next request the same read, nothing
+ * more, so it is noted and not raised.
+ */
+async function keepPreview(
+  backup: StoredBackupRef,
+  preview: BackupPreview,
+): Promise<void> {
+  try {
+    await prisma.dataBackup.updateMany({
+      where: {
+        id: backup.id,
+        chunkStreamId: backup.chunkStreamId,
+        chunkCount: backup.chunkCount,
+      },
+      data: { preview: preview as unknown as Prisma.InputJsonValue },
+    });
+  } catch (err) {
+    annotate({
+      action: { name: "admin.backups.summary.keep_failed" },
+      meta: {
+        backupId: backup.id,
+        error: err instanceof Error ? err.name : "unknown",
+      },
+    });
+  }
+}
+
 export const GET = apiHandler(
   async (_request: NextRequest, { params }: RouteParams) => {
     const { user: admin } = await requireAdmin();
@@ -53,6 +185,7 @@ export const GET = apiHandler(
       where: { id },
       select: {
         ...STORED_BACKUP_SELECT,
+        preview: true,
         createdAt: true,
         user: { select: { id: true, username: true } },
       },
@@ -61,41 +194,58 @@ export const GET = apiHandler(
       throw new HttpError(404, "Backup not found");
     }
 
-    // Opened and read as a stream: a large record's JSON is longer than any
-    // string V8 can hold (#1031).
-    let source: BackupSource;
-    try {
-      source = await openStoredBackup(prisma, backup);
-    } catch (err) {
-      // Same refusal the restore and the download give, for the same reason:
-      // a copy this instance cannot open is bad stored input, and the preview
-      // is the first place an operator meets it.
-      const refusal = storedBackupRefusal(err);
-      return apiError(refusal.message, refusal.status, {
-        errorCode: refusal.code,
-      });
-    }
-
-    let payload;
-    let measurementCount: number;
-    let streamed: StreamedBackup;
-    try {
-      streamed = await readStreamedBackup(source);
-      measurementCount = streamed.measurementCount;
-      payload = parseBackupPayload(streamed.raw);
-    } catch (err) {
-      if (isStoredBackupReadError(err)) {
+    let preview = storedPreviewFor(backup.preview, backup);
+    const fromStore = preview !== null;
+    if (preview) {
+      // The counts were kept; the copy's pieces are still counted, so a copy
+      // that lost one since is refused here as the restore would refuse it.
+      // Tampering inside a piece is found by the restore's own open, before
+      // it deletes anything.
+      try {
+        await checkStoredBackupShape(prisma, backup);
+      } catch (err) {
+        if (!isStoredBackupReadError(err)) throw err;
         const refusal = storedBackupRefusal(err);
-        return apiError(refusal.message, refusal.status, {
-          errorCode: refusal.code,
+        return refusalResponse({
+          status: refusal.status,
+          message: refusal.message,
+          code: refusal.code,
         });
       }
-      return apiError("Backup payload failed schema validation", 422);
+    }
+    if (!preview) {
+      const copy = storedCopyIdentity(backup);
+      if (copy === null) {
+        // A row holding both forms, or neither: the open refuses it, with
+        // the reason.
+        const outcome = await readPreview(backup, "");
+        if ("refusal" in outcome) return refusalResponse(outcome.refusal);
+        throw new Error("A stored backup with no nameable copy was read");
+      }
+      const flight = `${backup.id}:${copy}`;
+      let pending = inFlight.get(flight);
+      if (!pending) {
+        pending = readPreview(backup, copy).finally(() => {
+          inFlight.delete(flight);
+        });
+        inFlight.set(flight, pending);
+        // Kept whatever the verdicts below say: they are taken again on
+        // every read, and the counts do not change.
+        void pending.then(
+          (outcome) =>
+            "preview" in outcome ? keepPreview(backup, outcome.preview) : null,
+          () => null,
+        );
+      }
+      const outcome = await pending;
+      if ("refusal" in outcome) return refusalResponse(outcome.refusal);
+      preview = outcome.preview;
     }
 
-    if (!isCompatibleSchemaVersion(payload.schemaVersion)) {
+    const { summary } = preview;
+    if (!isCompatibleSchemaVersion(summary.schemaVersion)) {
       return apiError(
-        `Backup schema version ${payload.schemaVersion} is not restorable by this release`,
+        `Backup schema version ${summary.schemaVersion} is not restorable by this release`,
         422,
       );
     }
@@ -105,9 +255,10 @@ export const GET = apiHandler(
     // does not have. The instance settings are left out here, as the restore
     // leaves them out unless asked; the restore itself checks them when they
     // are asked for.
-    const keyVerdict = assessBackupKeys(streamed.keys, {
-      ignoreSections: new Set(["appSettings"]),
-    });
+    const keyVerdict = assessBackupKeys(
+      BackupKeyIdCollector.fromStored(preview.keys),
+      { ignoreSections: new Set(["appSettings"]) },
+    );
     const keyProblem = describeBackupKeyProblem(keyVerdict);
     if (keyProblem) {
       return apiError(keyProblem, 422, {
@@ -116,14 +267,9 @@ export const GET = apiHandler(
       });
     }
 
-    const summary = {
-      ...summarizeBackup(payload),
-      measurements: measurementCount,
-    };
-
     annotate({
       action: { name: "admin.backups.summary" },
-      meta: { backupId: id, ownerId: backup.userId },
+      meta: { backupId: id, ownerId: backup.userId, fromStore },
     });
 
     return apiSuccess({
@@ -133,3 +279,11 @@ export const GET = apiHandler(
     });
   },
 );
+
+function refusalResponse(refusal: Refusal) {
+  return apiError(
+    refusal.message,
+    refusal.status,
+    refusal.code ? { errorCode: refusal.code } : undefined,
+  );
+}