/** * A backup restore runs as a background job, end to end. * * The restore of a large account outlived a reverse proxy's 60-second request * limit, so the operator saw an error for a restore that went on to finish. * The request now answers 202 with a job id, the `backup-restore` queue runs * the restore, and `GET /api/admin/backups/restores/{id}` reports it. Pinned * here against a real Postgres: * * 1. upload → 202 → the job queued → run → the status route says succeeded, * with the counts, and the rows are back; * 2. a copy that cannot be decrypted is refused in the request itself, and * no job is created; * 3. a restore that fails part-way ends `failed` with a reason, and the * account is exactly as it was (the transaction rolled back); * 4. a second restore of the same account while one is queued answers 409, * and two requests racing each other admit exactly one; * 5. a job whose worker stopped is queued again by the sweep and then * finishes; after the second interruption it fails instead of looping; * the listing a reloaded page reads shows the job while it runs; * 6. a copy replaced while the job waited is not restored in its place; * 7. a file whose transaction would outlast the job's budget is refused * before anything is deleted. * * Mutation checks (each run once, each turned the named case red): * - drop the partial unique index from migration 0347: case 4's race admits * two jobs; * - skip the digest comparison in `runBackupRestoreJob`: case 6 restores the * replaced copy; * - remove the deadline check in `restoreBackup`: case 7 deletes the rows; * - drop the attempts cap in `sweepInterruptedRestores`: case 5's third * interruption is queued again. */ import type { PgBoss } from "pg-boss"; import { gzipSync } from "node:zlib"; import { beforeEach, describe, expect, it, vi } from "vitest"; process.env.ENCRYPTION_KEY ??= "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; import { encrypt } from "@/lib/crypto"; import { setGlobalBoss } from "@/lib/jobs/boss-instance"; import { runJob } from "@/lib/jobs/run-job"; import { BACKUP_RESTORE_MAX_ATTEMPTS, handleBackupRestore, runBackupRestoreJob, sweepInterruptedRestores, } from "@/lib/jobs/backup-restore"; import { readBackupRestoreJob } from "@/lib/jobs/backup-restore-view"; import { BACKUP_RESTORE_QUEUE } from "@/lib/jobs/backup-restore-queue"; import { backupPayloadSchema } from "@/lib/validations/backup"; import { cookieJar, headerJar } from "./mock-next-headers"; import { getPrismaClient, truncateAllTables } from "./setup"; vi.mock("next/headers", async () => { const { cookieJar, headerJar } = await import("./mock-next-headers"); return { headers: vi.fn(async () => ({ get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, })), cookies: vi.fn(async () => ({ get: (name: string) => { const value = cookieJar.get(name); return value ? { name, value } : undefined; }, set: (name: string, value: string) => { cookieJar.set(name, value); }, delete: (name: string) => { cookieJar.delete(name); }, })), }; }); vi.mock("@/lib/db-compat", () => ({ ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), })); /** A queue that records what it is sent and what it is told is finished. */ function recordingBoss() { const sent: Array<{ queue: string; data: unknown }> = []; const completed: string[] = []; let next = 0; const boss = { send: vi.fn(async (queue: string, data: unknown) => { sent.push({ queue, data }); next += 1; return `boss-${next}`; }), complete: vi.fn(async (_queue: string, id: string) => { completed.push(id); return {}; }), } as unknown as PgBoss; return { boss, sent, completed }; } let queue: ReturnType; beforeEach(async () => { await truncateAllTables(getPrismaClient()); cookieJar.clear(); headerJar.clear(); queue = recordingBoss(); setGlobalBoss(queue.boss); }); async function seedAdmin(username = "restore-job-admin") { const prisma = getPrismaClient(); const admin = await prisma.user.create({ data: { username, email: `${username}@example.test`, role: "ADMIN" }, }); const session = await prisma.session.create({ data: { userId: admin.id, expiresAt: new Date(Date.now() + 600_000) }, }); cookieJar.set("healthlog_session", session.id); return admin; } function payloadFor(userId: string, measurementIds: string[]) { return backupPayloadSchema.parse({ schemaVersion: "2", exportedAt: "2026-09-20T00:00:00.000Z", userId, measurements: measurementIds.map((id, i) => ({ id, type: "PULSE", value: 60 + i, unit: "bpm", measuredAt: new Date(Date.UTC(2026, 8, 1, 7, i)).toISOString(), source: "MANUAL", })), }); } async function storeBackup(userId: string, measurementIds: string[]) { return getPrismaClient().dataBackup.create({ data: { userId, type: `MANUAL_UPLOAD_${Date.now()}_${Math.random()}`, data: encrypt(JSON.stringify(payloadFor(userId, measurementIds))), }, }); } async function seedMeasurements(userId: string, ids: string[]) { await getPrismaClient().measurement.createMany({ data: ids.map((id, i) => ({ id, userId, type: "WEIGHT" as const, value: 80 + i, unit: "kg", measuredAt: new Date(Date.UTC(2026, 7, 1, 7, i)), })), }); } async function requestRestore(backupId: string, key?: string) { const { POST } = await import("@/app/api/admin/backups/[id]/restore/route"); return POST( new Request(`http://localhost/api/admin/backups/${backupId}/restore`, { method: "POST", headers: { "content-type": "application/json", ...(key ? { "idempotency-key": key } : {}), }, body: JSON.stringify({ confirm: "RESTORE" }), }) as never, { params: Promise.resolve({ id: backupId }) }, ); } async function readStatus(jobId: string) { const { GET } = await import("@/app/api/admin/backups/restores/[jobId]/route"); const res = await GET( new Request( `http://localhost/api/admin/backups/restores/${jobId}`, ) as never, { params: Promise.resolve({ jobId }) }, ); return res; } async function listStatus() { const { GET } = await import("@/app/api/admin/backups/restores/route"); // The listing handler declares no parameters; `apiHandler` reads the // request defensively, so calling it bare is what a GET without a body is. const res = await GET(); return (await res.json()) as { data: { jobs: Array<{ id: string; status: string }> }; }; } async function measurementIdsOf(userId: string) { const rows = await getPrismaClient().measurement.findMany({ where: { userId }, orderBy: { id: "asc" }, select: { id: true }, }); return rows.map((row) => row.id); } describe("backup restore as a background job", () => { it("upload → 202 → queued → run → succeeded, and the rows are back", async () => { const admin = await seedAdmin(); await seedMeasurements(admin.id, ["live-1", "live-2"]); const file = Buffer.from( JSON.stringify(payloadFor(admin.id, ["m-a", "m-b", "m-c"])), ); const { POST: upload } = await import("@/app/api/admin/backups/upload/route"); const uploaded = await upload( new Request("http://localhost/api/admin/backups/upload", { method: "POST", headers: { "content-type": "application/gzip" }, body: gzipSync(file), }) as never, ); expect(uploaded.status).toBe(201); const { data: stored } = (await uploaded.json()) as { data: { id: string }; }; const queued = await requestRestore(stored.id); expect(queued.status).toBe(202); const { data } = (await queued.json()) as { data: { jobId: string; status: string; statusUrl: string }; }; expect(data.status).toBe("queued"); expect(data.statusUrl).toBe(`/api/admin/backups/restores/${data.jobId}`); // The request queued the job and changed nothing yet. expect(queue.sent).toEqual([ { queue: BACKUP_RESTORE_QUEUE, data: { restoreJobId: data.jobId } }, ]); expect(await measurementIdsOf(admin.id)).toEqual(["live-1", "live-2"]); const waiting = (await (await readStatus(data.jobId)).json()) as { data: { status: string }; }; expect(waiting.data.status).toBe("queued"); // Through the same wrapper the worker binds, so the outcome pg-boss // persists is validated too, not only the handler's return value. const persisted = await runJob( BACKUP_RESTORE_QUEUE, handleBackupRestore, )([ { id: "boss-1", name: BACKUP_RESTORE_QUEUE, data: { restoreJobId: data.jobId }, expireInSeconds: 7200, signal: new AbortController().signal, } as never, ]); expect(persisted).toEqual({ ok: true, did: { restore_measurements: 3, restore_skipped_links: 0 }, }); const finished = await readStatus(data.jobId); expect(finished.status).toBe(200); const body = (await finished.json()) as { data: { status: string; phase: string | null; attempts: number; progress: { measurementsWritten: number; measurementsTotal: number }; result: { summary: { measurements: number }; cleared: { measurements: number }; }; failure: unknown; }; }; expect(body.data.status).toBe("succeeded"); expect(body.data.phase).toBeNull(); expect(body.data.attempts).toBe(1); expect(body.data.failure).toBeNull(); expect(body.data.result.summary.measurements).toBe(3); expect(body.data.result.cleared.measurements).toBe(2); expect(body.data.progress.measurementsTotal).toBe(3); expect(body.data.progress.measurementsWritten).toBe(3); expect(await measurementIdsOf(admin.id)).toEqual(["m-a", "m-b", "m-c"]); // The trail says who queued it and that it ran. const actions = ( await getPrismaClient().auditLog.findMany({ where: { userId: admin.id }, select: { action: true }, }) ).map((row) => row.action); expect(actions).toEqual( expect.arrayContaining([ "admin.backups.restore.queued", "admin.backups.restore.start", "admin.backups.restore", ]), ); }); it("refuses a copy that cannot be decrypted in the request, and queues nothing", async () => { const admin = await seedAdmin(); const backup = await getPrismaClient().dataBackup.create({ data: { userId: admin.id, type: "MANUAL_UPLOAD_1", data: "v1:not-a-real-ciphertext", }, }); const res = await requestRestore(backup.id); expect(res.status).toBe(422); const body = (await res.json()) as { meta?: { errorCode?: string } }; expect(body.meta?.errorCode).toBe("backup.payload.undecryptable"); expect(queue.sent).toEqual([]); expect(await getPrismaClient().backupRestoreJob.count()).toBe(0); }); it("a restore that fails part-way ends failed with a reason, and nothing changed", async () => { const admin = await seedAdmin(); await seedMeasurements(admin.id, ["live-1"]); // Another account already owns this id, so the insert fails AFTER the // restore deleted the owner's rows, inside the transaction. const other = await getPrismaClient().user.create({ data: { username: "someone-else", email: "else@example.test" }, }); await seedMeasurements(other.id, ["taken"]); const backup = await storeBackup(admin.id, ["m-a", "taken"]); const queued = await requestRestore(backup.id); expect(queued.status).toBe(202); const { data } = (await queued.json()) as { data: { jobId: string } }; const outcome = await runBackupRestoreJob(data.jobId); // A transaction that could not be written is a fault the operator's job // card should show, not a quiet success. expect(outcome.ok).toBe(false); const job = await readBackupRestoreJob(data.jobId); expect(job?.status).toBe("failed"); expect(job?.failure?.code).toBe("transaction_failed"); expect(job?.failure?.message).toContain("Nothing was changed"); expect(await measurementIdsOf(admin.id)).toEqual(["live-1"]); expect(await measurementIdsOf(other.id)).toEqual(["taken"]); // The staged copy stays, so a retry needs no new upload. expect( await getPrismaClient().dataBackup.findUnique({ where: { id: backup.id }, }), ).not.toBeNull(); }); it("refuses a second restore of the same account while one is queued", async () => { const admin = await seedAdmin(); const first = await storeBackup(admin.id, ["m-a"]); const second = await storeBackup(admin.id, ["m-b"]); const accepted = await requestRestore(first.id); expect(accepted.status).toBe(202); const { data } = (await accepted.json()) as { data: { jobId: string } }; const refused = await requestRestore(second.id); expect(refused.status).toBe(409); const body = (await refused.json()) as { meta?: { errorCode?: string; jobId?: string }; }; expect(body.meta?.errorCode).toBe("backup.restore.active"); expect(body.meta?.jobId).toBe(data.jobId); // Once it has finished, the account can be restored again. await runBackupRestoreJob(data.jobId); expect((await requestRestore(second.id)).status).toBe(202); }); it("two requests racing for one account admit exactly one", async () => { const admin = await seedAdmin(); const first = await storeBackup(admin.id, ["m-a"]); const second = await storeBackup(admin.id, ["m-b"]); const statuses = ( await Promise.all([ requestRestore(first.id, "race-a"), requestRestore(second.id, "race-b"), ]) ) .map((res) => res.status) .sort(); expect(statuses).toEqual([202, 409]); expect( await getPrismaClient().backupRestoreJob.count({ where: { status: { in: ["queued", "running"] } }, }), ).toBe(1); }); it("picks up a restore a stopped worker left running, and stops after the second time", async () => { const admin = await seedAdmin(); await seedMeasurements(admin.id, ["live-1"]); const backup = await storeBackup(admin.id, ["m-a", "m-b"]); const prisma = getPrismaClient(); const queued = await requestRestore(backup.id); const { data } = (await queued.json()) as { data: { jobId: string } }; // What a worker that died mid-restore leaves behind: the row says running, // its heartbeat stopped minutes ago, the transaction rolled back with the // process, so the account is untouched. const stale = new Date(Date.now() - 10 * 60_000); await prisma.backupRestoreJob.update({ where: { id: data.jobId }, data: { status: "running", phase: "measurements", attempts: 1, heartbeatAt: stale, startedAt: stale, }, }); // A page reloaded now finds the job and keeps polling it. expect((await listStatus()).data.jobs).toEqual([ expect.objectContaining({ id: data.jobId, status: "running" }), ]); const swept = await sweepInterruptedRestores(queue.boss); expect(swept).toEqual({ requeued: 1, failed: 0 }); // The delivery the dead worker held is finished, not left to time out. expect(queue.completed).toEqual(["boss-1"]); expect(queue.sent.at(-1)).toEqual({ queue: BACKUP_RESTORE_QUEUE, data: { restoreJobId: data.jobId }, }); expect((await readBackupRestoreJob(data.jobId))?.status).toBe("queued"); await runBackupRestoreJob(data.jobId); const resumed = await readBackupRestoreJob(data.jobId); expect(resumed?.status).toBe("succeeded"); expect(resumed?.attempts).toBe(2); expect(await measurementIdsOf(admin.id)).toEqual(["m-a", "m-b"]); // A job that has already been started the maximum number of times fails // instead of being queued a third time. const again = await requestRestore(backup.id); const { data: next } = (await again.json()) as { data: { jobId: string } }; await prisma.backupRestoreJob.update({ where: { id: next.jobId }, data: { status: "running", attempts: BACKUP_RESTORE_MAX_ATTEMPTS, heartbeatAt: stale, }, }); expect(await sweepInterruptedRestores(queue.boss)).toEqual({ requeued: 0, failed: 1, }); const gaveUp = await readBackupRestoreJob(next.jobId); expect(gaveUp?.status).toBe("failed"); expect(gaveUp?.failure?.code).toBe("interrupted"); }); it("never runs a committed restore again when its worker stopped in the rebuild", async () => { const admin = await seedAdmin(); const backup = await storeBackup(admin.id, ["m-a", "m-b"]); const prisma = getPrismaClient(); // Run the restore for real, so the commit marker is written by the code // that writes it, then put the row back the way a worker that died in the // rollup rebuild leaves it: running, phase rebuilding, heartbeat stale. const queued = await requestRestore(backup.id); const { data } = (await queued.json()) as { data: { jobId: string } }; await runBackupRestoreJob(data.jobId); const committed = await prisma.backupRestoreJob.findUniqueOrThrow({ where: { id: data.jobId }, }); expect(committed.committedAt).not.toBeNull(); const stale = new Date(Date.now() - 10 * 60_000); await prisma.backupRestoreJob.update({ where: { id: data.jobId }, data: { status: "running", phase: "rebuilding", heartbeatAt: stale, completedAt: null, result: undefined, }, }); // A reading the person logged after the restore committed. await seedMeasurements(admin.id, ["after-commit"]); expect(await sweepInterruptedRestores(queue.boss)).toEqual({ requeued: 0, failed: 1, }); const job = await readBackupRestoreJob(data.jobId); expect(job?.status).toBe("failed"); expect(job?.failure?.code).toBe("failed_after_commit"); expect(job?.failure?.message).not.toContain("Nothing was changed"); // Not queued again, so the reading from after the commit survives. expect( queue.sent.filter( (entry) => (entry.data as { restoreJobId?: string }).restoreJobId === data.jobId, ), ).toHaveLength(1); expect(await measurementIdsOf(admin.id)).toEqual([ "after-commit", "m-a", "m-b", ]); }); it("the admission check closes a committed stale job as restored, not rolled back", async () => { const admin = await seedAdmin(); const backup = await storeBackup(admin.id, ["m-a"]); const prisma = getPrismaClient(); const queued = await requestRestore(backup.id); const { data } = (await queued.json()) as { data: { jobId: string } }; const stale = new Date(Date.now() - 10 * 60_000); await prisma.backupRestoreJob.update({ where: { id: data.jobId }, data: { status: "running", phase: "rebuilding", attempts: 1, heartbeatAt: stale, committedAt: stale, }, }); expect((await requestRestore(backup.id)).status).toBe(202); const job = await readBackupRestoreJob(data.jobId); expect(job?.status).toBe("failed"); expect(job?.failure?.code).toBe("failed_after_commit"); }); it("dates a medication from a file without a creation date by its earliest dose", async () => { const admin = await seedAdmin(); const prisma = getPrismaClient(); // A portable file from before v1.39.1: no medication createdAt. const payload = backupPayloadSchema.parse({ schemaVersion: "1", exportedAt: "2026-09-20T00:00:00.000Z", userId: admin.id, medications: [{ name: "Ramipril", dose: "5mg", schedules: [] }], intakeEvents: [ { medication: "Ramipril", scheduledFor: "2025-03-10T08:00:00.000Z", takenAt: "2025-03-10T08:04:00.000Z", }, { medication: "Ramipril", scheduledFor: "2025-03-09T08:00:00.000Z", autoMissed: true, }, ], }); const backup = await prisma.dataBackup.create({ data: { userId: admin.id, type: "MANUAL_UPLOAD_OLD_PORTABLE", data: encrypt(JSON.stringify(payload)), }, }); const queued = await requestRestore(backup.id); const { data } = (await queued.json()) as { data: { jobId: string } }; await runBackupRestoreJob(data.jobId); expect((await readBackupRestoreJob(data.jobId))?.status).toBe("succeeded"); const medication = await prisma.medication.findFirstOrThrow({ where: { userId: admin.id }, }); // Not the restore time: the dose history reads this as the day the // medication began, and every miss before it would vanish. expect(medication.createdAt.toISOString()).toBe("2025-03-09T08:00:00.000Z"); }); it("a live job's heartbeat keeps it out of the sweep", async () => { const admin = await seedAdmin(); const backup = await storeBackup(admin.id, ["m-a"]); const queued = await requestRestore(backup.id); const { data } = (await queued.json()) as { data: { jobId: string } }; await getPrismaClient().backupRestoreJob.update({ where: { id: data.jobId }, data: { status: "running", attempts: 1, heartbeatAt: new Date() }, }); expect(await sweepInterruptedRestores(queue.boss)).toEqual({ requeued: 0, failed: 0, }); expect((await readBackupRestoreJob(data.jobId))?.status).toBe("running"); // And a second request for the account is still refused while it runs. expect((await requestRestore(backup.id)).status).toBe(409); }); it("does not restore a copy that was replaced while the job waited", async () => { const admin = await seedAdmin(); await seedMeasurements(admin.id, ["live-1"]); const backup = await storeBackup(admin.id, ["m-a"]); const queued = await requestRestore(backup.id); const { data } = (await queued.json()) as { data: { jobId: string } }; // The weekly job overwrites its row in place. await getPrismaClient().dataBackup.update({ where: { id: backup.id }, data: { data: encrypt(JSON.stringify(payloadFor(admin.id, ["m-newer"]))), }, }); const persisted = await runJob( BACKUP_RESTORE_QUEUE, handleBackupRestore, )([ { id: "boss-1", name: BACKUP_RESTORE_QUEUE, data: { restoreJobId: data.jobId }, expireInSeconds: 7200, signal: new AbortController().signal, } as never, ]); // A refusal is the job doing what it was asked, not a crash. expect(persisted).toEqual({ ok: true, did: { refused: "backup_changed" } }); const job = await readBackupRestoreJob(data.jobId); expect(job?.status).toBe("failed"); expect(job?.failure?.code).toBe("backup_changed"); expect(await measurementIdsOf(admin.id)).toEqual(["live-1"]); }); it("refuses, before deleting anything, a file that would outlast the job's budget", async () => { const admin = await seedAdmin(); await seedMeasurements(admin.id, ["live-1"]); const backup = await storeBackup(admin.id, ["m-a"]); const queued = await requestRestore(backup.id); const { data } = (await queued.json()) as { data: { jobId: string } }; // A deadline one minute out: shorter than the transaction's own // two-minute floor, so the restore cannot promise to finish in time. await runBackupRestoreJob(data.jobId, { deadline: Date.now() + 60_000 }); const job = await readBackupRestoreJob(data.jobId); expect(job?.status).toBe("failed"); expect(job?.failure?.code).toBe("time_budget"); // Both halves of the work are named: what the file carries and what the // account holds now, since the clearing of the latter costs time too. expect(job?.failure?.message).toContain("1 reading in the backup"); expect(job?.failure?.message).toContain("1 reading this account holds now"); expect(await measurementIdsOf(admin.id)).toEqual(["live-1"]); }); });